gooddata 0.6.18 → 0.6.19

data/lib/gooddata/models/project_creator.rb

@@ -1,7 +1,7 @@
 # encoding: UTF-8
 
 require_relative 'project'
-require_relative 'project_blueprint'
+require_relative 'blueprint/project_blueprint'
 
 require 'open-uri'
 
@@ -10,19 +10,16 @@ module GoodData
     class ProjectCreator
       class << self
         def migrate(opts = {})
-          opts = { client: GoodData.connection, project: GoodData.project }.merge(opts)
+          opts = { client: GoodData.connection }.merge(opts)
           client = opts[:client]
           fail ArgumentError, 'No :client specified' if client.nil?
 
           spec = opts[:spec] || fail('You need to provide spec for migration')
           bp = ProjectBlueprint.new(spec)
-          spec = bp.to_hash
-
           fail GoodData::ValidationError, "Blueprint is invalid #{bp.validate.inspect}" unless bp.valid?
+          spec = bp.to_hash
 
-          token = opts[:token]
-          project = opts[:project] || GoodData::Project.create(:title => spec[:title], :auth_token => token, :client => client)
-          fail('You need to specify token for project creation') if token.nil? && project.nil?
+          project = opts[:project] || client.create_project(opts.merge(:title => spec[:title], :client => client, :environment => opts[:environment]))
 
           begin
             migrate_datasets(spec, opts.merge(project: project, client: client))
@@ -46,10 +43,7 @@ module GoodData
 
           project = client.projects(p)
           fail ArgumentError, 'Wrong :project specified' if project.nil?
-
           bp = ProjectBlueprint.new(spec)
-          # schema = Schema.load(schema) unless schema.respond_to?(:to_maql_create)
-          # project = GoodData.project unless project
           uri = "/gdc/projects/#{project.pid}/model/diff"
           result = client.post(uri, bp.to_wire)
 
@@ -72,11 +66,6 @@ module GoodData
               result = project.execute_maql(chunk)
               fail 'Creating dataset failed' if result['wTaskStatus']['status'] == 'ERROR'
             end
-            bp.datasets.zip(GoodData::Model::ToManifest.to_manifest(bp.to_hash)).each do |ds|
-              dataset = ds[0]
-              manifest = ds[1]
-              GoodData::ProjectMetadata["manifest_#{dataset.name}", :client => client, :project => project] = manifest.to_json
-            end
           end
           chunks
         end

data/lib/gooddata/models/schedule.rb

@@ -103,6 +103,7 @@ module GoodData
 
         schedule = c.create(GoodData::Schedule, GoodData::Helpers.stringify_keys_deep!(SCHEDULE_TEMPLATE.deep_dup), client: c, project: p)
 
+        schedule.name = options[:name]
         schedule.hidden_params = options[:hidden_params]
         schedule.set_trigger(trigger)
         schedule.params = default_opts[:params].merge(options[:params] || {})

data/lib/gooddata/models/user_filters/user_filter_builder.rb

@@ -216,9 +216,9 @@ module GoodData
                    elsif filter[:over] && filter[:to]
                      over = attr_cache[filter[:over]]
                      to = attr_cache[filter[:to]]
-                     "([#{label.attribute_uri}] IN (#{ element_uris.compact.sort.map { |e| '[' + e + ']' }.join(', ') })) OVER [#{over && over.uri}] TO [#{to && to.uri}]"
+                     "([#{label.attribute_uri}] IN (#{element_uris.compact.sort.map { |e| '[' + e + ']' }.join(', ')})) OVER [#{over && over.uri}] TO [#{to && to.uri}]"
                    else
-                     "[#{label.attribute_uri}] IN (#{ element_uris.compact.sort.map { |e| '[' + e + ']' }.join(', ') })"
+                     "[#{label.attribute_uri}] IN (#{element_uris.compact.sort.map { |e| '[' + e + ']' }.join(', ')})"
                    end
       [expression, errors]
     end

data/lib/gooddata/rest/client.rb

@@ -61,18 +61,21 @@ module GoodData
         # @param username [String] Username to be used for authentication
         # @param password [String] Password to be used for authentication
         # @return [GoodData::Rest::Client] Client
-        def connect(username, password, opts = { :verify_ssl => true })
+        def connect(username, password, opts = { verify_ssl: true })
           if username.nil? && password.nil?
             username = ENV['GD_GEM_USER']
             password = ENV['GD_GEM_PASSWORD']
           end
 
+          username = username.symbolize_keys if username.is_a?(Hash)
+
           new_opts = opts.dup
           if username.is_a?(Hash) && username.key?(:sst_token)
-            new_opts[:sst_token] = username[:sst_token]
+            new_opts = username
           elsif username.is_a? Hash
             new_opts[:username] = username[:login] || username[:user] || username[:username]
             new_opts[:password] = username[:password]
+            new_opts[:verify_ssl] = username[:verify_ssl] if username[:verify_ssl] == false || !username[:verify_ssl].blank?
           elsif username.nil? && password.nil? && (opts.nil? || opts.empty?)
             new_opts = Helpers::AuthHelper.read_credentials
           else
@@ -100,14 +103,14 @@ module GoodData
         end
 
         def disconnect
-          if @@instance # rubocop:disable ClassVars, Style/GuardClause
-            @@instance.disconnect # rubocop:disable ClassVars
+          if @@instance # rubocop:disable Style/GuardClause
+            @@instance.disconnect
             @@instance = nil # rubocop:disable ClassVars
           end
         end
 
         def connection
-          @@instance # rubocop:disable ClassVars
+          @@instance
         end
 
         # Retry block if exception thrown
@@ -146,7 +149,7 @@ module GoodData
       end
 
       def create_project_from_blueprint(blueprint, options = {})
-        GoodData::Model::ProjectCreator.migrate(spec: blueprint, token: options[:auth_token], client: self)
+        GoodData::Model::ProjectCreator.migrate(options.merge(spec: blueprint, client: self))
       end
 
       def domain(domain_name)
@@ -209,7 +212,7 @@ module GoodData
         @stats = true
       end
 
-      def stats_on? # rubocop:disable Style/TrivialAccessors
+      def stats_on?
         @stats
       end
 
@@ -234,7 +237,7 @@ module GoodData
         @connection.get uri, opts, & block
       end
 
-      def project_webdav_path(opts = { :project => GoodData.project })
+      def project_webdav_path(opts = { project: GoodData.project })
         p = opts[:project]
         fail ArgumentError, 'No :project specified' if p.nil?
 
@@ -246,7 +249,7 @@ module GoodData
         url
       end
 
-      def user_webdav_path(opts = { :project => GoodData.project })
+      def user_webdav_path(opts = { project: GoodData.project })
         p = opts[:project]
         fail ArgumentError, 'No :project specified' if p.nil?
 
@@ -294,21 +297,18 @@ module GoodData
         sleep_interval = options[:sleep_interval] || DEFAULT_SLEEP_INTERVAL
         time_limit = options[:time_limit] || DEFAULT_POLL_TIME_LIMIT
 
-        # by default the response is processed
-        process = options[:process]
-
         # get the first status and start the timer
-        response = get(link, :process => process)
+        response = get(link, options)
         poll_start = Time.now
 
         while bl.call(response)
-          if time_limit && (Time.now - poll_start > time_limit)
-            fail "The time limit #{time_limit} secs for polling on #{link} is over"
+          limit_breached = time_limit && (Time.now - poll_start > time_limit)
+          if limit_breached
+            fail ExecutionLimitExceeded, "The time limit #{time_limit} secs for polling on #{link} is over"
           end
           sleep sleep_interval
           GoodData::Rest::Client.retryable(:tries => 3, :refresh_token => proc { connection.refresh_token }) do
-            sleep sleep_interval
-            response = get(link, :process => process)
+            response = get(link, options)
           end
         end
         response
@@ -345,7 +345,7 @@ module GoodData
         @connection.download source_relative_path, target_file_path, options
       end
 
-      def download_from_user_webdav(source_relative_path, target_file_path, options = { :client => GoodData.client, :project => project })
+      def download_from_user_webdav(source_relative_path, target_file_path, options = { client: GoodData.client, project: project })
         download(source_relative_path, target_file_path, options.merge(:directory => options[:directory],
                                                                        :staging_url => get_user_webdav_url(options)))
       end

data/lib/gooddata/rest/connection.rb

@@ -38,6 +38,9 @@ module GoodData
         Timeout::Error
       ]
 
+      RETRIES_ON_TOO_MANY_REQUESTS_ERROR = 10
+      RETRY_TIME_INITIAL_VALUE = 1
+      RETRY_TIME_COEFFICIENT = 1.5
       RETRYABLE_ERRORS << Net::ReadTimeout if Net.const_defined?(:ReadTimeout)
 
       class << self
@@ -58,12 +61,13 @@ module GoodData
 
           retry_exception = opts[:on]
           retries = opts[:tries]
+          too_many_requests_tries = RETRIES_ON_TOO_MANY_REQUESTS_ERROR
 
           unless retry_exception.is_a?(Array)
             retry_exception = [retry_exception]
           end
 
-          retry_time = 1
+          retry_time = RETRY_TIME_INITIAL_VALUE
           begin
             return yield
           rescue RestClient::Unauthorized, RestClient::Forbidden => e # , RestClient::Unauthorized => e
@@ -71,16 +75,16 @@ module GoodData
             raise e if options[:dont_reauth]
             options[:refresh_token].call # (dont_reauth: true)
             retry if (retries -= 1) > 0
-          rescue RestClient::TooManyRequests
+          rescue RestClient::TooManyRequests, RestClient::ServiceUnavailable
             GoodData.logger.warn "Too many requests, retrying in #{retry_time} seconds"
             sleep retry_time
-            retry_time *= 1.5
-            retry
+            retry_time *= RETRY_TIME_COEFFICIENT
+            # 10 requests with 1.5 coefficent should take ~ 3 mins to finish
+            retry if (too_many_requests_tries -= 1) > 1
           rescue *retry_exception => e
             GoodData.logger.warn e.inspect
-            retry if (retries -= 1) > 0
+            retry if (retries -= 1) > 1
           end
-
           yield
         end
       end
@@ -89,12 +93,12 @@ module GoodData
 
       # backward compatibility
       alias_method :cookies, :request_params
+      attr_reader :server
       attr_reader :stats
       attr_reader :user
 
       def initialize(opts)
         @stats = {}
-        @opts = opts
 
         headers = opts[:headers] || {}
         @webdav_headers = DEFAULT_WEBDAV_HEADERS.merge(headers)
@@ -112,11 +116,12 @@ module GoodData
       # Connect using username and password
       def connect(username, password, options = {})
         server = options[:server] || DEFAULT_URL
-
         options = DEFAULT_LOGIN_PAYLOAD.merge(options)
         headers = options[:headers] || {}
 
-        @server = RestClient::Resource.new server, DEFAULT_LOGIN_PAYLOAD.merge(headers)
+        options = options.merge(headers)
+
+        @server = RestClient::Resource.new server, options
 
         # Install at_exit handler first
         unless @at_exit_handler_installed
@@ -200,7 +205,8 @@ module GoodData
               :user_agent => GoodData.gem_version_string
             },
             :method => :get,
-            :url => url
+            :url => url,
+            :verify_ssl => (@opts[:verify_ssl] == false || @opts[:verify_ssl] == OpenSSL::SSL::VERIFY_NONE) ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
           }.merge(cookies)
 
           if where.is_a?(IO) || where.is_a?(StringIO)
@@ -250,14 +256,16 @@ module GoodData
       #
       # @param uri [String] Target URI
       def delete(uri, options = {})
+        options = log_info(options)
         GoodData.logger.debug "DELETE: #{@server.url}#{uri}"
         profile "DELETE #{uri}" do
           b = proc do
+            params = fresh_request_params(options[:request_id])
             begin
-              @server[uri].delete(fresh_request_params(options[:request_id]))
+              @server[uri].delete(params)
             rescue RestClient::Exception => e
               # log the error if it happens
-              GoodData.logger.error(e.inspect)
+              GoodData.logger.error(format_error(e, params))
               raise e
             end
           end
@@ -269,14 +277,16 @@ module GoodData
       #
       # @param uri [String] Target URI
       def get(uri, options = {}, &user_block)
+        options = log_info(options)
         GoodData.logger.debug "GET: #{@server.url}#{uri}"
         profile "GET #{uri}" do
           b = proc do
+            params = fresh_request_params(options[:request_id])
             begin
-              @server[uri].get(fresh_request_params(options[:request_id]), &user_block)
+              @server[uri].get(params, &user_block)
             rescue RestClient::Exception => e
               # log the error if it happens
-              GoodData.logger.error(e.inspect)
+              GoodData.logger.error(format_error(e, params))
               raise e
             end
           end
@@ -288,15 +298,17 @@ module GoodData
       #
       # @param uri [String] Target URI
       def put(uri, data, options = {})
+        options = log_info(options)
         payload = data.is_a?(Hash) ? data.to_json : data
         GoodData.logger.debug "PUT: #{@server.url}#{uri}, #{scrub_params(data, KEYS_TO_SCRUB)}"
         profile "PUT #{uri}" do
           b = proc do
+            params = fresh_request_params(options[:request_id])
             begin
-              @server[uri].put(payload, fresh_request_params(options[:request_id]))
+              @server[uri].put(payload, params)
             rescue RestClient::Exception => e
               # log the error if it happens
-              GoodData.logger.error(e.inspect)
+              GoodData.logger.error(format_error(e, params))
               raise e
             end
           end
@@ -308,15 +320,17 @@ module GoodData
       #
       # @param uri [String] Target URI
       def post(uri, data, options = {})
+        options = log_info(options)
         GoodData.logger.debug "POST: #{@server.url}#{uri}, #{scrub_params(data, KEYS_TO_SCRUB)}"
         profile "POST #{uri}" do
           payload = data.is_a?(Hash) ? data.to_json : data
           b = proc do
+            params = fresh_request_params(options[:request_id])
             begin
-              @server[uri].post(payload, fresh_request_params(options[:request_id]))
+              @server[uri].post(payload, params)
             rescue RestClient::Exception => e
               # log the error if it happens
-              GoodData.logger.error(e.inspect)
+              GoodData.logger.error(format_error(e, params))
               raise e
             end
           end
@@ -357,68 +371,6 @@ module GoodData
 
       # Uploads a file to GoodData server
       def upload(file, options = {})
-        def do_stream_file(uri, filename, _options = {})
-          puts "uploading the file #{uri}"
-
-          to_upload = File.new(filename)
-          cookies_str = request_params[:cookies].map { |cookie| "#{cookie[0]}=#{cookie[1]}" }.join(';')
-          req = Net::HTTP::Put.new(uri.path, 'User-Agent' => GoodData.gem_version_string, 'Cookie' => cookies_str)
-          req.content_length = to_upload.size
-          req.body_stream = to_upload
-          http = Net::HTTP.new(uri.host, uri.port)
-          http.use_ssl = true
-          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-
-          response = nil
-          GoodData::Rest::Connection.retryable(:tries => 2, :refresh_token => proc { refresh_token }) do
-            response = http.start { |client| client.request(req) }
-          end
-          response
-        end
-
-        def webdav_dir_exists?(url)
-          method = :get
-          GoodData.logger.debug "#{method}: #{url}"
-
-          b = proc do
-            raw = {
-              :method => method,
-              :url => url,
-              :headers => @webdav_headers
-            }.merge(cookies)
-            begin
-              RestClient::Request.execute(raw)
-            rescue RestClient::Exception => e
-              false if e.http_code == 404
-            end
-          end
-
-          res = nil
-          GoodData::Rest::Connection.retryable(:tries => 2, :refresh_token => proc { refresh_token }) do
-            res = b.call
-          end
-          res
-        end
-
-        def create_webdav_dir_if_needed(url)
-          return if webdav_dir_exists?(url)
-
-          method = :mkcol
-          GoodData.logger.debug "#{method}: #{url}"
-          b = proc do
-            raw = {
-              :method => method,
-              :url => url,
-              :headers => @webdav_headers
-            }.merge(cookies)
-            RestClient::Request.execute(raw)
-          end
-
-          GoodData::Rest::Connection.retryable(:tries => 2, :refresh_token => proc { refresh_token }) do
-            b.call
-          end
-        end
-
         dir = options[:directory] || ''
         staging_uri = options[:staging_url].to_s
         url = dir.empty? ? staging_uri : URI.join(staging_uri, "#{dir}/").to_s
@@ -438,6 +390,49 @@ module GoodData
 
       ID_LENGTH = 16
 
+      def create_webdav_dir_if_needed(url)
+        return if webdav_dir_exists?(url)
+
+        method = :mkcol
+        GoodData.logger.debug "#{method}: #{url}"
+        b = proc do
+          raw = {
+            :method => method,
+            :url => url,
+            :headers => @webdav_headers,
+            :verify_ssl => (@opts[:verify_ssl] == false || @opts[:verify_ssl] == OpenSSL::SSL::VERIFY_NONE) ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
+          }.merge(cookies)
+          RestClient::Request.execute(raw)
+        end
+
+        GoodData::Rest::Connection.retryable(:tries => 2, :refresh_token => proc { refresh_token }) do
+          b.call
+        end
+      end
+
+      def do_stream_file(uri, filename, _options = {})
+        GoodData.logger.info "Uploading file user storage #{uri}"
+
+        to_upload = File.new(filename)
+        cookies_str = request_params[:cookies].map { |cookie| "#{cookie[0]}=#{cookie[1]}" }.join(';')
+        req = Net::HTTP::Put.new(uri.path, 'User-Agent' => GoodData.gem_version_string, 'Cookie' => cookies_str)
+        req.content_length = to_upload.size
+        req.body_stream = to_upload
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        http.verify_mode = (@opts[:verify_ssl] == false || @opts[:verify_ssl] == OpenSSL::SSL::VERIFY_NONE) ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
+
+        response = nil
+        GoodData::Rest::Connection.retryable(:tries => 2, :refresh_token => proc { refresh_token }) do
+          response = http.start { |client| client.request(req) }
+        end
+        response
+      end
+
+      def format_error(e, params)
+        "#{params[:x_gdc_request]} #{e.inspect}"
+      end
+
       def generate_string
         SecureRandom.urlsafe_base64(ID_LENGTH)
       end
@@ -460,6 +455,16 @@ module GoodData
         @session_id = nil
       end
 
+      # log info_message given in options and make sure request_id is there
+      def log_info(options)
+        # if info_message given, log it with request_id (given or generated)
+        if options[:info_message]
+          request_id = options[:request_id] || generate_request_id
+          GoodData.logger.info "#{options[:info_message]} Request id: #{request_id}"
+        end
+        options
+      end
+
       # request heders with freshly generated request id
       def fresh_request_params(request_id = nil)
         @request_params.merge(:x_gdc_request => request_id || generate_request_id)
@@ -470,20 +475,9 @@ module GoodData
       end
 
       def process_response(options = {}, &block)
-        # begin
-        #   # Simply try again when ConnectionReset, ConnectionRefused etc.. (see e.g. MSF-7591)
-        #   response = GoodData::Rest::Connection.retryable(:tries => 2, :refresh_token => Proc.new { refresh_token }) do
-        #     block.call
-        #   end
-        # rescue RestClient::Unauthorized
-        #   raise $ERROR_INFO if options[:dont_reauth]
-        #   GoodData::Rest::Connection.retryable(:tries => 2) do
-        #     refresh_token
-        #     response = block.call
-        #   end
-        # end
-
-        response = GoodData::Rest::Connection.retryable(:tries => 2, :refresh_token => proc { refresh_token unless options[:dont_reauth] }) do
+        retries = options[:tries] || 3
+
+        response = GoodData::Rest::Connection.retryable(:tries => retries, :refresh_token => proc { refresh_token unless options[:dont_reauth] }) do
           block.call
         end
 
@@ -596,6 +590,31 @@ module GoodData
 
         stats[title] = stat
       end
+
+      def webdav_dir_exists?(url)
+        method = :get
+        GoodData.logger.debug "#{method}: #{url}"
+
+        b = proc do
+          raw = {
+            :method => method,
+            :url => url,
+            :headers => @webdav_headers,
+            :verify_ssl => (@opts[:verify_ssl] == false || @opts[:verify_ssl] == OpenSSL::SSL::VERIFY_NONE) ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
+          }.merge(cookies)
+          begin
+            RestClient::Request.execute(raw)
+          rescue RestClient::Exception => e
+            false if e.http_code == 404
+          end
+        end
+
+        res = nil
+        GoodData::Rest::Connection.retryable(:tries => 2, :refresh_token => proc { refresh_token }) do
+          res = b.call
+        end
+        res
+      end
     end
   end
 end