gooddata 0.6.16 → 0.6.17

data/lib/gooddata/rest/client.rb

@@ -20,6 +20,8 @@ module GoodData
       # Constants
       #################################
       DEFAULT_CONNECTION_IMPLEMENTATION = GoodData::Rest::Connection
+      DEFAULT_SLEEP_INTERVAL = 10
+      DEFAULT_POLL_TIME_LIMIT = 5 * 60 * 60 # 5 hours
 
       #################################
       # Class variables
@@ -207,6 +209,10 @@ module GoodData
         @stats
       end
 
+      def generate_request_id
+        @connection.generate_request_id
+      end
+
       #######################
       # Rest
       #######################
@@ -231,8 +237,9 @@ module GoodData
         project = GoodData::Project[p, opts]
         fail ArgumentError, 'Wrong :project specified' if project.nil?
 
-        u = URI(project.links['uploads'])
-        URI.join(u.to_s.chomp(u.path.to_s), '/project-uploads/', "#{project.pid}/")
+        url = project.links['uploads']
+        fail 'Project WebDAV not supported in this Data Center' unless url
+        url
       end
 
       def user_webdav_path(opts = { :project => GoodData.project })
@@ -257,17 +264,13 @@ module GoodData
       # @return [Hash] Result of polling
       def poll_on_code(link, options = {})
         code = options[:code] || 202
-        sleep_interval = options[:sleep_interval] || DEFAULT_SLEEP_INTERVAL
-        response = get(link, :process => false)
+        process = options[:process]
 
-        while response.code == code
-          sleep sleep_interval
-          GoodData::Rest::Client.retryable(:tries => 3, :refresh_token => proc { connection.refresh_token }) do
-            sleep sleep_interval
-            response = get(link, :process => false)
-          end
+        response = poll_on_response(link, options.merge(:process => false)) do |resp|
+          resp.code == code
         end
-        if options[:process] == false
+
+        if process == false
           response
         else
           get(link)
@@ -285,12 +288,23 @@ module GoodData
       # @return [Hash] Result of polling
       def poll_on_response(link, options = {}, &bl)
         sleep_interval = options[:sleep_interval] || DEFAULT_SLEEP_INTERVAL
-        response = get(link)
+        time_limit = options[:time_limit] || DEFAULT_POLL_TIME_LIMIT
+
+        # by default the response is processed
+        process = options[:process]
+
+        # get the first status and start the timer
+        response = get(link, :process => process)
+        poll_start = Time.now
+
         while bl.call(response)
+          if time_limit && (Time.now - poll_start > time_limit)
+            fail "The time limit #{time_limit} secs for polling on #{link} is over"
+          end
           sleep sleep_interval
           GoodData::Rest::Client.retryable(:tries => 3, :refresh_token => proc { connection.refresh_token }) do
             sleep sleep_interval
-            response = get(link)
+            response = get(link, :process => process)
           end
         end
         response

data/lib/gooddata/rest/connection.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 require 'terminal-table'
+require 'securerandom'
 
 require_relative '../version'
 require_relative '../exceptions/exceptions'
@@ -9,7 +10,7 @@ module GoodData
   module Rest
     # Wrapper of low-level HTTP/REST client/library
     class Connection
-      DEFAULT_URL = 'https://secure.gooddata.com'
+      DEFAULT_URL = ENV['GD_SERVER'] || 'https://secure.gooddata.com'
       LOGIN_PATH = '/gdc/account/login'
       TOKEN_PATH = '/gdc/account/token'
       KEYS_TO_SCRUB = [:password, :verifyPassword, :authorizationToken]
@@ -20,9 +21,13 @@ module GoodData
         :user_agent => GoodData.gem_version_string
       }
 
+      DEFAULT_WEBDAV_HEADERS = {
+        :user_agent => GoodData.gem_version_string
+      }
+
       DEFAULT_LOGIN_PAYLOAD = {
         :headers => DEFAULT_HEADERS,
-        :verify_ssl => OpenSSL::SSL::VERIFY_NONE
+        :verify_ssl => true
       }
 
       RETRYABLE_ERRORS = [
@@ -51,7 +56,8 @@ module GoodData
         def retryable(options = {}, &_block)
           opts = { :tries => 1, :on => RETRYABLE_ERRORS }.merge(options)
 
-          retry_exception, retries = opts[:on], opts[:tries]
+          retry_exception = opts[:on]
+          retries = opts[:tries]
 
           unless retry_exception.is_a?(Array)
             retry_exception = [retry_exception]
@@ -79,7 +85,10 @@ module GoodData
         end
       end
 
-      attr_reader :cookies
+      attr_reader :request_params
+
+      # backward compatibility
+      alias_method :cookies, :request_params
       attr_reader :stats
       attr_reader :user
 
@@ -87,13 +96,12 @@ module GoodData
         @stats = {}
         @opts = opts
 
-        @headers = DEFAULT_HEADERS.dup
+        headers = opts[:headers] || {}
+        @webdav_headers = DEFAULT_WEBDAV_HEADERS.merge(headers)
+
         @user = nil
         @server = nil
-
         @opts = opts
-        headers = opts[:headers] || {}
-        @headers.merge! headers
 
         # Initialize cookies
         reset_cookies!
@@ -104,7 +112,11 @@ module GoodData
       # Connect using username and password
       def connect(username, password, options = {})
         server = options[:server] || DEFAULT_URL
-        @server = RestClient::Resource.new server, DEFAULT_LOGIN_PAYLOAD
+
+        options = DEFAULT_LOGIN_PAYLOAD.merge(options)
+        headers = options[:headers] || {}
+
+        @server = RestClient::Resource.new server, DEFAULT_LOGIN_PAYLOAD.merge(headers)
 
         # Install at_exit handler first
         unless @at_exit_handler_installed
@@ -125,6 +137,7 @@ module GoodData
           refresh_token :dont_reauth => true
         else
           credentials = Connection.construct_login_payload(username, password)
+          generate_session_id
           @auth = post(LOGIN_PATH, credentials)['userLogin']
 
           refresh_token :dont_reauth => true
@@ -138,6 +151,7 @@ module GoodData
         url = @auth['state']
 
         begin
+          clear_session_id
           delete url if url
         rescue RestClient::Unauthorized
           GoodData.logger.info 'Already disconnected'
@@ -189,7 +203,7 @@ module GoodData
             :url => url
           }.merge(cookies)
 
-          if where.is_a?(IO)
+          if where.is_a?(IO) || where.is_a?(StringIO)
             RestClient::Request.execute(raw) do |chunk, _x, response|
               if response.code.to_s != '200'
                 fail ArgumentError, "Error downloading #{url}. Got response: #{response.code} #{response} #{response.body}"
@@ -238,7 +252,15 @@ module GoodData
       def delete(uri, options = {})
         GoodData.logger.debug "DELETE: #{@server.url}#{uri}"
         profile "DELETE #{uri}" do
-          b = proc { @server[uri].delete cookies }
+          b = proc do
+            begin
+              @server[uri].delete(fresh_request_params(options[:request_id]))
+            rescue RestClient::Exception => e
+              # log the error if it happens
+              GoodData.logger.error(e.inspect)
+              raise e
+            end
+          end
           process_response(options, &b)
         end
       end
@@ -249,7 +271,15 @@ module GoodData
       def get(uri, options = {}, &user_block)
         GoodData.logger.debug "GET: #{@server.url}#{uri}"
         profile "GET #{uri}" do
-          b = proc { @server[uri].get(cookies, &user_block) }
+          b = proc do
+            begin
+              @server[uri].get(fresh_request_params(options[:request_id]), &user_block)
+            rescue RestClient::Exception => e
+              # log the error if it happens
+              GoodData.logger.error(e.inspect)
+              raise e
+            end
+          end
           process_response(options, &b)
         end
       end
@@ -261,7 +291,15 @@ module GoodData
         payload = data.is_a?(Hash) ? data.to_json : data
         GoodData.logger.debug "PUT: #{@server.url}#{uri}, #{scrub_params(data, KEYS_TO_SCRUB)}"
         profile "PUT #{uri}" do
-          b = proc { @server[uri].put payload, cookies }
+          b = proc do
+            begin
+              @server[uri].put(payload, fresh_request_params(options[:request_id]))
+            rescue RestClient::Exception => e
+              # log the error if it happens
+              GoodData.logger.error(e.inspect)
+              raise e
+            end
+          end
           process_response(options, &b)
         end
       end
@@ -273,7 +311,15 @@ module GoodData
         GoodData.logger.debug "POST: #{@server.url}#{uri}, #{scrub_params(data, KEYS_TO_SCRUB)}"
         profile "POST #{uri}" do
           payload = data.is_a?(Hash) ? data.to_json : data
-          b = proc { @server[uri].post payload, cookies }
+          b = proc do
+            begin
+              @server[uri].post(payload, fresh_request_params(options[:request_id]))
+            rescue RestClient::Exception => e
+              # log the error if it happens
+              GoodData.logger.error(e.inspect)
+              raise e
+            end
+          end
           process_response(options, &b)
         end
       end
@@ -282,7 +328,7 @@ module GoodData
       #
       # @return uri [String] SST token
       def sst_token
-        cookies[:cookies]['GDCAuthSST']
+        request_params[:cookies]['GDCAuthSST']
       end
 
       def stats_table(values = stats)
@@ -306,7 +352,7 @@ module GoodData
       #
       # @return uri [String] TT token
       def tt_token
-        cookies[:cookies]['GDCAuthTT']
+        request_params[:cookies]['GDCAuthTT']
       end
 
       # Uploads a file to GoodData server
@@ -315,7 +361,7 @@ module GoodData
           puts "uploading the file #{uri}"
 
           to_upload = File.new(filename)
-          cookies_str = cookies[:cookies].map { |cookie| "#{cookie[0]}=#{cookie[1]}" }.join(';')
+          cookies_str = request_params[:cookies].map { |cookie| "#{cookie[0]}=#{cookie[1]}" }.join(';')
           req = Net::HTTP::Put.new(uri.path, 'User-Agent' => GoodData.gem_version_string, 'Cookie' => cookies_str)
           req.content_length = to_upload.size
           req.body_stream = to_upload
@@ -338,10 +384,10 @@ module GoodData
             raw = {
               :method => method,
               :url => url,
-              :headers => @headers
-            }
+              :headers => @webdav_headers
+            }.merge(cookies)
             begin
-              RestClient::Request.execute(raw.merge(cookies))
+              RestClient::Request.execute(raw)
             rescue RestClient::Exception => e
               false if e.http_code == 404
             end
@@ -363,7 +409,7 @@ module GoodData
             raw = {
               :method => method,
               :url => url,
-              :headers => @headers
+              :headers => @webdav_headers
             }.merge(cookies)
             RestClient::Request.execute(raw)
           end
@@ -384,10 +430,43 @@ module GoodData
         do_stream_file URI.join(url, CGI.escape(webdav_filename)), file
       end
 
+      def generate_request_id
+        "#{session_id}:#{call_id}"
+      end
+
       private
 
+      ID_LENGTH = 16
+
+      def generate_string
+        SecureRandom.urlsafe_base64(ID_LENGTH)
+      end
+
+      # generate session id to be passed as the first part to
+      # x_gdc_request header
+      def session_id
+        @session_id ||= generate_string
+      end
+
+      def call_id
+        generate_string
+      end
+
+      def generate_session_id
+        @session_id = generate_string
+      end
+
+      def clear_session_id
+        @session_id = nil
+      end
+
+      # request heders with freshly generated request id
+      def fresh_request_params(request_id = nil)
+        @request_params.merge(:x_gdc_request => request_id || generate_request_id)
+      end
+
       def merge_cookies!(cookies)
-        @cookies[:cookies].merge! cookies
+        @request_params[:cookies].merge! cookies
       end
 
       def process_response(options = {}, &block)
@@ -447,7 +526,7 @@ module GoodData
       end
 
       def reset_cookies!
-        @cookies = { :cookies => {} }
+        @request_params = { :cookies => {} }
       end
 
       def scrub_params(params, keys)

data/lib/gooddata/version.rb

@@ -2,7 +2,7 @@
 
 # GoodData Module
 module GoodData
-  VERSION = '0.6.16'
+  VERSION = '0.6.17'
 
   class << self
     # Version

data/spec/data/gd_gse_data_blueprint.json

@@ -294,6 +294,7 @@
     "columns": [{
       "type": "anchor",
       "name": "techoppanalysis",
+      "folder": "Opportunity Benchmark",
       "title": "Tech Opp. Analysis",
       "gd_data_type": "VARCHAR(128)",
       "gd_type": "GDC.text"

data/spec/data/test_project_model_spec.json

@@ -4,10 +4,12 @@
     {
       "type": "dataset",
       "name": "repos",
+      "folder": "Foldered Repos",
       "columns": [
         {
           "type": "anchor",
-          "name": "repo_id"
+          "name": "repo_id",
+          "description": "This is anchor description"
         },
         {
           "type": "label",
@@ -42,7 +44,8 @@
       "columns": [
         {
           "type": "fact",
-          "name": "lines_changed"
+          "name": "lines_changed",
+          "description": "Fact description"
         },
         {
           "type": "date",

data/spec/data/wire_models/model_view.json

@@ -31,6 +31,7 @@
               "fact": {
                 "identifier": "fact.stage_history.stage_velocity",
                 "title": "Stage Velocity",
+                "description": "Velocity description",
                 "dataType": "DECIMAL(12,2)"
               }
             }, {
@@ -1498,6 +1499,7 @@
               "attribute": {
                 "identifier": "attr.opp_owner.region",
                 "title": "Opp. Owner Region",
+                "description": "Owner Region description",
                 "folder": "Opp. Owner",
                 "labels": [{
                   "label": {
@@ -1601,6 +1603,7 @@
               "attribute": {
                 "identifier": "attr.opportunity.id",
                 "title": "Opportunity",
+                "description": "This is opportunity attribute description",
                 "folder": "Opportunity",
                 "labels": [{
                   "label": {

data/spec/data/wire_test_project.json

@@ -11,6 +11,8 @@
                 "attribute": {
                   "identifier": "attr.repos.repo_id",
                   "title": "Repo",
+                  "description": "This is anchor description",
+                  "folder": "Foldered Repos",
                   "labels": [
                     {
                       "label": {
@@ -37,6 +39,7 @@
                   "attribute": {
                     "identifier": "attr.repos.department",
                     "title": "Department",
+                    "folder": "Foldered Repos",
                     "labels": [
                       {
                         "label": {
@@ -67,6 +70,7 @@
                 "attribute": {
                   "identifier": "attr.devs.dev_id",
                   "title": "Dev",
+                  "folder": "Devs",
                   "labels": [
                     {
                       "label": {
@@ -106,7 +110,8 @@
               "anchor": {
                 "attribute": {
                   "identifier": "attr.commits.factsof",
-                  "title": "Records of Commits"
+                  "title": "Records of Commits",
+                  "folder": "Commits"
                 }
               },
               "attributes": [
@@ -117,6 +122,8 @@
                   "fact": {
                     "identifier": "fact.commits.lines_changed",
                     "title": "Lines Changed",
+                    "description": "Fact description",
+                    "folder": "Commits",
                     "dataType": "INT"
                   }
                 }