gooddata 0.6.10 → 0.6.11

data/lib/gooddata/models/schedule.rb

@@ -2,6 +2,10 @@
 
 require_relative '../rest/resource'
 require_relative '../extensions/hash'
+require_relative '../mixins/rest_resource'
+require_relative '../helpers/global_helpers'
+
+require_relative 'execution'
 
 module GoodData
   class Schedule < Rest::Resource
@@ -11,6 +15,9 @@ module GoodData
     alias_method :raw_data, :json
     alias_method :to_hash, :json
 
+    include GoodData::Mixin::RestResource
+    root_key :schedule
+
     class << self
       # Looks for schedule
       # @param id [String] URL, ID of schedule or :all
@@ -58,11 +65,11 @@ module GoodData
       # Creates new schedules from parameters passed
       #
       # @param process_id [String] Process ID
-      # @param cron [String] Cron Settings
+      # @param trigger [String|GoodData::Schedule] Trigger of schedule. Can be cron string or reference to another schedule.
       # @param executable [String] Execution executable
       # @param options [Hash] Optional options
       # @return [GoodData::Schedule] New GoodData::Schedule instance
-      def create(process_id, cron, executable, options = {})
+      def create(process_id, trigger, executable, options = {})
         c = client(options)
         fail ArgumentError, 'No :client specified' if c.nil?
 
@@ -75,50 +82,39 @@ module GoodData
         default_opts = {
           :type => 'MSETL',
           :timezone => 'UTC',
-          :cron => cron,
           :params => {
-            :process_id => process_id,
-            :executable => executable
+            :PROCESS_ID => process_id,
+            :EXECUTABLE => executable
           },
-          :hidden_params => {},
+          :hiddenParams => {},
           :reschedule => options[:reschedule] || 0
         }
 
-        inject_schema = {
-          :hidden_params => 'hiddenParams'
-        }
-
-        inject_params = {
-          :process_id => 'PROCESS_ID',
-          :executable => 'EXECUTABLE'
-        }
-
-        default_params = default_opts[:params].reduce({}) do |new_hash, (k, v)|
-          key = inject_params[k] || k
-          new_hash[key] = v
-          new_hash
+        if trigger =~ /[a-fA-Z0-9]{24}/
+          default_opts[:triggerScheduleId] = trigger
+        elsif trigger.is_a?(GoodData::Schedule)
+          default_opts[:triggerScheduleId] = trigger.obj_id
+        else
+          default_opts[:cron] = trigger
         end
 
-        default = default_opts.reduce({}) do |new_hash, (k, v)|
-          key = inject_schema[k] || k
-          new_hash[key] = v
-          new_hash
+        if options.key?(:hidden_params)
+          options[:hiddenParams] = options[:hidden_params]
+          options.delete :hidden_params
         end
 
-        default[:params] = default_params
-
         json = {
-          'schedule' => default.merge(options.except(:project, :client))
+          'schedule' => default_opts.deep_merge(options.except(:project, :client))
         }
 
-        tmp = json['schedule'][:params]['PROCESS_ID']
+        tmp = json['schedule'][:params][:PROCESS_ID]
         fail 'Process ID has to be provided' if tmp.nil? || tmp.empty?
 
-        tmp = json['schedule'][:params]['EXECUTABLE']
+        tmp = json['schedule'][:params][:EXECUTABLE]
         fail 'Executable has to be provided' if tmp.nil? || tmp.empty?
 
-        tmp = json['schedule'][:cron]
-        fail 'Cron schedule has to be provided' if tmp.nil? || tmp.empty?
+        tmp = json['schedule'][:cron] || json['schedule'][:triggerScheduleId]
+        fail 'trigger schedule has to be provided' if !tmp || tmp.nil? || tmp.empty?
 
         tmp = json['schedule'][:timezone]
         fail 'A timezone has to be provided' if tmp.nil? || tmp.empty?
@@ -126,6 +122,16 @@ module GoodData
         tmp = json['schedule'][:type]
         fail 'Schedule type has to be provided' if tmp.nil? || tmp.empty?
 
+        params = json['schedule'][:params]
+        params = GoodData::Helpers.encode_params(params, false)
+        json['schedule'][:params] = params
+
+        hidden_params = json['schedule'][:hiddenParams]
+        if hidden_params && !hidden_params.empty?
+          hidden_params = GoodData::Helpers.encode_params(json['schedule'][:hiddenParams], true)
+          json['schedule'][:hiddenParams] = hidden_params
+        end
+
         url = "/gdc/projects/#{project.pid}/schedules"
         res = c.post url, json
 
@@ -190,9 +196,10 @@ module GoodData
         :execution => {}
       }
       execution = client.post(execution_url, data)
-      client.poll_on_response(execution['execution']['links']['self']) do |body|
+      res = client.poll_on_response(execution['execution']['links']['self']) do |body|
         body['execution'] && (body['execution']['status'] == 'RUNNING' || body['execution']['status'] == 'SCHEDULED')
       end
+      client.create(GoodData::Execution, res, client: client, project: project)
     end
 
     # Returns execution URL
@@ -300,7 +307,7 @@ module GoodData
     #
     # @return [Array] Raw Executions JSON
     def executions
-      if @json
+      if @json # rubocop:disable Style/GuardClause
         url = @json['schedule']['links']['executions']
         res = client.get url
         res['executions']['items']

data/lib/gooddata/models/to_manifest.rb

@@ -10,7 +10,7 @@ module GoodData
       # @param attribute [Hash] Attribute or Anchor
       # @param mode [String] Mode of the load. Either FULL or INCREMENTAL
       # @return [Hash] Manifest for a particular reference
-      def self.attribute_to_manifest(project, dataset, a, mode)
+      def self.attribute_to_manifest(_project, dataset, a, mode)
         [{
           'referenceKey' => 1,
           'populates' => [GoodData::Model.identifier_for(dataset, a.merge(type: :primary_label))],
@@ -71,7 +71,7 @@ module GoodData
       # @param reference [Hash] Reference
       # @param mode [String] Mode of the load. Either FULL or INCREMENTAL
       # @return [Hash] Manifest for a particular date reference
-      def self.date_ref_to_manifest(project, dataset, reference, mode)
+      def self.date_ref_to_manifest(project, _dataset, reference, mode)
         referenced_dataset = ProjectBlueprint.find_date_dimension(project, reference[:dataset])
         [{
           'populates' => [GoodData::Model.identifier_for(referenced_dataset, type: :date_ref)],
@@ -89,7 +89,7 @@ module GoodData
       # @param fact [Hash] Fact
       # @param mode [String] Mode of the load. Either FULL or INCREMENTAL
       # @return [Hash] Manifest for a particular fact
-      def self.fact_to_manifest(project, dataset, fact, mode)
+      def self.fact_to_manifest(_project, dataset, fact, mode)
         [{
           'populates' => [GoodData::Model.identifier_for(dataset, fact)],
           'mode' => mode,
@@ -104,7 +104,7 @@ module GoodData
       # @param label [Hash] Label
       # @param mode [String] Mode of the load. Either FULL or INCREMENTAL
       # @return [Hash] Manifest for a particular label
-      def self.label_to_manifest(project, dataset, label, mode)
+      def self.label_to_manifest(_project, dataset, label, mode)
         a = DatasetBlueprint.attribute_for_label(dataset, label)
         [{
           'populates' => [GoodData::Model.identifier_for(dataset, label, a)],
@@ -145,7 +145,7 @@ module GoodData
       # @param reference [Hash] Reference
       # @param mode [String] Mode of the load. Either FULL or INCREMENTAL
       # @return [Hash] Manifest for a particular reference
-      def self.reference_to_manifest(project, dataset, reference, mode)
+      def self.reference_to_manifest(project, _dataset, reference, mode)
         referenced_dataset = ProjectBlueprint.find_dataset(project, reference[:dataset])
         anchor = DatasetBlueprint.anchor(referenced_dataset)
         [{

data/lib/gooddata/models/to_wire.rb

@@ -11,7 +11,7 @@ module GoodData
       # @param project [Hash] Project blueprint hash represenation
       # @param dataset [Hash] Dataset blueprint hash represenation
       # @return [Hash] Manifest for a particular reference
-      def self.anchor_to_wire(project, dataset)
+      def self.anchor_to_wire(_project, dataset)
         if DatasetBlueprint.anchor(dataset)
           attribute_to_wire(dataset, DatasetBlueprint.anchor(dataset))
         else
@@ -29,7 +29,7 @@ module GoodData
       # @param project [Hash] Project blueprint hash represenation
       # @param dataset [Hash] Dataset blueprint hash represenation
       # @return [Hash] Manifest for a particular reference
-      def self.attributes_to_wire(project, dataset)
+      def self.attributes_to_wire(_project, dataset)
         DatasetBlueprint.attributes(dataset).map do |a|
           attribute_to_wire(dataset, a)
         end
@@ -85,7 +85,7 @@ module GoodData
       # @param project [Hash] Project blueprint hash represenation
       # @param dataset [Hash] Dataset blueprint hash represenation
       # @return [Hash] Manifest for a particular reference
-      def self.date_dimensions_to_wire(project, dataset)
+      def self.date_dimensions_to_wire(_project, dataset)
         {
           dateDimension: {
             name: dataset[:name],

data/lib/gooddata/rest/client.rb

@@ -35,6 +35,7 @@ module GoodData
 
       # TODO: Decide if we need provide direct access to factory
       attr_reader :factory
+      attr_reader :opts
 
       include Mixin::Inspector
       inspector :object_id
@@ -58,7 +59,7 @@ module GoodData
         # @param username [String] Username to be used for authentication
         # @param password [String] Password to be used for authentication
         # @return [GoodData::Rest::Client] Client
-        def connect(username, password, opts = {})
+        def connect(username, password, opts = { :verify_ssl => true })
           new_opts = opts.dup
           if username.is_a?(Hash) && username.key?(:sst_token)
             new_opts[:sst_token] = username[:sst_token]
@@ -92,7 +93,7 @@ module GoodData
         end
 
         def disconnect
-          if @@instance # rubocop:disable ClassVars
+          if @@instance # rubocop:disable ClassVars, Style/GuardClause
             @@instance.disconnect # rubocop:disable ClassVars
             @@instance = nil # rubocop:disable ClassVars
           end
@@ -102,6 +103,21 @@ module GoodData
           @@instance # rubocop:disable ClassVars
         end
 
+        # Retry block if exception thrown
+        def retryable(options = {}, &_block)
+          opts = { :tries => 1, :on => Exception }.merge(options)
+
+          retry_exception, retries = opts[:on], opts[:tries]
+
+          begin
+            return yield
+          rescue retry_exception
+            retry if (retries -= 1) > 0
+          end
+
+          yield
+        end
+
         alias_method :client, :connection
       end
 
@@ -128,8 +144,8 @@ module GoodData
         @factory = ObjectFactory.new(self)
       end
 
-      def create_project(options = {})
-        GoodData::Project.create(title: 'Project for schedule testing', auth_token: ConnectionHelper::GD_PROJECT_TOKEN, client: self)
+      def create_project(options = { title: 'Project', auth_token: ENV['GD_PROJECT_TOKEN'] })
+        GoodData::Project.create(options.merge(client: self))
       end
 
       def create_project_from_blueprint(blueprint, options = {})
@@ -193,8 +209,8 @@ module GoodData
       # HTTP GET
       #
       # @param uri [String] Target URI
-      def get(uri, opts = {})
-        @connection.get uri, opts
+      def get(uri, opts = {}, & block)
+        @connection.get uri, opts, & block
       end
 
       # FIXME: Invstigate _file argument
@@ -237,7 +253,7 @@ module GoodData
 
         while response.code == code
           sleep sleep_interval
-          retryable(:tries => 3, :on => RestClient::InternalServerError) do
+          GoodData::Rest::Client.retryable(:tries => 3, :on => RestClient::InternalServerError) do
             sleep sleep_interval
             response = get(link, :process => false)
           end
@@ -263,7 +279,7 @@ module GoodData
         response = get(link)
         while bl.call(response)
           sleep sleep_interval
-          retryable(:tries => 3, :on => RestClient::InternalServerError) do
+          GoodData::Rest::Client.retryable(:tries => 3, :on => RestClient::InternalServerError) do
             sleep sleep_interval
             response = get(link)
           end
@@ -285,43 +301,60 @@ module GoodData
         @connection.post uri, data, opts
       end
 
-      # Retry blok if exception thrown
-      def retryable(options = {}, &block)
-        opts = { :tries => 1, :on => Exception }.merge(options)
+      # Uploads file to staging
+      #
+      # @param file [String] file to be uploaded
+      # @param options [Hash] must contain :staging_url key (file will be uploaded to :staging_url + File.basename(file))
+      def upload(file, options = {})
+        @connection.upload file, options
+      end
 
-        retry_exception, retries = opts[:on], opts[:tries]
+      # Downloads file from staging
+      #
+      # @param source_relative_path [String] path relative to @param options[:staging_url]
+      # @param target_file_path [String] path to be downloaded to
+      # @param options [Hash] must contain :staging_url key (file will be downloaded from :staging_url + source_relative_path)
+      def download(source_relative_path, target_file_path, options = {})
+        @connection.download source_relative_path, target_file_path, options
+      end
 
-        begin
-          return yield
-        rescue retry_exception
-          retry if (retries -= 1) > 0
-        end
+      def download_from_user_webdav(source_relative_path, target_file_path, options = {})
+        download(source_relative_path, target_file_path, options.merge(
+            :directory => options[:directory],
+            :staging_url => get_user_webdav_url(options)
+        ))
+      end
 
-        yield
+      def upload_to_user_webdav(file, options = {})
+        upload(file, options.merge(
+          :directory => options[:directory],
+          :staging_url => get_user_webdav_url(options)
+        ))
       end
 
-      # Uploads file
-      def upload(file, options = {})
-        @connection.upload file, options
+      def with_project(pid, &block)
+        GoodData.with_project(pid, client: self, &block)
       end
 
-      def upload_to_user_webdav(file, options = {})
+      ###################### PRIVATE ######################
+
+      private
+
+      def get_user_webdav_url(options = {})
         p = options[:project]
         fail ArgumentError, 'No :project specified' if p.nil?
 
-        project = GoodData::Project[p, options]
+        project = options[:project] || GoodData::Project[p, options]
         fail ArgumentError, 'Wrong :project specified' if project.nil?
 
         u = URI(project.links['uploads'])
-        url = URI.join(u.to_s.chomp(u.path.to_s), '/uploads/')
-        upload(file, options.merge(
-          :directory => options[:directory],
-          :staging_url => url
-        ))
-      end
+        us = u.to_s
+        ws = options[:client].opts[:webdav_server]
+        if !us.empty? && !us.downcase.start_with?('http') && !ws.empty?
+          u = URI.join(ws, us)
+        end
 
-      def with_project(pid, &block)
-        GoodData.with_project(pid, client: self, &block)
+        URI.join(u.to_s.chomp(u.path.to_s), '/uploads/')
       end
     end
   end

data/lib/gooddata/rest/connection.rb

@@ -80,7 +80,7 @@ module GoodData
         reset_cookies!
       end
 
-      def refresh_token(options = {})
+      def refresh_token(_options = {})
         begin # rubocop:disable RedundantBegin
           get TOKEN_PATH, :dont_reauth => true # avoid infinite loop GET fails with 401
         rescue Exception => e # rubocop:disable RescueException
@@ -99,28 +99,28 @@ module GoodData
       # HTTP DELETE
       #
       # @param uri [String] Target URI
-      def delete(uri, options = {})
+      def delete(uri, _options = {})
         fail NotImplementedError "DELETE #{uri}"
       end
 
       # HTTP GET
       #
       # @param uri [String] Target URI
-      def get(uri, options = {})
+      def get(uri, _options = {})
         fail NotImplementedError "GET #{uri}"
       end
 
       # HTTP PUT
       #
       # @param uri [String] Target URI
-      def put(uri, data, options = {})
+      def put(uri, _data, _options = {})
         fail NotImplementedError "PUT #{uri}"
       end
 
       # HTTP POST
       #
       # @param uri [String] Target URI
-      def post(uri, data, options = {})
+      def post(uri, _data, _options = {})
         fail NotImplementedError "POST #{uri}"
       end
 
@@ -132,7 +132,7 @@ module GoodData
       end
 
       def stats_table(values = stats)
-        sorted = values.sort_by { |k, v| v[:avg] }
+        sorted = values.sort_by { |_k, v| v[:avg] }
         Terminal::Table.new :headings => %w(title avg min max total calls) do |t|
           sorted.each do |l|
             row = [
@@ -179,7 +179,7 @@ module GoodData
         keys = keys.reduce([]) { |a, e| a.concat([e.to_s, e.to_sym]) }
 
         new_params = params.deep_dup
-        GoodData::Helpers.hash_dfs(new_params) do |k, key|
+        GoodData::Helpers.hash_dfs(new_params) do |k, _key|
           keys.each do |key_to_scrub|
             k[key_to_scrub] = ('*' * k[key_to_scrub].length) if k && k.key?(key_to_scrub) && k[key_to_scrub]
           end

data/lib/gooddata/rest/connections/dummy_connection.rb

@@ -12,7 +12,7 @@ module GoodData
         end
 
         # Connect using username and password
-        def connect(username, password)
+        def connect(_username, _password)
         end
 
         # Disconnect
@@ -22,28 +22,28 @@ module GoodData
         # HTTP DELETE
         #
         # @param uri [String] Target URI
-        def delete(uri, options = {})
+        def delete(uri, _options = {})
           puts "DELETE #{uri}"
         end
 
         # HTTP GET
         #
         # @param uri [String] Target URI
-        def get(uri, options = {})
+        def get(uri, _options = {})
           puts "GET #{uri}"
         end
 
         # HTTP PUT
         #
         # @param uri [String] Target URI
-        def put(uri, data, options = {})
+        def put(uri, _data, _options = {})
           puts "PUT #{uri}"
         end
 
         # HTTP POST
         #
         # @param uri [String] Target URI
-        def post(uri, data, options = {})
+        def post(uri, _data, _options = {})
           puts "POST #{uri}"
         end
       end

data/lib/gooddata/rest/connections/rest_client_connection.rb

@@ -44,10 +44,10 @@ module GoodData
         # HTTP GET
         #
         # @param uri [String] Target URI
-        def get(uri, options = {})
+        def get(uri, options = {}, &user_block)
           GoodData.logger.debug "GET: #{@server.url}#{uri}"
           profile "GET #{uri}" do
-            b = proc { @server[uri].get cookies }
+            b = proc { @server[uri].get(cookies, &user_block) }
             process_response(options, &b)
           end
         end
@@ -77,67 +77,129 @@ module GoodData
         end
 
         # Uploads a file to GoodData server
-        # /uploads/ resources are special in that they use a different
-        # host and a basic authentication.
         def upload(file, options = {})
-          dir = options[:directory] || ''
-          staging_uri = options[:staging_url].to_s
-          url = dir.empty? ? staging_uri : URI.join(staging_uri, "#{dir}/").to_s
+          def do_stream_file(uri, filename, options = {})
+            puts "uploading the file #{uri}"
+
+            to_upload = File.new(filename)
+            cookies_str = cookies[:cookies].map { |cookie| "#{cookie[0]}=#{cookie[1]}" }.join(';')
+            req = Net::HTTP::Put.new(uri.path, 'User-Agent' => GoodData.gem_version_string, 'Cookie' => cookies_str)
+            req.content_length = to_upload.size
+            req.body_stream = to_upload
+            http = Net::HTTP.new(uri.host, uri.port)
+            http.use_ssl = true
+            http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            response = http.start { |client| client.request(req) }
+            case response
+            when Net::HTTPSuccess then
+              true
+            when Net::HTTPUnauthorized
+              refresh_token
+              do_stream_file uri, filename, options
+            else
+              fail "Can't upload file to webdav. Path: #{uri}, response code: #{response.code}, response body: #{response.body}"
+            end
+          end
 
-          # Make a directory, if needed
-          unless dir.empty?
+          def webdav_dir_exists?(url)
             method = :get
             GoodData.logger.debug "#{method}: #{url}"
-            begin
-              # first check if it does exits
+
+            b = proc do
+              raw = {
+                :method => method,
+                :url => url,
+                :headers => @headers
+              }
+              begin
+                RestClient::Request.execute(raw.merge(cookies))
+              rescue  RestClient::Exception => e
+                false if e.http_code == 404
+              end
+            end
+
+            process_with_tt_refresh(&b)
+          end
+
+          def create_webdav_dir_if_needed(url)
+            return if webdav_dir_exists?(url)
+
+            method = :mkcol
+            GoodData.logger.debug "#{method}: #{url}"
+            b = proc do
               raw = {
                 :method => method,
                 :url => url,
-                # :timeout => @options[:timeout],
                 :headers => @headers
               }.merge(cookies)
               RestClient::Request.execute(raw)
-            rescue RestClient::Exception => e
-              if e.http_code == 404
-                method = :mkcol
-                GoodData.logger.debug "#{method}: #{url}"
-                raw = {
-                  :method => method,
-                  :url => url,
-                  # :timeout => @options[:timeout],
-                  :headers => @headers
-                }.merge(cookies)
-                RestClient::Request.execute(raw)
+            end
+
+            process_with_tt_refresh(&b)
+          end
+
+          dir = options[:directory] || ''
+          staging_uri = options[:staging_url].to_s
+          url = dir.empty? ? staging_uri : URI.join(staging_uri, "#{dir}/").to_s
+
+          # Make a directory, if needed
+          create_webdav_dir_if_needed url unless dir.empty?
+
+          webdav_filename = options[:filename] || File.basename(file)
+          do_stream_file URI.join(url, CGI.escape(webdav_filename)), file
+        end
+
+        def download(what, where, options = {})
+          dir = options[:directory] || ''
+          staging_uri = options[:staging_url].to_s
+
+          base_url = dir.empty? ? staging_uri : URI.join(staging_uri, "#{dir}/").to_s
+          url = URI.join(base_url, CGI.escape(what)).to_s
+
+          b = proc do
+            raw = {
+              :headers => {
+                :user_agent => GoodData.gem_version_string
+              },
+              :method => :get,
+              :url => url
+            }.merge(cookies)
+
+            if where.is_a?(String)
+              File.open(where, 'w') do |f|
+                RestClient::Request.execute(raw) do |chunk, _x, _y|
+                  f.write chunk
+                end
+              end
+
+            else
+              # Assume it is a IO stream
+              RestClient::Request.execute(raw) do |chunk, _x, _y|
+                where.write chunk
               end
             end
           end
 
-          payload = options[:stream] ? 'file' : File.read(file)
-          filename = options[:filename] || options[:stream] ? 'randome-filename.txt' : File.basename(file)
-
-          # Upload the file
-          # puts "uploading the file #{URI.join(url, filename).to_s}"
-          raw = {
-            :method => :put,
-            :url => URI.join(url, filename).to_s,
-            # :timeout => @options[:timeout],
-            :headers => {
-              :user_agent => GoodData.gem_version_string
-            },
-            :payload => payload,
-            :raw_response => true,
-            # :user => @username,
-            # :password => @password
-          }.merge(cookies)
-          RestClient::Request.execute(raw)
-          true
+          process_with_tt_refresh(&b)
+        end
+
+        private
+
+        def process_with_tt_refresh(&block)
+          block.call
+        rescue RestClient::Unauthorized
+          refresh_token
+          block.call
         end
 
         private
 
         def process_response(options = {}, &block)
           begin
-            response = block.call
+            # Simply try again when ConnectionReset, ConnectionRefused etc.. (see e.g. MSF-7591)
+            response = GoodData::Rest::Client.retryable(:tries => 2, :on => SystemCallError) do
+              block.call
+            end
           rescue RestClient::Unauthorized
             raise $ERROR_INFO if options[:dont_reauth]
             refresh_token
@@ -150,7 +212,7 @@ module GoodData
 
           if content_type == 'application/json' || content_type == 'application/json;charset=UTF-8'
             result = response.to_str == '""' ? {} : MultiJson.load(response.to_str)
-            GoodData.logger.debug "Response: #{result.inspect}"
+            GoodData.logger.debug "Request ID: #{response.headers[:x_gdc_request]} - Response: #{result.inspect}"
           elsif ['text/plain;charset=UTF-8', 'text/plain; charset=UTF-8', 'text/plain'].include?(content_type)
             result = response
             GoodData.logger.debug 'Response: plain text'