gooddata 2.3.1-java → 2.3.2-java

data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb

@@ -33,6 +33,9 @@ module GoodData
 
         description 'Sync failed list'
         param :sync_failed_list, instance_of(Type::HashType), required: false
+
+        description 'Number Of Threads'
+        param :number_of_threads, instance_of(Type::StringType), required: false, default: '10'
       end
 
       RESULT_HEADER = %i[from to count status]
@@ -45,8 +48,9 @@ module GoodData
 
           client = params.gdc_gd_client
           development_client = params.development_client
+          number_of_threads = Integer(params.number_of_threads || '8')
 
-          params.synchronize.peach do |info|
+          params.synchronize.peach(number_of_threads) do |info|
             from_project = info.from
             to_projects = info.to
 
@@ -60,7 +64,7 @@ module GoodData
             if dataset_mapping&.dig('datasetMappings', 'items').nil? || dataset_mapping['datasetMappings']['items'].empty?
               params.gdc_logger.info "Project: '#{from.title}', PID: '#{from.pid}' has no model mapping, skip synchronizing model mapping."
             else
-              to_projects.peach do |to|
+              to_projects.peach(number_of_threads) do |to|
                 pid = to[:pid]
                 next if sync_failed_project(pid, params)
 

data/lib/gooddata/lcm/actions/synchronize_user_filters.rb

@@ -74,6 +74,9 @@ module GoodData
 
         description 'Makes the brick run without altering user filters'
         param :dry_run, instance_of(Type::StringType), required: false, default: false
+
+        description 'Number Of Threads'
+        param :number_of_threads, instance_of(Type::StringType), required: false, default: '10'
       end
 
       class << self
@@ -104,6 +107,7 @@ module GoodData
           symbolized_config = GoodData::Helpers.symbolize_keys(symbolized_config)
           symbolized_config[:labels] = symbolized_config[:labels].map { |l| GoodData::Helpers.symbolize_keys(l) }
           multiple_projects_column = params.multiple_projects_column
+          number_of_threads = Integer(params.number_of_threads || '10')
 
           mode = params.sync_mode
           unless MODES.include?(mode)
@@ -196,7 +200,7 @@ module GoodData
 
             unless run_params[:do_not_touch_filters_that_are_not_mentioned]
               to_be_deleted_clients = UserBricksHelper.non_working_clients(domain_clients, working_client_ids)
-              to_be_deleted_clients.peach do |c|
+              to_be_deleted_clients.peach(number_of_threads) do |c|
                 begin
                   current_project = c.project
                   users = users_by_project[c.client_id]

data/lib/gooddata/lcm/lcm2.rb

@@ -295,6 +295,10 @@ module GoodData
       end
 
       def perform(mode, params = {})
+        # Default setting for $pmap_default_thread_count = 20 in gooddata.rb file. We are going to decrease default
+        # number of threads count to 10 for LCM bricks only
+        $pmap_default_thread_count = 10 # rubocop:disable GlobalVars
+
         params = convert_params(params)
 
         GoodData.gd_logger.brick = mode

data/lib/gooddata/models/domain.rb

@@ -22,7 +22,15 @@ module GoodData
       'pt-PT' => 'Portuguese/Portugal',
       'fr-FR' => 'French',
       'de-DE' => 'German',
-      'ja-JP' => 'Japanese'
+      'ja-JP' => 'Japanese',
+      'it-IT' => 'Italian',
+      'es-419' => 'Spanish/Latin America',
+      'fr-CA' => 'French/Canada',
+      'en-GB' => 'English/UK',
+      'en-AU' => 'English/Australian',
+      'fi-FI' => 'Finnish',
+      'zh-Hant' => 'Traditional Chinese',
+      'zh-HK' => 'Cantonese'
     }
 
     class << self
@@ -276,12 +284,16 @@ to new properties (email=#{user_data[:email]}, sso_provider=#{user_data[:sso_pro
               [{ type: :successful, :action => :user_changed_in_domain, user: updated_user }]
             end
           rescue RuntimeError => e
+            error_message = e.message
+            user.delete(:password)
             if !domain_user
-              GoodData.logger.error("Failed to add user=#{user_login} to domain=#{default_domain_name}. Error: #{e.message}")
+              GoodData.logger.error("Failed to add user=#{user_login} to domain=#{default_domain_name}. Error: #{error_message}")
             else
-              GoodData.logger.error("Failed to update user=#{user_login} in domain=#{default_domain_name}. Error: #{e.message}")
+              error_message = 'Invalid user data or update new password cannot be the same as old password' if error_message == '400 Bad Request'
+
+              GoodData.logger.error("Failed to update user=#{user_login} in domain=#{default_domain_name}. Error: #{error_message}")
             end
-            [{ type: :failed, :user => user, message: e }]
+            [{ type: :failed, :user => user, message: error_message }]
           end
         end
       end

data/lib/gooddata/models/metadata/label.rb

@@ -44,14 +44,31 @@ module GoodData
     # In the case filter a specific value, because the API /validElements only filter by partial match, we need to filter again at client side for exact match.
     # @return [Array] Results
     def get_valid_elements(*args)
+      results = {}
       if args && !args.empty? && args.first[:filter]
+        # Support paging in case filter by a specific value
         params = args.first
-        params[:limit] = 100_000
-        results, = valid_elements params
-        results['validElements']['items'] = results['validElements']['items'].select do |i|
-          i['element']['title'] == params[:filter]
+        all_valid_elements = []
+        offset = 0
+        paging_limit = 10_000
+
+        loop do
+          params[:offset] = offset
+          params[:limit] = paging_limit
+          results, = valid_elements params
+          all_valid_elements += results['validElements']['items'].select do |i|
+            i['element']['title'] == params[:filter]
+          end
+
+          if results['validElements']['items'].count < paging_limit
+            results['validElements']['items'] = all_valid_elements
+            break
+          else
+            offset += paging_limit
+          end
         end
       else
+        # This case will support paging by the method which call this method eg: values(...) method
         results, = valid_elements(*args)
       end
       results

data/lib/gooddata/models/project.rb

@@ -1434,9 +1434,13 @@ module GoodData
           :crossDataCenterExport => '1'
         }
       }
-      result = client.post("#{md['maintenance']}/partialmdexport", export_payload)
+      export_uri = "/gdc/md/#{pid}/maintenance/partialmdexport"
+      GoodData.gd_logger.info("Project export action=objects_export, project_id=#{pid}, uri=#{export_uri}, export_status=start, export_objs=#{export_payload}") if GoodData.gd_logger
+
+      result = client.post(export_uri, export_payload)
       polling_url = result['partialMDArtifact']['status']['uri']
       token = result['partialMDArtifact']['token']
+      GoodData.gd_logger.info("Project export action=objects_export, project_id=#{pid}, uri=#{polling_url}, export_status=polling") if GoodData.gd_logger
 
       polling_result = client.poll_on_response(polling_url, options) do |body|
         body['wTaskStatus'] && body['wTaskStatus']['status'] == 'RUNNING'
@@ -1445,6 +1449,9 @@ module GoodData
         messages = GoodData::Helpers.interpolate_error_messages(polling_result['wTaskStatus']['messages']).join(' ')
         fail ObjectsExportError, "Exporting objects failed with messages. #{messages}"
       end
+
+      GoodData.gd_logger.info("Project export action=objects_export, project_id=#{pid}, export_status=success") if GoodData.gd_logger
+
       token
     end
 
@@ -1467,8 +1474,12 @@ module GoodData
         }
       }
 
-      result = client.post("#{md['maintenance']}/partialmdimport", import_payload)
+      import_uri = "/gdc/md/#{pid}/maintenance/partialmdimport"
+      GoodData.gd_logger.info("Project import action=objects_import, project_id=#{pid}, uri=#{import_uri}, import_status=start") if GoodData.gd_logger
+
+      result = client.post(import_uri, import_payload)
       polling_url = result['uri']
+      GoodData.gd_logger.info("Project import action=objects_import, project_id=#{pid}, uri=#{polling_url}, import_status=polling") if GoodData.gd_logger
 
       polling_result = client.poll_on_response(polling_url, options) do |body|
         body['wTaskStatus'] && body['wTaskStatus']['status'] == 'RUNNING'
@@ -1478,6 +1489,9 @@ module GoodData
         messages = GoodData::Helpers.interpolate_error_messages(polling_result['wTaskStatus']['messages']).join(' ')
         fail ObjectsImportError, "Importing objects failed with messages. #{messages}"
       end
+
+      GoodData.gd_logger.info("Project import action=objects_import, project_id=#{pid}, uri=#{import_uri}, import_status=success") if GoodData.gd_logger
+
       true
     end
 

data/lib/gooddata/models/user_filters/user_filter_builder.rb

@@ -180,7 +180,8 @@ module GoodData
 
     def self.create_lookups_cache(small_labels)
       small_labels.reduce({}) do |a, e|
-        lookup = e.values(:limit => 1_000_000).reduce({}) do |a1, e1|
+        # The validElements API allow maximum paging with 10000 items
+        lookup = e.values(:limit => 10_000).reduce({}) do |a1, e1|
           a1[e1[:value]] = e1[:uri]
           a1
         end
@@ -513,7 +514,7 @@ module GoodData
       if to_create.empty?
         create_results = []
       else
-        create_results = to_create.each_slice(100).flat_map do |batch|
+        create_results = to_create.each_slice(50).flat_map do |batch|
           batch.pmapcat do |related_uri, group|
             group.each(&:save)
             res = client.get("/gdc/md/#{project.pid}/userfilters?users=#{related_uri}")

data/lib/gooddata/rest/connection.rb

@@ -65,7 +65,7 @@ module GoodData
       ]
 
       RETRY_TIME_INITIAL_VALUE = 1
-      RETRY_TIME_COEFFICIENT = 1.5
+      RETRY_TIME_COEFFICIENT = 2
       RETRYABLE_ERRORS << Net::ReadTimeout if Net.const_defined?(:ReadTimeout)
 
       RETRYABLE_ERRORS << OpenSSL::SSL::SSLErrorWaitReadable if OpenSSL::SSL.const_defined?(:SSLErrorWaitReadable)
@@ -94,7 +94,7 @@ module GoodData
 
         # Retry block if exception thrown
         def retryable(options = {}, &_block)
-          opts = { :tries => 12, :on => RETRYABLE_ERRORS }.merge(options)
+          opts = { :tries => 14, :on => RETRYABLE_ERRORS }.merge(options)
 
           retry_exception = opts[:on]
           retries = opts[:tries]
@@ -114,21 +114,27 @@ module GoodData
             if (retries -= 1) > 0
               retry
             else
-              fail e
+              process_retry_error(e, retry_time)
             end
           rescue RestClient::TooManyRequests, RestClient::ServiceUnavailable, *retry_exception => e
-            GoodData.logger.warn "#{e.message}, retrying in #{retry_time} seconds"
             sleep retry_time
-            retry_time *= RETRY_TIME_COEFFICIENT
-            # 10 requests with 1.5 coefficent should take ~ 3 mins to finish
+            # Total 10 retry requests with 1.5 coefficent should take ~ 2 mins to finish
             if (retries -= 1) > 0
+              retry_time *= RETRY_TIME_COEFFICIENT
               retry
             else
-              fail e
+              process_retry_error(e, retry_time)
             end
           end
           yield
         end
+
+        def process_retry_error(e, retry_time)
+          error_message =  "#{e.message}, retrying in #{retry_time} seconds"
+          GoodData.logger.warn error_message
+          GoodData.gd_logger.warn error_message
+          fail e
+        end
       end
 
       attr_reader :request_params