gooddata 2.2.0 → 2.3.0

data/lib/gooddata/models/project_creator.rb

@@ -39,6 +39,9 @@ module GoodData
           opts = { client: GoodData.connection }.merge(opts)
           dry_run = opts[:dry_run]
           replacements = opts['maql_replacements'] || opts[:maql_replacements] || {}
+          update_preference = opts[:update_preference]
+          exist_fallback_to_hard_sync_config = !update_preference.nil? && !update_preference[:fallback_to_hard_sync].nil?
+          include_maql_fallback_hard_sync = exist_fallback_to_hard_sync_config && GoodData::Helpers.to_boolean(update_preference[:fallback_to_hard_sync])
 
           _, project = GoodData.get_client_and_project(opts)
 
@@ -48,6 +51,7 @@ module GoodData
             maql_diff_params = [:includeGrain]
             maql_diff_params << :excludeFactRule if opts[:exclude_fact_rule]
             maql_diff_params << :includeDeprecated if opts[:include_deprecated]
+            maql_diff_params << :includeMaqlFallbackHardSync if include_maql_fallback_hard_sync
 
             maql_diff_time = Benchmark.realtime do
               response = project.maql_diff(blueprint: bp, params: maql_diff_params)
@@ -62,7 +66,7 @@ module GoodData
           ca_maql = response['projectModelDiff']['computedAttributesScript'] if response['projectModelDiff']['computedAttributesScript']
           ca_chunks = ca_maql && ca_maql['maqlDdlChunks']
 
-          maqls = pick_correct_chunks(chunks, opts)
+          maqls = include_maql_fallback_hard_sync ? pick_correct_chunks_hard_sync(chunks, opts) : pick_correct_chunks(chunks, opts)
           replaced_maqls = apply_replacements_on_maql(maqls, replacements)
           apply_maqls(ca_chunks, project, replaced_maqls, opts) unless dry_run
           [replaced_maqls, ca_maql]
@@ -72,9 +76,11 @@ module GoodData
           errors = []
           replaced_maqls.each do |replaced_maql_chunks|
             begin
+              fallback_hard_sync = replaced_maql_chunks['updateScript']['fallbackHardSync'].nil? ? false : replaced_maql_chunks['updateScript']['fallbackHardSync']
               replaced_maql_chunks['updateScript']['maqlDdlChunks'].each do |chunk|
                 GoodData.logger.debug(chunk)
-                project.execute_maql(chunk)
+                execute_maql_result = project.execute_maql(chunk)
+                process_fallback_hard_sync_result(execute_maql_result, project) if fallback_hard_sync
               end
             rescue => e
               GoodData.logger.error("Error occured when executing MAQL, project: \"#{project.title}\" reason: \"#{e.message}\", chunks: #{replaced_maql_chunks.inspect}")
@@ -140,8 +146,8 @@ module GoodData
           preference = Hash[preference.map { |k, v| [k, GoodData::Helpers.to_boolean(v)] }]
 
           # will use new parameters instead of the old ones
-          if preference.empty? || [:allow_cascade_drops, :keep_data].any? { |k| preference.key?(k) }
-            if [:cascade_drops, :preserve_data].any? { |k| preference.key?(k) }
+          if preference.empty? || %i[allow_cascade_drops keep_data].any? { |k| preference.key?(k) }
+            if %i[cascade_drops preserve_data].any? { |k| preference.key?(k) }
               fail "Please do not mix old parameters (:cascade_drops, :preserve_data) with the new ones (:allow_cascade_drops, :keep_data)."
             end
             preference = { allow_cascade_drops: false, keep_data: true }.merge(preference)
@@ -174,8 +180,8 @@ module GoodData
           results_from_api = GoodData::Helpers.join(
             rules,
             stuff,
-            [:cascade_drops, :preserve_data],
-            [:cascade_drops, :preserve_data],
+            %i[cascade_drops preserve_data],
+            %i[cascade_drops preserve_data],
             inner: true
           ).sort_by { |l| l[:priority] } || []
 
@@ -204,6 +210,53 @@ module GoodData
           end
         end
 
+        def pick_correct_chunks_hard_sync(chunks, opts = {})
+          preference = GoodData::Helpers.symbolize_keys(opts[:update_preference] || {})
+          preference = Hash[preference.map { |k, v| [k, GoodData::Helpers.to_boolean(v)] }]
+
+          # Old configure using cascade_drops and preserve_data parameters. New configure using allow_cascade_drops and
+          # keep_data parameters. Need translate from new configure to old configure before processing
+          if preference.empty? || %i[allow_cascade_drops keep_data].any? { |k| preference.key?(k) }
+            if %i[cascade_drops preserve_data].any? { |k| preference.key?(k) }
+              fail "Please do not mix old parameters (:cascade_drops, :preserve_data) with the new ones (:allow_cascade_drops, :keep_data)."
+            end
+
+            # Default allow_cascade_drops=false and keep_data=true
+            preference = { allow_cascade_drops: false, keep_data: true }.merge(preference)
+
+            new_preference = {}
+            new_preference[:cascade_drops] = preference[:allow_cascade_drops]
+            new_preference[:preserve_data] = preference[:keep_data]
+            preference = new_preference
+          end
+          preference[:fallback_to_hard_sync] = true
+
+          # Filter chunk with fallbackHardSync = true
+          result = chunks.select do |chunk|
+            chunk['updateScript']['maqlDdlChunks'] && !chunk['updateScript']['fallbackHardSync'].nil? && chunk['updateScript']['fallbackHardSync']
+          end
+
+          # The API model/diff only returns one result for MAQL fallback hard synchronize
+          result = pick_chunks_hard_sync(result[0], preference) if !result.nil? && !result.empty?
+
+          if result.nil? || result.empty?
+            available_chunks = chunks
+                                   .map do |chunk|
+                                     {
+                                       cascade_drops: chunk['updateScript']['cascadeDrops'],
+                                       preserve_data: chunk['updateScript']['preserveData'],
+                                       fallback_hard_sync: chunk['updateScript']['fallbackHardSync'].nil? ? false : chunk['updateScript']['fallbackHardSync']
+                                     }
+                                   end
+                                   .map(&:to_s)
+                                   .join(', ')
+
+            fail "Synchronize LDM cannot proceed. Adjust your update_preferences and try again. Available chunks with preference: #{available_chunks}"
+          end
+
+          result
+        end
+
         private
 
         def apply_replacements_on_maql(maqls, replacements = {})
@@ -215,6 +268,30 @@ module GoodData
             end
           end
         end
+
+        # Fallback hard synchronize although execute result success but some cases there are errors during executing.
+        # In this cases, then export the errors to execution log as warning
+        def process_fallback_hard_sync_result(result, project)
+          messages = result['wTaskStatus']['messages']
+          if !messages.nil? && messages.size.positive?
+            warning_message = GoodData::Helpers.interpolate_error_messages(messages)
+            log_message = "Project #{project.pid} failed to preserve data, truncated data of some datasets. MAQL diff execution messages: \"#{warning_message}\""
+            GoodData.logger.warn(log_message)
+          end
+        end
+
+        # In case fallback hard synchronize, then the API model/diff only returns one result with preserve_data is always false and
+        # cascade_drops is true or false. So pick chunk for fallback hard synchronize, we will ignore the preserve_data parameter
+        # and only check the cascade_drops parameter in preference.
+        def pick_chunks_hard_sync(chunk, preference)
+          # Make sure default values for cascade_drops
+          working_preference = { cascade_drops: false }.merge(preference)
+          if working_preference[:cascade_drops] || chunk['updateScript']['cascadeDrops'] == working_preference[:cascade_drops]
+            [chunk]
+          else
+            []
+          end
+        end
       end
     end
   end

data/lib/gooddata/models/segment.rb

@@ -257,7 +257,8 @@ module GoodData
       res = client.poll_on_code(res['asyncTask']['links']['poll'])
       failed_count = GoodData::Helpers.get_path(res, %w(clientProjectProvisioningResult failed count), 0)
       created_count = GoodData::Helpers.get_path(res, %w(clientProjectProvisioningResult created count), 0)
-      return Enumerator.new([]) if (failed_count + created_count).zero?
+      return [].to_enum if (failed_count + created_count).zero?
+
       Enumerator.new do |y|
         uri = GoodData::Helpers.get_path(res, %w(clientProjectProvisioningResult links details))
         loop do

data/lib/gooddata/models/user_filters/user_filter_builder.rb

@@ -19,6 +19,9 @@ using NilExtensions
 
 module GoodData
   module UserFilterBuilder
+    @all_domain_users = {}
+    @mutex = Mutex.new
+
     # Main Entry function. Gets values and processes them to get filters
     # that are suitable for other function to process.
     # Values can be read from file or provided inline as an array.
@@ -48,11 +51,26 @@ module GoodData
     end
 
     def self.read_file(file, options = {})
-      memo = {}
-      params = row_based?(options) ? { headers: false } : { headers: true }
+      memo = Hash[]
+      if row_based?(options)
+        read_data_without_header(file, memo, options)
+      else
+        read_data_with_header(file, memo, options)
+      end
+      memo
+    end
 
-      CSV.foreach(file, params.merge(return_headers: false)) do |e|
-        key, data = process_line(e, options)
+    def self.read_data_without_header(file, memo, options)
+      CSV.foreach(file, headers: false, return_headers: false) do |row|
+        key, data = process_line(row, options)
+        memo[key] = [] unless memo.key?(key)
+        memo[key].concat(data)
+      end
+    end
+
+    def self.read_data_with_header(file, memo, options)
+      CSV.foreach(file, headers: true, return_headers: false) do |row|
+        key, data = process_line(row, options)
         memo[key] = [] unless memo.key?(key)
         memo[key].concat(data)
       end
@@ -210,13 +228,23 @@ module GoodData
     # Takes the filter definition looks up any necessary values and provides API executable MAQL
     # @param labels_cache e.g. { 'label_uri': label_object }
     # @param lookups_cache e.g. { 'label_uri': { "jirka@gooddata.com": 'value_uri' }}
-    def self.create_expression(filter, labels_cache, lookups_cache, attr_cache, options = {})
+    # rubocop:disable Metrics/ParameterLists
+    def self.create_expression(filter, labels_cache, lookups_cache, attr_cache, options = {}, login)
       values = filter[:values]
       # Do not create MUF for label when all its values is NULL (https://jira.intgdc.com/browse/TMA-1361)
       non_null_values = values.select { |value| !value.nil? && value.downcase != 'null' }
       return ['TRUE', []] if non_null_values.empty?
 
       label = labels_cache[filter[:label]]
+      if label.nil?
+        err_message = "Unable to apply filter values: #{values} since the project: #{options[:project].pid} doesn't have label: #{filter[:label]} for login: #{login}"
+        if options[:ignore_missing_values]
+          GoodData.logger.warn(err_message)
+          return ['TRUE', []]
+        else
+          fail err_message
+        end
+      end
       errors = []
 
       element_uris_by_values = Hash[values.map do |v|
@@ -262,6 +290,7 @@ module GoodData
                    end
       [expression, errors]
     end
+    # rubocop:enable Metrics/ParameterLists
 
     # Encapuslates the creation of filter
     def self.create_user_filter(expression, related)
@@ -273,6 +302,66 @@ module GoodData
       }
     end
 
+    def self.create_user_profile_mapping(filters, project_users, options = {})
+      domain = options[:domain]
+      found_list = {}
+      missing_list = []
+
+      # Get the list of user login from filters
+      login_list = filters.flat_map do |filter|
+        filter[:login]
+      end
+
+      # Then find user login in the users_brick_input
+      users_brick_input = options[:users_brick_input]
+      if users_brick_input&.any?
+        users_brick_input.map do |user|
+          login_list << user.with_indifferent_access['login']
+        end
+      end
+
+      login_list.uniq.flat_map do |login|
+        user = project_users.find { |u| u.login == login }
+        if user
+          found_list[login] = user.profile_url
+        else
+          missing_list << login
+        end
+      end
+      # rubocop:disable Metrics/BlockNesting
+      unless missing_list.empty? || domain.nil?
+        if missing_list.size < 100
+          missing_list.each do |login|
+            user = domain.find_user_by_login(login)
+            found_list[login] = user.links['self'] if user
+          end
+        else
+          if @all_domain_users[domain.name].nil?
+            @mutex.lock
+            if @all_domain_users[domain.name].nil?
+              domain_users = domain.users
+              @all_domain_users[domain.name] = domain_users
+              GoodData.logger.info("action=lcm_get_domain_users domain=#{domain.name} number_users=#{domain_users.size} number_missing_users=#{missing_list.size} use_cache=false")
+            else
+              domain_users = @all_domain_users[domain.name]
+              GoodData.logger.info("action=lcm_get_domain_users domain=##{domain.name} number_users=#{domain_users.size} number_missing_users=#{missing_list.size} use_cache=true")
+            end
+            @mutex.unlock
+          else
+            domain_users = @all_domain_users[domain.name]
+            GoodData.logger.info("action=lcm_get_domain_users domain=##{domain.name} number_users=#{domain_users.size} number_missing_users=#{missing_list.size} use_cache=true")
+          end
+
+          missing_list.each do |login|
+            user = domain_users.find { |u| u.login == login }
+            found_list[login] = user.links['self'] if user
+          end
+        end
+      end
+      # rubocop:enable Metrics/BlockNesting
+      found_list
+    end
+
     # Resolves and creates maql statements from filter definitions.
     # This method does not perform any modifications on API but
     # collects all the information that is needed to do so.
@@ -283,7 +372,7 @@ module GoodData
     #
     # @param filters [Array<Hash>] Filters definition
     # @return [Array] first is list of MAQL statements
-    def self.maqlify_filters(filters, project_users, options = {})
+    def self.maqlify_filters(filters, user_profile_mapping, options = {})
       fail_early = options[:fail_early] == false ? false : true
       users_cache = options[:users_cache]
       labels_cache = create_label_cache(filters, options)
@@ -291,11 +380,10 @@ module GoodData
       lookups_cache = create_lookups_cache(small_labels)
       attrs_cache = create_attrs_cache(filters, options)
       create_filter_proc = proc do |login, f|
-        expression, errors = create_expression(f, labels_cache, lookups_cache, attrs_cache, options)
+        expression, errors = create_expression(f, labels_cache, lookups_cache, attrs_cache, options, login)
         safe_login = login.downcase
         profiles_uri = if options[:type] == :muf
-                         project_user = project_users.find { |u| u.login == safe_login }
-                         project_user.nil? ? ('/gdc/account/profile/' + safe_login) : project_user.profile_url
+                         user_profile_mapping[safe_login].nil? ? ('/gdc/account/profile/' + safe_login) : user_profile_mapping[safe_login]
                        elsif options[:type] == :variable
                          (users_cache[login] && users_cache[login].uri)
                        else
@@ -393,7 +481,8 @@ module GoodData
 
       project_users = project.users
       filters = normalize_filters(user_filters)
-      user_filters, errors = maqlify_filters(filters, project_users, options.merge(users_must_exist: users_must_exist, type: :muf))
+      user_profile_mapping = create_user_profile_mapping(filters, project_users, options)
+      user_filters, errors = maqlify_filters(filters, user_profile_mapping, options.merge(users_must_exist: users_must_exist, type: :muf))
       if !ignore_missing_values && !errors.empty?
         errors = errors.map do |e|
           e.merge(pid: project.pid)
@@ -404,7 +493,7 @@ module GoodData
       filters = user_filters.map { |data| client.create(MandatoryUserFilter, data, project: project) }
       to_create, to_delete = resolve_user_filters(filters, project.data_permissions)
 
-      to_delete = sanitize_filters_to_delete(to_delete, options[:users_brick_input], project_users) unless options[:no_sanitize]
+      to_delete = sanitize_filters_to_delete(to_delete, options[:users_brick_input], user_profile_mapping) unless options[:no_sanitize]
 
       if options[:do_not_touch_filters_that_are_not_mentioned]
         GoodData.logger.warn("Data permissions computed: #{to_create.count} to create")
@@ -499,7 +588,7 @@ module GoodData
     # @param file [String | Array] File or array of values to be parsed for filters
     # @param options [Hash] Filter definitions
     # @return [Array<Hash>]
-    def self.get_values(file, options)
+    def self.get_values(file, options = {})
       file.is_a?(Array) ? read_array(file, options) : read_file(file, options)
     end
 
@@ -584,12 +673,12 @@ module GoodData
     # Removes MUFs from to_delete unless in user is in users_brick_input
     # if this does not happen, users that are about to be deleted by users_brick
     # would have all their filters removed now, which is not desirable
-    def self.sanitize_filters_to_delete(to_delete, users_brick_input, project_users)
+    def self.sanitize_filters_to_delete(to_delete, users_brick_input, user_profile_mapping)
       return [] unless users_brick_input && users_brick_input.any?
       user_profiles = users_brick_input.map do |user|
-        result = project_users.find { |u| u.login == user.with_indifferent_access['login'] }
+        result = user_profile_mapping[user.with_indifferent_access['login']]
         next unless result
-        result.profile_url
+        result
       end.compact
       return [] unless user_profiles.any?
       to_delete.reject do |_, value|

data/lib/gooddata/rest/connection.rb

@@ -68,6 +68,8 @@ module GoodData
       RETRY_TIME_COEFFICIENT = 1.5
       RETRYABLE_ERRORS << Net::ReadTimeout if Net.const_defined?(:ReadTimeout)
 
+      RETRYABLE_ERRORS << OpenSSL::SSL::SSLErrorWaitReadable if OpenSSL::SSL.const_defined?(:SSLErrorWaitReadable)
+
       class << self
         def construct_login_payload(username, password)
           res = {
@@ -304,12 +306,12 @@ module GoodData
       end
 
       def refresh_token(_options = {})
-        begin # rubocop:disable RedundantBegin
+        begin # rubocop:disable Style/RedundantBegin
           # avoid infinite loop GET fails with 401
           response = get(TOKEN_PATH, :x_gdc_authsst => sst_token, :dont_reauth => true)
           # Remove when TT sent in headers. Currently we need to parse from body
           merge_headers!(:x_gdc_authtt => GoodData::Helpers.get_path(response, %w(userToken token)))
-        rescue Exception => e # rubocop:disable RescueException
+        rescue Exception => e # rubocop:disable Style/RescueException
           raise e
         end
       end
@@ -688,7 +690,7 @@ ERR
           execution_id = request_id
 
           GoodData.gd_logger.update_store(domain, method, duration, endpoint)
-          GoodData.gd_logger.add Logger::INFO, { endpoint: endpoint, duration: duration, domain: domain,
+          GoodData.gd_logger.add Logger::DEBUG, { endpoint: endpoint, duration: duration, domain: domain,
                                                  execution_id: execution_id, time_stamp: time_stamp }, "rest_call"
         end
       end

data/lib/gooddata/rest/phmap.rb

@@ -83,6 +83,7 @@ module GoodData
         ['/gdc/internal/projects/{id}/objects/setPermissions', %r{/gdc/internal/projects/[^\/]+/objects/setPermissions}],
         ['/gdc/internal/projects/{id}/roles', %r{/gdc/internal/projects/[^\/]+/roles}],
         ['/gdc/dataload/projects/{id}/modelMapping/datasets/bulk/upsert', %r{/gdc/dataload/projects/[^\/]+/modelMapping/datasets/bulk/upsert}],
+        ['/gdc/dataload/internal/projects/{id}/ldmLayout', %r{/gdc/dataload/internal/projects/[^\/]+/ldmLayout}],
         ['/gdc/md/{id}/variables/item/{id}', %r{/gdc/md/[^\/]+/variables/item/[\d]+}],
         ['/gdc/md/{id}/validate/task/{id}', %r{/gdc/md/[^\/]+/validate/task/[^\/]+}],
         ['/gdc/md/{id}/using2/{id}/{id}', %r{/gdc/md/[^\/]+/using2/[\d]+/[\d]+}],

data/lib/gooddata.rb

@@ -37,5 +37,6 @@ require 'backports/2.1.0/array/to_h'
 
 # Helpers
 require 'gooddata/helpers/global_helpers'
+require 'gooddata/lcm/helpers/check_helper'
 
 require 'active_support/core_ext/hash/compact' unless RUBY_VERSION >= '2.5'

data/lib/gooddata_brick_base.rb

@@ -0,0 +1,35 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2010-2022 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'pmap'
+$pmap_default_thread_count = 20 # rubocop:disable GlobalVars
+
+# GoodData Module
+module GoodData
+end
+
+# Modules
+require_relative 'gooddata/core/core'
+require_relative 'gooddata/models/models'
+require_relative 'gooddata/exceptions/exceptions'
+require_relative 'gooddata/helpers/helpers'
+
+# Files
+require_relative 'gooddata/bricks/utils'
+require_relative 'gooddata/bricks/brick'
+require_relative 'gooddata/bricks/base_pipeline'
+require_relative 'gooddata/bricks/middleware/base_middleware'
+require_relative 'gooddata/bricks/middleware/bench_middleware'
+require_relative 'gooddata/bricks/middleware/logger_middleware'
+require_relative 'gooddata/bricks/middleware/decode_params_middleware'
+require_relative 'gooddata/bricks/middleware/aws_middleware'
+require_relative 'gooddata/bricks/middleware/dwh_middleware'
+require_relative 'gooddata/bricks/middleware/bench_middleware'
+
+# CSV Downloader
+require_relative 'gooddata/core/logging'
+require_relative 'gooddata/connection'

data/sonar-project.properties

@@ -0,0 +1,6 @@
+sonar.projectKey=gooddata-ruby
+sonar.organization=gooddata-github
+# Config scan necessary files/folder only
+sonar.sources=bin,lib
+sonar.tests=spec
+sonar.exclusions=**/lib/templates/project/*.erb