gooddata 2.1.19 → 2.3.0

data/lib/gooddata/models/user_filters/user_filter_builder.rb

@@ -19,6 +19,9 @@ using NilExtensions
 
 module GoodData
   module UserFilterBuilder
+    @all_domain_users = {}
+    @mutex = Mutex.new
+
     # Main Entry function. Gets values and processes them to get filters
     # that are suitable for other function to process.
     # Values can be read from file or provided inline as an array.
@@ -48,11 +51,26 @@ module GoodData
     end
 
     def self.read_file(file, options = {})
-      memo = {}
-      params = row_based?(options) ? { headers: false } : { headers: true }
+      memo = Hash[]
+      if row_based?(options)
+        read_data_without_header(file, memo, options)
+      else
+        read_data_with_header(file, memo, options)
+      end
+      memo
+    end
 
-      CSV.foreach(file, params.merge(return_headers: false)) do |e|
-        key, data = process_line(e, options)
+    def self.read_data_without_header(file, memo, options)
+      CSV.foreach(file, headers: false, return_headers: false) do |row|
+        key, data = process_line(row, options)
+        memo[key] = [] unless memo.key?(key)
+        memo[key].concat(data)
+      end
+    end
+
+    def self.read_data_with_header(file, memo, options)
+      CSV.foreach(file, headers: true, return_headers: false) do |row|
+        key, data = process_line(row, options)
         memo[key] = [] unless memo.key?(key)
         memo[key].concat(data)
       end
@@ -203,20 +221,30 @@ module GoodData
     # so it precaches the values and still be able to function for larger ones even
     # though that would mean tons of requests
     def self.get_small_labels(labels_cache)
-      labels_cache.values.select { |label| label && label.values_count && label.values_count < 100_000 }
+      labels_cache.values.select { |label| label &.values_count &. < 100_000 }
     end
 
     # Creates a MAQL expression(s) based on the filter defintion.
     # Takes the filter definition looks up any necessary values and provides API executable MAQL
     # @param labels_cache e.g. { 'label_uri': label_object }
     # @param lookups_cache e.g. { 'label_uri': { "jirka@gooddata.com": 'value_uri' }}
-    def self.create_expression(filter, labels_cache, lookups_cache, attr_cache, options = {})
+    # rubocop:disable Metrics/ParameterLists
+    def self.create_expression(filter, labels_cache, lookups_cache, attr_cache, options = {}, login)
       values = filter[:values]
       # Do not create MUF for label when all its values is NULL (https://jira.intgdc.com/browse/TMA-1361)
       non_null_values = values.select { |value| !value.nil? && value.downcase != 'null' }
       return ['TRUE', []] if non_null_values.empty?
 
       label = labels_cache[filter[:label]]
+      if label.nil?
+        err_message = "Unable to apply filter values: #{values} since the project: #{options[:project].pid} doesn't have label: #{filter[:label]} for login: #{login}"
+        if options[:ignore_missing_values]
+          GoodData.logger.warn(err_message)
+          return ['TRUE', []]
+        else
+          fail err_message
+        end
+      end
       errors = []
 
       element_uris_by_values = Hash[values.map do |v|
@@ -262,6 +290,7 @@ module GoodData
                    end
       [expression, errors]
     end
+    # rubocop:enable Metrics/ParameterLists
 
     # Encapuslates the creation of filter
     def self.create_user_filter(expression, related)
@@ -273,6 +302,66 @@ module GoodData
       }
     end
 
+    def self.create_user_profile_mapping(filters, project_users, options = {})
+      domain = options[:domain]
+      found_list = {}
+      missing_list = []
+
+      # Get the list of user login from filters
+      login_list = filters.flat_map do |filter|
+        filter[:login]
+      end
+
+      # Then find user login in the users_brick_input
+      users_brick_input = options[:users_brick_input]
+      if users_brick_input&.any?
+        users_brick_input.map do |user|
+          login_list << user.with_indifferent_access['login']
+        end
+      end
+
+      login_list.uniq.flat_map do |login|
+        user = project_users.find { |u| u.login == login }
+        if user
+          found_list[login] = user.profile_url
+        else
+          missing_list << login
+        end
+      end
+      # rubocop:disable Metrics/BlockNesting
+      unless missing_list.empty? || domain.nil?
+        if missing_list.size < 100
+          missing_list.each do |login|
+            user = domain.find_user_by_login(login)
+            found_list[login] = user.links['self'] if user
+          end
+        else
+          if @all_domain_users[domain.name].nil?
+            @mutex.lock
+            if @all_domain_users[domain.name].nil?
+              domain_users = domain.users
+              @all_domain_users[domain.name] = domain_users
+              GoodData.logger.info("action=lcm_get_domain_users domain=#{domain.name} number_users=#{domain_users.size} number_missing_users=#{missing_list.size} use_cache=false")
+            else
+              domain_users = @all_domain_users[domain.name]
+              GoodData.logger.info("action=lcm_get_domain_users domain=##{domain.name} number_users=#{domain_users.size} number_missing_users=#{missing_list.size} use_cache=true")
+            end
+            @mutex.unlock
+          else
+            domain_users = @all_domain_users[domain.name]
+            GoodData.logger.info("action=lcm_get_domain_users domain=##{domain.name} number_users=#{domain_users.size} number_missing_users=#{missing_list.size} use_cache=true")
+          end
+
+          missing_list.each do |login|
+            user = domain_users.find { |u| u.login == login }
+            found_list[login] = user.links['self'] if user
+          end
+        end
+      end
+      # rubocop:enable Metrics/BlockNesting
+      found_list
+    end
+
     # Resolves and creates maql statements from filter definitions.
     # This method does not perform any modifications on API but
     # collects all the information that is needed to do so.
@@ -283,7 +372,7 @@ module GoodData
     #
     # @param filters [Array<Hash>] Filters definition
     # @return [Array] first is list of MAQL statements
-    def self.maqlify_filters(filters, project_users, options = {})
+    def self.maqlify_filters(filters, user_profile_mapping, options = {})
       fail_early = options[:fail_early] == false ? false : true
       users_cache = options[:users_cache]
       labels_cache = create_label_cache(filters, options)
@@ -291,11 +380,10 @@ module GoodData
       lookups_cache = create_lookups_cache(small_labels)
       attrs_cache = create_attrs_cache(filters, options)
       create_filter_proc = proc do |login, f|
-        expression, errors = create_expression(f, labels_cache, lookups_cache, attrs_cache, options)
+        expression, errors = create_expression(f, labels_cache, lookups_cache, attrs_cache, options, login)
         safe_login = login.downcase
         profiles_uri = if options[:type] == :muf
-                         project_user = project_users.find { |u| u.login == safe_login }
-                         project_user.nil? ? ('/gdc/account/profile/' + safe_login) : project_user.profile_url
+                         user_profile_mapping[safe_login].nil? ? ('/gdc/account/profile/' + safe_login) : user_profile_mapping[safe_login]
                        elsif options[:type] == :variable
                          (users_cache[login] && users_cache[login].uri)
                        else
@@ -393,7 +481,8 @@ module GoodData
 
       project_users = project.users
       filters = normalize_filters(user_filters)
-      user_filters, errors = maqlify_filters(filters, project_users, options.merge(users_must_exist: users_must_exist, type: :muf))
+      user_profile_mapping = create_user_profile_mapping(filters, project_users, options)
+      user_filters, errors = maqlify_filters(filters, user_profile_mapping, options.merge(users_must_exist: users_must_exist, type: :muf))
       if !ignore_missing_values && !errors.empty?
         errors = errors.map do |e|
           e.merge(pid: project.pid)
@@ -404,7 +493,7 @@ module GoodData
       filters = user_filters.map { |data| client.create(MandatoryUserFilter, data, project: project) }
       to_create, to_delete = resolve_user_filters(filters, project.data_permissions)
 
-      to_delete = sanitize_filters_to_delete(to_delete, options[:users_brick_input], project_users) unless options[:no_sanitize]
+      to_delete = sanitize_filters_to_delete(to_delete, options[:users_brick_input], user_profile_mapping) unless options[:no_sanitize]
 
       if options[:do_not_touch_filters_that_are_not_mentioned]
         GoodData.logger.warn("Data permissions computed: #{to_create.count} to create")
@@ -421,68 +510,73 @@ module GoodData
                  results: create_results + delete_results }
       end
 
-      create_results = to_create.each_slice(100).flat_map do |batch|
-        batch.pmapcat do |related_uri, group|
-          group.each(&:save)
-          res = client.get("/gdc/md/#{project.pid}/userfilters?users=#{related_uri}")
-          items = res['userFilters']['items'].empty? ? [] : res['userFilters']['items'].first['userFilters']
-
-          payload = {
-            'userFilters' => {
-              'items' => [{
-                'user' => related_uri,
-                'userFilters' => items.concat(group.map(&:uri))
-              }]
+      if to_create.empty?
+        create_results = []
+      else
+        create_results = to_create.each_slice(100).flat_map do |batch|
+          batch.pmapcat do |related_uri, group|
+            group.each(&:save)
+            res = client.get("/gdc/md/#{project.pid}/userfilters?users=#{related_uri}")
+            items = res['userFilters']['items'].empty? ? [] : res['userFilters']['items'].first['userFilters']
+
+            payload = {
+              'userFilters' => {
+                'items' => [{
+                  'user' => related_uri,
+                  'userFilters' => items.concat(group.map(&:uri))
+                }]
+              }
             }
-          }
-          res = client.post("/gdc/md/#{project.pid}/userfilters", payload)
+            res = client.post("/gdc/md/#{project.pid}/userfilters", payload)
 
-          # turn the errors from hashes into array of hashes
-          update_result = res['userFiltersUpdateResult'].flat_map do |k, v|
-            v.map { |r| { status: k.to_sym, user: r, type: :create } }
-          end
+            # turn the errors from hashes into array of hashes
+            update_result = res['userFiltersUpdateResult'].flat_map do |k, v|
+              v.map { |r| { status: k.to_sym, user: r, type: :create } }
+            end
 
-          update_result.map do |result|
-            result[:status] == :failed ? result.merge(GoodData::Helpers.symbolize_keys(result[:user])) : result
+            update_result.map do |result|
+              result[:status] == :failed ? result.merge(GoodData::Helpers.symbolize_keys(result[:user])) : result
+            end
           end
         end
+        project_log_formatter.log_user_filter_results(create_results, to_create)
+        create_errors = create_results.select { |r| r[:status] == :failed }
+        fail "Creating MUFs resulted in errors: #{create_errors}" if create_errors.any?
       end
 
-      project_log_formatter.log_user_filter_results(create_results, to_create)
-      create_errors = create_results.select { |r| r[:status] == :failed }
-      fail "Creating MUFs resulted in errors: #{create_errors}" if create_errors.any?
-
-      delete_results = unless options[:do_not_touch_filters_that_are_not_mentioned]
-                         to_delete.each_slice(100).flat_map do |batch|
-                           batch.flat_map do |related_uri, group|
-                             results = []
-                             if related_uri
-                               res = client.get("/gdc/md/#{project.pid}/userfilters?users=#{related_uri}")
-                               items = res['userFilters']['items'].empty? ? [] : res['userFilters']['items'].first['userFilters']
-                               payload = {
-                                 'userFilters' => {
-                                   'items' => [
-                                     {
-                                       'user' => related_uri,
-                                       'userFilters' => items - group.map(&:uri)
-                                     }
-                                   ]
-                                 }
-                               }
-                               res = client.post("/gdc/md/#{project.pid}/userfilters", payload)
-                               results.concat(res['userFiltersUpdateResult']
+      if to_delete.empty?
+        delete_results = []
+      elsif !options[:do_not_touch_filters_that_are_not_mentioned]
+        delete_results = to_delete.each_slice(100).flat_map do |batch|
+          batch.flat_map do |related_uri, group|
+            results = []
+            if related_uri
+              res = client.get("/gdc/md/#{project.pid}/userfilters?users=#{related_uri}")
+              items = res['userFilters']['items'].empty? ? [] : res['userFilters']['items'].first['userFilters']
+              payload = {
+                'userFilters' => {
+                  'items' => [
+                    {
+                      'user' => related_uri,
+                      'userFilters' => items - group.map(&:uri)
+                    }
+                  ]
+                }
+              }
+              res = client.post("/gdc/md/#{project.pid}/userfilters", payload)
+              results.concat(res['userFiltersUpdateResult']
                                  .flat_map { |k, v| v.map { |r| { status: k.to_sym, user: r, type: :delete } } }
                                  .map { |result| result[:status] == :failed ? result.merge(GoodData::Helpers.symbolize_keys(result[:user])) : result })
-                             end
-                             group.peach(&:delete)
-                             results
-                           end
-                         end
-                       end
+            end
+            group.peach(&:delete)
+            results
+          end
 
-      project_log_formatter.log_user_filter_results(delete_results, to_delete)
-      delete_errors = delete_results.select { |r| r[:status] == :failed } if delete_results
-      fail "Deleting MUFs resulted in errors: #{delete_errors}" if delete_errors && delete_errors.any?
+          project_log_formatter.log_user_filter_results(delete_results, to_delete)
+          delete_errors = delete_results.select { |r| r[:status] == :failed } if delete_results
+          fail "Deleting MUFs resulted in errors: #{delete_errors}" if delete_errors&.any?
+        end
+      end
 
       { created: to_create, deleted: to_delete, results: create_results + (delete_results || []) }
     end
@@ -494,7 +588,7 @@ module GoodData
     # @param file [String | Array] File or array of values to be parsed for filters
     # @param options [Hash] Filter definitions
     # @return [Array<Hash>]
-    def self.get_values(file, options)
+    def self.get_values(file, options = {})
       file.is_a?(Array) ? read_array(file, options) : read_file(file, options)
     end
 
@@ -579,12 +673,12 @@ module GoodData
     # Removes MUFs from to_delete unless in user is in users_brick_input
     # if this does not happen, users that are about to be deleted by users_brick
     # would have all their filters removed now, which is not desirable
-    def self.sanitize_filters_to_delete(to_delete, users_brick_input, project_users)
+    def self.sanitize_filters_to_delete(to_delete, users_brick_input, user_profile_mapping)
       return [] unless users_brick_input && users_brick_input.any?
       user_profiles = users_brick_input.map do |user|
-        result = project_users.find { |u| u.login == user.with_indifferent_access['login'] }
+        result = user_profile_mapping[user.with_indifferent_access['login']]
         next unless result
-        result.profile_url
+        result
       end.compact
       return [] unless user_profiles.any?
       to_delete.reject do |_, value|

data/lib/gooddata/rest/connection.rb

@@ -68,6 +68,8 @@ module GoodData
       RETRY_TIME_COEFFICIENT = 1.5
       RETRYABLE_ERRORS << Net::ReadTimeout if Net.const_defined?(:ReadTimeout)
 
+      RETRYABLE_ERRORS << OpenSSL::SSL::SSLErrorWaitReadable if OpenSSL::SSL.const_defined?(:SSLErrorWaitReadable)
+
       class << self
         def construct_login_payload(username, password)
           res = {
@@ -304,12 +306,12 @@ module GoodData
       end
 
       def refresh_token(_options = {})
-        begin # rubocop:disable RedundantBegin
+        begin # rubocop:disable Style/RedundantBegin
           # avoid infinite loop GET fails with 401
           response = get(TOKEN_PATH, :x_gdc_authsst => sst_token, :dont_reauth => true)
           # Remove when TT sent in headers. Currently we need to parse from body
           merge_headers!(:x_gdc_authtt => GoodData::Helpers.get_path(response, %w(userToken token)))
-        rescue Exception => e # rubocop:disable RescueException
+        rescue Exception => e # rubocop:disable Style/RescueException
           raise e
         end
       end
@@ -688,7 +690,7 @@ ERR
           execution_id = request_id
 
           GoodData.gd_logger.update_store(domain, method, duration, endpoint)
-          GoodData.gd_logger.add Logger::INFO, { endpoint: endpoint, duration: duration, domain: domain,
+          GoodData.gd_logger.add Logger::DEBUG, { endpoint: endpoint, duration: duration, domain: domain,
                                                  execution_id: execution_id, time_stamp: time_stamp }, "rest_call"
         end
       end

data/lib/gooddata/rest/phmap.rb

@@ -82,6 +82,8 @@ module GoodData
 
         ['/gdc/internal/projects/{id}/objects/setPermissions', %r{/gdc/internal/projects/[^\/]+/objects/setPermissions}],
         ['/gdc/internal/projects/{id}/roles', %r{/gdc/internal/projects/[^\/]+/roles}],
+        ['/gdc/dataload/projects/{id}/modelMapping/datasets/bulk/upsert', %r{/gdc/dataload/projects/[^\/]+/modelMapping/datasets/bulk/upsert}],
+        ['/gdc/dataload/internal/projects/{id}/ldmLayout', %r{/gdc/dataload/internal/projects/[^\/]+/ldmLayout}],
         ['/gdc/md/{id}/variables/item/{id}', %r{/gdc/md/[^\/]+/variables/item/[\d]+}],
         ['/gdc/md/{id}/validate/task/{id}', %r{/gdc/md/[^\/]+/validate/task/[^\/]+}],
         ['/gdc/md/{id}/using2/{id}/{id}', %r{/gdc/md/[^\/]+/using2/[\d]+/[\d]+}],

data/lib/gooddata.rb

@@ -37,5 +37,6 @@ require 'backports/2.1.0/array/to_h'
 
 # Helpers
 require 'gooddata/helpers/global_helpers'
+require 'gooddata/lcm/helpers/check_helper'
 
 require 'active_support/core_ext/hash/compact' unless RUBY_VERSION >= '2.5'

data/lib/gooddata_brick_base.rb

@@ -0,0 +1,35 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2010-2022 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'pmap'
+$pmap_default_thread_count = 20 # rubocop:disable GlobalVars
+
+# GoodData Module
+module GoodData
+end
+
+# Modules
+require_relative 'gooddata/core/core'
+require_relative 'gooddata/models/models'
+require_relative 'gooddata/exceptions/exceptions'
+require_relative 'gooddata/helpers/helpers'
+
+# Files
+require_relative 'gooddata/bricks/utils'
+require_relative 'gooddata/bricks/brick'
+require_relative 'gooddata/bricks/base_pipeline'
+require_relative 'gooddata/bricks/middleware/base_middleware'
+require_relative 'gooddata/bricks/middleware/bench_middleware'
+require_relative 'gooddata/bricks/middleware/logger_middleware'
+require_relative 'gooddata/bricks/middleware/decode_params_middleware'
+require_relative 'gooddata/bricks/middleware/aws_middleware'
+require_relative 'gooddata/bricks/middleware/dwh_middleware'
+require_relative 'gooddata/bricks/middleware/bench_middleware'
+
+# CSV Downloader
+require_relative 'gooddata/core/logging'
+require_relative 'gooddata/connection'

data/sonar-project.properties

@@ -0,0 +1,6 @@
+sonar.projectKey=gooddata-ruby
+sonar.organization=gooddata-github
+# Config scan necessary files/folder only
+sonar.sources=bin,lib
+sonar.tests=spec
+sonar.exclusions=**/lib/templates/project/*.erb