gooddata 2.1.6 → 2.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 40420df95b074cfc5ddbc5fb7713d05f15998b91e4214fdea4c00a634d7d4a82
-  data.tar.gz: 829426dcd8a5907471c7962c738caf2925797e05a6e0c6572acf2a230e268dfc
+  metadata.gz: 3e17ae0abbc9f71e3d9448bfe5c37de8504d0624b1135dd084134a41fb09bbf9
+  data.tar.gz: b3149a806b9b190d49cf3f7788a893e78cd7a5c6faf9ff062f7d913bca1614fa
 SHA512:
-  metadata.gz: bb18709196a65a5d062774160c8d9819d443acb2efb8f3504faa6e6f201f61051d1628ae8e510d581498272d30be8354b55015ef9f4debca751c8146c5eca860
-  data.tar.gz: 2fd6d24efcbd79f99d40b1c13be8c355ec075abc96843b0cae8397daba2f8b3031bdfb76a164055a668f0844a24bdcb71c1f9f393d7bf58ea892d901094ed433
+  metadata.gz: d1f731c22d261086b28a46db79dba73f4780a51d681d7f5361a23633b5dfdaf3f0780caebad83c572972b5eb9517bae9e94f37aeefcf4923d3139d8b5f49db67
+  data.tar.gz: c98718e4a72a3e66c2211f51568d1f5dce93ba9464efbd4930cbd0a692bb3a341cf4d858d85c36307e86dbeeee859677c53d670662bc00879b19eb8d4f9454a6

data/.gdc-ii-config.yaml

@@ -6,10 +6,11 @@ microservices:
         BRICKS_VERSION: 'VERSION'
 
 integratedTests:
-  - env: bash
+  - kind: 'image'
+    env: bash
     path: .
     vault:
-      GD_SPEC_PASSWORD: ruby/test/bia_encryption_key
+      GD_SPEC_PASSWORD: "$VAULT_SPECIAL_PREFIX/ruby-test-bia-encryption-key"
     command: ./bin/run_smoke_tests.sh
     image: harbor.intgdc.com/staging/lcm-bricks:GIT_REV
     repo_mount_dir: /src

data/.pronto.yml

@@ -1,3 +1,6 @@
+all:
+  exclude:
+    - 'spec/**/*'
 verbose: true
 github:
   slug: gooddata/gooddata-ruby

data/CHANGELOG.md

@@ -1,4 +1,16 @@
 # GoodData Ruby SDK Changelog
+## 2.1.7
+ - BUGFIX: TMA-1577 Bump brick version to 3.7.9
+ - FEATURE: MSF-16279 Allow gooddata-ruby to be installed with Rails 6 apps
+ - BUGFIX: TMA-1580 Increase java heap space and logging loading rows
+ - BUGFIX: TMA-1577 fix user group cache
+ - BUGFIX: TMA-1589 K8s bricks don't propagate errors to execution log
+ - BUGFIX: TMA-1571 Users Brick fails when segments_filter is used
+ - BUGFIX: TMA-1569 Don't create MUF when all its value is NULL
+ - II-603: Start using new NGVault secret paths
+ - II-550: Update structure of integrated tests
+ - CONFIG: TMA-1555 Separate test results for sdk and project test
+
 ## 2.1.6
  - FEATURE: TMA-783 Use SST for ADS connection as default
  - BUGFIX: TMA-1549 Using valid sso provider during testing update users

data/SDK_VERSION

@@ -1 +1 @@
-2.1.6
+2.1.7

data/VERSION

@@ -1 +1 @@
-3.7.4
+3.7.9

data/bin/run_brick.rb

@@ -33,7 +33,9 @@ begin
   params['GDC_EXECUTION_ID'] = execution_id
   log.info "action=#{brick_type}_execution status=start commit_hash=#{commit_hash} execution_id=#{execution_id}"
   brick_pipeline.call(params)
-rescue StandardError => e
+rescue Exception => e # rubocop:disable RescueException
+	execution_log = GoodData.logger
+	execution_log.error "Execution failed. Message: #{e.message}. Error: #{e}" unless execution_log.nil?
   log.info "action=#{brick_type}_execution status=failed commit_hash=#{commit_hash} execution_id=#{execution_id} exception=#{e}"
   raise
 end

data/bin/users.sh

@@ -1,3 +1,5 @@
 #!/usr/bin/env bash
 
+export JRUBY_OPTS=-J-Xmx2560m
+
 /bin/bash -l -c "bundle exec ./bin/run_brick.rb users_brick"

data/gooddata.gemspec

@@ -54,7 +54,7 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'sqlite3' if RUBY_PLATFORM != 'java'
 
-  s.add_dependency 'activesupport', '> 4.2.9', '< 5.2'
+  s.add_dependency 'activesupport', '> 4.2.9', '< 6.1'
 
   s.add_dependency 'aws-sdk-s3', '~> 1.16'
   s.add_dependency 'docile', '~> 1.1'

data/lib/gooddata/bricks/middleware/logger_middleware.rb

@@ -45,7 +45,8 @@ module GoodData
           logger.level = params['GDC_LOG_LEVEL'] || 'info'
         end
         GoodData.logger = logger
-        logger.info('Pipeline starts')
+        request_id = params.include?('GDC_REQUEST_ID') ? ", request_id=#{params['GDC_REQUEST_ID']}" : ''
+        logger.info("Pipeline starts #{request_id}")
         params['GDC_LOGGER'] = logger
         GoodData.logging_http_on if params['HTTP_LOGGING'] && params['HTTP_LOGGING'].to_b
 

data/lib/gooddata/bricks/pipeline.rb

@@ -30,8 +30,8 @@ module GoodData
 
       def self.users_brick_pipeline
         prepare([
-          DecodeParamsMiddleware,
           LoggerMiddleware,
+          DecodeParamsMiddleware,
           BenchMiddleware,
           GoodDataMiddleware,
           AWSMiddleware,
@@ -42,8 +42,8 @@ module GoodData
 
       def self.user_filters_brick_pipeline
         prepare([
-          DecodeParamsMiddleware,
           LoggerMiddleware,
+          DecodeParamsMiddleware,
           BenchMiddleware,
           GoodDataMiddleware,
           AWSMiddleware,
@@ -55,8 +55,8 @@ module GoodData
 
       def self.release_brick_pipeline
         prepare([
-        DecodeParamsMiddleware,
         LoggerMiddleware,
+        DecodeParamsMiddleware,
         BenchMiddleware,
         GoodDataMiddleware,
         AWSMiddleware,
@@ -67,8 +67,8 @@ module GoodData
 
       def self.provisioning_brick_pipeline
         prepare([
-        DecodeParamsMiddleware,
         LoggerMiddleware,
+        DecodeParamsMiddleware,
         BenchMiddleware,
         GoodDataMiddleware,
         AWSMiddleware,
@@ -79,8 +79,8 @@ module GoodData
 
       def self.rollout_brick_pipeline
         prepare([
-        DecodeParamsMiddleware,
         LoggerMiddleware,
+        DecodeParamsMiddleware,
         BenchMiddleware,
         GoodDataMiddleware,
         AWSMiddleware,
@@ -92,8 +92,8 @@ module GoodData
       def self.hello_world_brick_pipeline
         prepare(
           [
-            DecodeParamsMiddleware,
             LoggerMiddleware,
+            DecodeParamsMiddleware,
             BenchMiddleware,
             HelloWorldBrick
           ]
@@ -103,8 +103,8 @@ module GoodData
       def self.help_brick_pipeline
         prepare(
           [
-            DecodeParamsMiddleware,
             LoggerMiddleware,
+            DecodeParamsMiddleware,
             BenchMiddleware,
             HelpBrick
           ]

data/lib/gooddata/lcm/actions/synchronize_users.rb

@@ -251,7 +251,8 @@ module GoodData
                         project.import_users(users, common_params)
                       end
                     when 'sync_domain_client_workspaces'
-                      domain_clients = domain.clients(:all, data_product)
+                      all_domain_clients = domain.clients(:all, data_product)
+                      domain_clients = all_domain_clients
                       if params.segments
                         segment_uris = params.segments.map(&:uri)
                         domain_clients = domain_clients.select { |c| segment_uris.include?(c.segment_uri) }
@@ -262,7 +263,13 @@ module GoodData
                         fail "Client id cannot be empty" if client_id.blank?
 
                         c = domain_clients.detect { |specific_client| specific_client.id == client_id }
-                        fail "The client \"#{client_id}\" does not exist in data product \"#{data_product.data_product_id}\"" if c.nil?
+                        if c.nil?
+                          filtered_client = all_domain_clients.detect { |f_client| f_client.id == client_id }
+                          fail "The client \"#{client_id}\" does not exist in data product \"#{data_product.data_product_id}\"" if filtered_client.nil?
+
+                          GoodData.logger.info("Client \"#{client_id}\" is not belong to filtered segments")
+                          next
+                        end
 
                         if params.segments && !segment_uris.include?(c.segment_uri)
                           GoodData.logger.info("Client #{client_id} is outside segments_filter #{params.segments}")
@@ -347,7 +354,12 @@ module GoodData
             tmp = without_check(PARAMS, params) do
               File.open(data_source.realize(params), 'r:UTF-8')
             end
-            data = CSV.read(tmp, headers: true)
+
+            begin
+              data = read_csv_file(tmp)
+            rescue Exception => e # rubocop:disable RescueException
+              fail "There was an error during loading users from csv file. Message: #{e.message}. Error: #{e}"
+            end
           end
 
           data.map do |row|
@@ -387,6 +399,30 @@ module GoodData
             }
           end
         end
+
+        def read_csv_file(path)
+          GoodData.logger.info('Start reading csv file')
+          res = []
+          row_count = 0
+
+          CSV.foreach(path, :headers => true) do |row|
+            if block_given?
+              data = yield row
+            else
+              data = row
+            end
+
+            if data
+              row_count += 1
+              res << data
+            end
+
+            GoodData.logger.info("Read #{row_count} rows") if (row_count % 50_000).zero?
+          end
+
+          GoodData.logger.info("Done reading csv file, total #{row_count} rows")
+          res
+        end
       end
     end
   end

data/lib/gooddata/lcm/lcm2.rb

@@ -315,7 +315,7 @@ module GoodData
           # Invoke action
           begin
             out = run_action action, params
-          rescue => e
+          rescue Exception => e # rubocop:disable RescueException
             errors << {
               action: action,
               err: e,

data/lib/gooddata/models/project.rb

@@ -1611,7 +1611,7 @@ module GoodData
       whitelisted_new_users, whitelisted_users = whitelist_users(new_users.map(&:to_hash), users_list, options[:whitelists])
 
       # First check that if groups are provided we have them set up
-      options[:user_groups_cache] = check_groups(new_users.map(&:to_hash).flat_map { |u| u[:user_group] || [] }.uniq, options[:user_groups_cache], options)
+      user_groups_cache = check_groups(new_users.map(&:to_hash).flat_map { |u| u[:user_group] || [] }.uniq, options[:user_groups_cache], options)
 
       # conform the role on list of new users so we can diff them with the users coming from the project
       diffable_new_with_default_role = whitelisted_new_users.map do |u|
@@ -1707,11 +1707,17 @@ module GoodData
         end
         mappings.group_by { |_, g| g }.each do |g, mapping|
           remote_users = mapping.map { |user, _| user }.map { |login| users_lookup[login] && users_lookup[login].uri }.reject(&:nil?)
+          GoodData.logger.info("Assigning users #{remote_users} to group #{g}")
           next if remote_users.empty?
-          user_groups(g).set_members(remote_users)
+          existing_group = user_groups(g)
+          if existing_group.nil?
+            GoodData.logger.warn("Group #{g} not found!!!")
+          else
+            existing_group.set_members(remote_users)
+          end
         end
         mentioned_groups = mappings.map(&:last).uniq
-        groups_to_cleanup = options[:user_groups_cache].reject { |g| mentioned_groups.include?(g.name) }
+        groups_to_cleanup = user_groups_cache.reject { |g| mentioned_groups.include?(g.name) }
         # clean all groups not mentioned with exception of whitelisted users
         groups_to_cleanup.each do |g|
           g.set_members(whitelist_users(g.members.map(&:to_hash), [], options[:whitelists], :include).first.map { |x| x[:uri] })
@@ -1750,11 +1756,12 @@ module GoodData
     end
 
     def check_groups(specified_groups, user_groups_cache = nil, options = {})
-      user_groups_cache = user_groups if user_groups_cache.nil? || user_groups_cache.empty?
-      groups = user_groups_cache.map(&:name)
+      current_user_groups = user_groups if user_groups_cache.nil? || user_groups_cache.empty?
+      groups = current_user_groups.map(&:name)
       missing_groups = specified_groups - groups
       if options[:create_non_existing_user_groups]
         missing_groups.each do |g|
+          GoodData.logger.info("Creating group #{g}")
           create_group(name: g, description: g)
         end
       else
@@ -1764,7 +1771,7 @@ module GoodData
             "#{groups.join(',')} and you asked for #{missing_groups.join(',')}"
         end
       end
-      user_groups_cache
+      current_user_groups
     end
 
     # Update user
@@ -1778,7 +1785,8 @@ module GoodData
       payload = generate_user_payload(user_uri, 'ENABLED', roles)
       res = client.post(url, payload)
       failure = GoodData::Helpers.get_path(res, %w(projectUsersUpdateResult failed))
-      fail ArgumentError, "User #{user_uri} could not be aded. #{failure.first['message']}" unless failure.blank?
+      fail ArgumentError, "User #{user_uri} could not be added. #{failure.first['message']}" unless failure.blank?
+
       res
     end
     alias_method :add_user, :set_user_roles

data/lib/gooddata/models/user_filters/user_filter_builder.rb

@@ -199,16 +199,6 @@ module GoodData
       attr_cache
     end
 
-    def self.create_muf_expression_cache(filters)
-      filters.reduce({}) do |a, e|
-        a[e[:login]] = e[:filters].reduce({}) do |h, filter|
-          h[filter[:label]] = :none
-          h
-        end
-        a
-      end
-    end
-
     # Walks over provided labels and picks those that have fewer than certain amount of values
     # This tries to balance for speed when working with small datasets (like users)
     # so it precaches the values and still be able to function for larger ones even
@@ -223,9 +213,12 @@ module GoodData
     # @param lookups_cache e.g. { 'label_uri': { "jirka@gooddata.com": 'value_uri' }}
     def self.create_expression(filter, labels_cache, lookups_cache, attr_cache, options = {})
       values = filter[:values]
+      # Do not create MUF for label when all its values is NULL (https://jira.intgdc.com/browse/TMA-1361)
+      non_null_values = values.select { |value| !value.nil? && value.downcase != 'null' }
+      return ['TRUE', []] if non_null_values.empty?
+
       label = labels_cache[filter[:label]]
       errors = []
-      muf_expression_cache = options[:muf_expression_cache]
 
       element_uris_by_values = Hash[values.map do |v|
         if lookups_cache.key?(label.uri)
@@ -254,12 +247,7 @@ module GoodData
                      # as the proper MUF can not be constructed yet
                      case options[:type]
                      when :muf
-                       muf_expression_cache[filter[:label]] = :all
-                       if muf_expression_cache.value?(:none) || muf_expression_cache.value?(:restrict)
-                         'TRUE'
-                       else
-                         '1 <> 1'
-                       end
+                       '1 <> 1'
                      when :variable
                        nil
                      end
@@ -267,12 +255,10 @@ module GoodData
                      # create a filter that is always true to ensure the user can see all data
                      'TRUE'
                    elsif filter[:over] && filter[:to]
-                     muf_expression_cache[filter[:label]] = :restrict
                      over = attr_cache[filter[:over]]
                      to = attr_cache[filter[:to]]
                      "([#{label.attribute_uri}] IN (#{element_uris.sort.map { |e| '[' + e + ']' }.join(', ')})) OVER [#{over && over.uri}] TO [#{to && to.uri}]"
                    else
-                     muf_expression_cache[filter[:label]] = :restrict
                      "[#{label.attribute_uri}] IN (#{element_uris.sort.map { |e| '[' + e + ']' }.join(', ')})"
                    end
       [expression, errors]
@@ -305,9 +291,8 @@ module GoodData
       small_labels = get_small_labels(labels_cache)
       lookups_cache = create_lookups_cache(small_labels)
       attrs_cache = create_attrs_cache(filters, options)
-      muf_expression_cache = create_muf_expression_cache(filters)
       create_filter_proc = proc do |login, f|
-        expression, errors = create_expression(f, labels_cache, lookups_cache, attrs_cache, options.merge(muf_expression_cache: muf_expression_cache[login]))
+        expression, errors = create_expression(f, labels_cache, lookups_cache, attrs_cache, options)
         safe_login = login.downcase
         profiles_uri = if options[:type] == :muf
                          project_user = project_users.find { |u| u.login == safe_login }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gooddata
 version: !ruby/object:Gem::Version
-  version: 2.1.6
+  version: 2.1.7
 platform: ruby
 authors:
 - Pavel Kolesnikov
@@ -14,7 +14,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-19 00:00:00.000000000 Z
+date: 2019-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: license_finder
@@ -291,7 +291,7 @@ dependencies:
         version: 4.2.9
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -301,7 +301,7 @@ dependencies:
         version: 4.2.9
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.1'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement