gooddata 2.1.9 → 2.1.14

data/lib/gooddata/lcm/actions/collect_tagged_objects.rb

@@ -50,7 +50,8 @@ module GoodData
             if transfer_all
               vizs = MdObject.query('visualizationObject', MdObject, client: development_client, project: from_project)
               viz_widgets = MdObject.query('visualizationWidget', MdObject, client: development_client, project: from_project)
-              objects = (from_project.reports.to_a + from_project.metrics.to_a + from_project.variables.to_a + vizs.to_a + viz_widgets.to_a).map(&:uri)
+              theme = MdObject.query('theme', MdObject, client: development_client, project: from_project)
+              objects = (from_project.reports.to_a + from_project.metrics.to_a + from_project.variables.to_a + vizs.to_a + viz_widgets.to_a + theme.to_a).map(&:uri)
             elsif production_tags.any?
               objects = from_project.find_by_tag(production_tags)
             end

data/lib/gooddata/lcm/actions/collect_users_brick_users.rb

@@ -35,7 +35,7 @@ module GoodData
       class << self
         def call(params)
           users_brick_users = []
-          login_column = params.users_brick_config.login_column || 'login'
+          login_column = params.users_brick_config.login_column&.downcase || 'login'
           users_brick_data_source = GoodData::Helpers::DataSource.new(params.users_brick_config.input_source)
 
           users_brick_data_source_file = without_check(PARAMS, params) do
@@ -45,14 +45,15 @@ module GoodData
             )
           end
           CSV.foreach(users_brick_data_source_file,
-                      headers: true,
-                      return_headers: false,
-                      encoding: 'utf-8') do |row|
-            pid = row[params.multiple_projects_column]
+                      :headers => true,
+                      :return_headers => false,
+                      :header_converters => :downcase,
+                      :encoding => 'utf-8') do |row|
+            pid = row[params.multiple_projects_column&.downcase]
             fail "The set multiple_projects_column '#{params.multiple_projects_column}' of the users input is empty" if !pid && MULTIPLE_COLUMN_MODES.include?(params.sync_mode)
 
             users_brick_users << {
-              login: row[login_column].downcase,
+              login: row[login_column].nil? ? nil : row[login_column].strip.downcase,
               pid: pid
             }
           end

data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+# (C) 2019-2020 GoodData Corporation
+require_relative 'base_action'
+
+# Migrate date dimension urn:gooddata:date or urn:custom:date to urn:custom_v2:date
+module GoodData
+  module LCM2
+    class MigrateGdcDateDimension < BaseAction
+      DESCRIPTION = 'Migrate Gdc Date Dimension'
+      DATE_DIMENSION_CUSTOM_V2 = 'urn:custom_v2:date'
+      DATE_DIMENSION_OLD = %w[urn:gooddata:date urn:custom:date]
+
+      PARAMS = define_params(self) do
+        description 'Client Used for Connecting to GD'
+        param :gdc_gd_client, instance_of(Type::GdClientType), required: true
+
+        description 'Specifies how to synchronize LDM and resolve possible conflicts'
+        param :synchronize_ldm, instance_of(Type::SynchronizeLDM), required: false, default: 'diff_against_master_with_fallback'
+
+        description 'Synchronization Info'
+        param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
+      end
+
+      RESULT_HEADER = %i[from to status]
+
+      class << self
+        def call(params)
+          results = []
+          params.synchronize.map do |segment_info|
+            result = migrate_date_dimension(params, segment_info)
+            results.concat(result)
+          end
+
+          {
+            results: results
+          }
+        end
+
+        def migrate_date_dimension(params, segment_info)
+          results = []
+          client = params.gdc_gd_client
+          latest_blueprint = segment_info[:from_blueprint]
+          # don't migrate when latest master doesn't contain custom v2 date.
+          return results unless contain_v2?(latest_blueprint)
+
+          previous_blueprint = segment_info[:previous_master]&.blueprint
+          # check latest master and previous master
+          master_upgrade_datasets = get_upgrade_dates(latest_blueprint, previous_blueprint) if params[:synchronize_ldm].downcase == 'diff_against_master' && previous_blueprint
+          unless master_upgrade_datasets&.empty?
+            segment_info[:to].pmap do |entry|
+              pid = entry[:pid]
+              to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
+              to_blueprint = to_project.blueprint
+              upgrade_datasets = get_upgrade_dates(latest_blueprint, to_blueprint)
+              next if upgrade_datasets.empty?
+
+              message = get_upgrade_message(upgrade_datasets)
+
+              results << {
+                from: segment_info[:from],
+                to: pid,
+                status: to_project.upgrade_custom_v2(message)
+              }
+            end
+          end
+
+          results
+        end
+
+        def get_upgrade_dates(src_blueprint, dest_blueprint)
+          dest_dates = get_date_dimensions(dest_blueprint) if dest_blueprint
+          src_dates = get_date_dimensions(src_blueprint) if src_blueprint
+
+          return false if dest_dates.empty? || src_dates.empty?
+
+          upgrade_datasets = []
+          dest_dates.each do |dest|
+            src_dim = get_date_dimension(src_blueprint, dest[:id])
+            next unless src_dim
+
+            upgrade_datasets << src_dim[:identifier] if upgrade?(src_dim, dest) && src_dim[:identifier]
+          end
+
+          upgrade_datasets
+        end
+
+        def get_upgrade_message(upgrade_datasets)
+          {
+            upgrade: {
+              dateDatasets: {
+                upgrade: "exact",
+                datasets: upgrade_datasets
+              }
+            }
+          }
+        end
+
+        def upgrade?(src_dim, dest_dim)
+          src_dim[:urn] == DATE_DIMENSION_CUSTOM_V2 && DATE_DIMENSION_OLD.any? { |e| dest_dim[:urn] == e }
+        end
+
+        def contain_v2?(blueprint)
+          get_date_dimensions(blueprint).any? { |e| e[:urn] == DATE_DIMENSION_CUSTOM_V2 }
+        end
+
+        def get_date_dimension(blueprint, id)
+          GoodData::Model::ProjectBlueprint.find_date_dimension(blueprint, id)
+        end
+
+        def get_date_dimensions(blueprint)
+          GoodData::Model::ProjectBlueprint.date_dimensions(blueprint)
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/lcm/actions/set_master_project.rb

@@ -0,0 +1,76 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_action'
+
+module GoodData
+  module LCM2
+    class SetMasterProject < BaseAction
+      DESCRIPTION = 'Set master project'
+
+      PARAMS = define_params(self) do
+        description 'Organization Name'
+        param :organization, instance_of(Type::StringType), required: false
+
+        description 'Domain'
+        param :domain, instance_of(Type::StringType), required: false
+
+        description 'ADS Client'
+        param :ads_client, instance_of(Type::AdsClientType), required: false
+
+        description 'Table Name'
+        param :release_table_name, instance_of(Type::StringType), required: false
+
+        description 'Segments to manage'
+        param :segments, array_of(instance_of(Type::SegmentType)), required: true
+
+        description 'DataProduct to manage'
+        param :data_product, instance_of(Type::GDDataProductType), required: false
+
+        description 'Released master project should be used in next rollout'
+        param :set_master_project, instance_of(Type::StringType), required: false
+      end
+
+      class << self
+        def call(params)
+          results = []
+          domain_name = params.organization || params.domain
+          data_product = params.data_product
+          params.segments.each do |segment_in|
+            version = get_latest_version(params, domain_name, data_product.data_product_id, segment_in.segment_id) + 1
+            segment_in[:data_product_id] = data_product.data_product_id
+            segment_in[:master_pid] = params.set_master_project
+            segment_in[:version] = version
+            segment_in[:timestamp] = Time.now.utc.iso8601
+
+            results << {
+              data_product_id: data_product.data_product_id,
+              segment_id: segment_in.segment_id,
+              version: version
+            }
+          end
+          results
+        end
+
+        def get_latest_version(params, domain_name, data_product_id, segment_id)
+          if params.ads_client
+            current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
+              params.release_table_name,
+              params.ads_client,
+              segment_id
+            )
+          else
+            current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product_id, segment_id)
+          end
+          return 0 unless current_master
+
+          current_master[:version].to_i
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/lcm/actions/synchronize_ldm.rb

@@ -49,6 +49,8 @@ module GoodData
         param :include_deprecated, instance_of(Type::BooleanType), required: false, default: false
       end
 
+      RESULT_HEADER = %i[from to status]
+
       class << self
         def call(params)
           results = []
@@ -76,9 +78,9 @@ module GoodData
           include_deprecated = params.include_deprecated.to_b
           from_pid = segment_info[:from]
           from = params.development_client.projects(from_pid) || fail("Invalid 'from' project specified - '#{from_pid}'")
-
           GoodData.logger.info "Creating Blueprint, project: '#{from.title}', PID: #{from_pid}"
           blueprint = from.blueprint(include_ca: params.include_computed_attributes.to_b)
+          segment_info[:from_blueprint] = blueprint
           maql_diff = nil
           previous_master = segment_info[:previous_master]
           diff_against_master = %w(diff_against_master_with_fallback diff_against_master)
@@ -89,6 +91,13 @@ module GoodData
             maql_diff_params << :excludeFactRule if exclude_fact_rule
             maql_diff_params << :includeDeprecated if include_deprecated
             maql_diff = previous_master.maql_diff(blueprint: blueprint, params: maql_diff_params)
+            chunks = maql_diff['projectModelDiff']['updateScripts']
+            if chunks.empty?
+              GoodData.logger.info "Synchronize LDM to clients will not proceed in mode \
+'#{params[:synchronize_ldm].downcase}' due to no LDM changes in the new master project. \
+If you had changed LDM of clients manually, please use mode 'diff_against_clients' \
+to force synchronize LDM to clients"
+            end
           end
 
           segment_info[:to] = segment_info[:to].pmap do |entry|

data/lib/gooddata/lcm/actions/synchronize_user_filters.rb

@@ -230,8 +230,8 @@ module GoodData
           begin
             GoodData.logger.info('Start reading data')
             row_count = 0
-            CSV.foreach(tmp, headers: csv_with_headers, return_headers: false, encoding: 'utf-8') do |row|
-              filters << row.to_hash.merge(pid: row[multiple_projects_column])
+            CSV.foreach(tmp, :headers => csv_with_headers, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
+              filters << row.to_hash.merge(pid: row[multiple_projects_column.downcase])
               row_count += 1
               GoodData.logger.info("Read #{row_count} rows") if (row_count % 50_000).zero?
             end

data/lib/gooddata/lcm/actions/synchronize_users.rb

@@ -347,38 +347,33 @@ module GoodData
         end
 
         def load_data(params, data_source)
-          first_name_column           = params.first_name_column || 'first_name'
-          last_name_column            = params.last_name_column || 'last_name'
-          login_column                = params.login_column || 'login'
-          password_column             = params.password_column || 'password'
-          email_column                = params.email_column || 'email'
-          role_column                 = params.role_column || 'role'
-          sso_provider_column         = params.sso_provider_column || 'sso_provider'
-          authentication_modes_column = params.authentication_modes_column || 'authentication_modes'
-          user_groups_column          = params.user_groups_column || 'user_groups'
-          language_column             = params.language_column || 'language'
-          company_column              = params.company_column || 'company'
-          position_column             = params.position_column || 'position'
-          country_column              = params.country_column || 'country'
-          phone_column                = params.phone_column || 'phone'
-          ip_whitelist_column         = params.ip_whitelist_column || 'ip_whitelist'
+          first_name_column           = params.first_name_column&.downcase || 'first_name'
+          last_name_column            = params.last_name_column&.downcase || 'last_name'
+          login_column                = params.login_column&.downcase || 'login'
+          password_column             = params.password_column&.downcase || 'password'
+          email_column                = params.email_column&.downcase || 'email'
+          role_column                 = params.role_column&.downcase || 'role'
+          sso_provider_column         = params.sso_provider_column&.downcase || 'sso_provider'
+          authentication_modes_column = params.authentication_modes_column&.downcase || 'authentication_modes'
+          user_groups_column          = params.user_groups_column&.downcase || 'user_groups'
+          language_column             = params.language_column&.downcase || 'language'
+          company_column              = params.company_column&.downcase || 'company'
+          position_column             = params.position_column&.downcase || 'position'
+          country_column              = params.country_column&.downcase || 'country'
+          phone_column                = params.phone_column&.downcase || 'phone'
+          ip_whitelist_column         = params.ip_whitelist_column&.downcase || 'ip_whitelist'
 
           sso_provider = params.sso_provider
           authentication_modes = params.authentication_modes || []
 
-          dwh = params.ads_client
-          if dwh
-            data = dwh.execute_select(params.input_source.query)
-          else
-            tmp = without_check(PARAMS, params) do
-              File.open(data_source.realize(params), 'r:UTF-8')
-            end
+          tmp = without_check(PARAMS, params) do
+            File.open(data_source.realize(params), 'r:UTF-8')
+          end
 
-            begin
-              data = read_csv_file(tmp)
-            rescue Exception => e # rubocop:disable RescueException
-              fail "There was an error during loading users from csv file. Message: #{e.message}. Error: #{e}"
-            end
+          begin
+            data = read_csv_file(tmp)
+          rescue Exception => e # rubocop:disable RescueException
+            fail "There was an error during loading users from csv file. Message: #{e.message}. Error: #{e}"
           end
 
           data.map do |row|
@@ -398,12 +393,18 @@ module GoodData
             ip_whitelist = row[ip_whitelist_column] || row[ip_whitelist_column.to_sym]
             ip_whitelist = ip_whitelist.split(',').map(&:strip) if ip_whitelist
 
+            user_login = row[login_column] || row[login_column.to_sym]
+            user_login = user_login.strip unless user_login.nil?
+
+            user_email = row[email_column] || row[login_column] || row[email_column.to_sym] || row[login_column.to_sym]
+            user_email = user_email.strip unless user_email.nil?
+
             {
               :first_name => row[first_name_column] || row[first_name_column.to_sym],
               :last_name => row[last_name_column] || row[last_name_column.to_sym],
-              :login => row[login_column] || row[login_column.to_sym],
+              :login => user_login,
               :password => row[password_column] || row[password_column.to_sym],
-              :email => row[email_column] || row[login_column] || row[email_column.to_sym] || row[login_column.to_sym],
+              :email => user_email,
               :role => row[role_column] || row[role_column.to_sym],
               :sso_provider => sso_provider || row[sso_provider_column] || row[sso_provider_column.to_sym],
               :authentication_modes => modes,
@@ -424,7 +425,7 @@ module GoodData
           res = []
           row_count = 0
 
-          CSV.foreach(path, :headers => true) do |row|
+          CSV.foreach(path, :headers => true, :header_converters => :downcase, :encoding => 'utf-8') do |row|
             if block_given?
               data = yield row
             else

data/lib/gooddata/lcm/lcm2.rb

@@ -106,6 +106,14 @@ module GoodData
         UpdateReleaseTable
       ],
 
+      release_set_master_project: [
+        EnsureReleaseTable,
+        CollectDataProduct,
+        SegmentsFilter,
+        SetMasterProject,
+        UpdateReleaseTable
+      ],
+
       provision: [
         EnsureReleaseTable,
         CollectDataProduct,
@@ -130,6 +138,7 @@ module GoodData
         EnsureTechnicalUsersDomain,
         EnsureTechnicalUsersProject,
         SynchronizeLdm,
+        MigrateGdcDateDimension,
         SynchronizeClients,
         SynchronizeComputedAttributes,
         CollectDymanicScheduleParams,
@@ -271,8 +280,14 @@ module GoodData
 
         GoodData.gd_logger.brick = mode
 
+        final_mode = if params.set_master_project && mode == 'release'
+                       'release_set_master_project'
+                     else
+                       mode
+                     end
+
         # Get actions for mode specified
-        actions = get_mode_actions(mode)
+        actions = get_mode_actions(final_mode)
 
         if params.actions
           actions = params.actions.map do |action|
@@ -305,6 +320,12 @@ module GoodData
           skip_actions.include?(action.name.split('::').last)
         end
 
+        sync_mode = params.fetch(:sync_mode, nil)
+        if mode == 'users' && %w[add_to_organization remove_from_organization].include?(sync_mode)
+          actions = actions.reject do |action|
+            %w[CollectDataProduct CollectSegments].include?(action.name.split('::').last)
+          end
+        end
         check_unused_params(actions, params)
         print_action_names(mode, actions)
 

data/lib/gooddata/models/domain.rb

@@ -222,24 +222,26 @@ module GoodData
         domain = client.domain(domain)
         if id == :all
           GoodData.logger.warn("Retrieving all users from domain #{domain.name}")
-          Enumerator.new do |y|
-            page_limit = opts[:page_limit] || 1000
-            offset = opts[:offset] || 0
-            loop do
-              begin
-                tmp = client(opts).get("#{domain.uri}/users", params: { offset: offset, limit: page_limit })
-              end
-
-              tmp['accountSettings']['items'].each do |user_data|
-                user = client.create(GoodData::Profile, user_data)
-                y << user if user
-              end
-              break if tmp['accountSettings']['items'].count < page_limit
-              offset += page_limit
+          all_users = []
+          page_limit = opts[:page_limit] || 1000
+          offset = opts[:offset] || 0
+          loop do
+            begin
+              tmp = client(opts).get("#{domain.uri}/users", params: { offset: offset, limit: page_limit })
             end
+
+            tmp['accountSettings']['items'].each do |user_data|
+              user = client.create(GoodData::Profile, user_data)
+              all_users << user if user
+            end
+            break if tmp['accountSettings']['items'].count < page_limit
+
+            offset += page_limit
           end
+
+          all_users
         else
-          find_user_by_login(domain, id)
+          find_user_by_login(domain, id, opts)
         end
       end