gooddata 2.1.7-java → 2.1.12-java

data/lib/gooddata/lcm/actions/create_segment_masters.rb

@@ -71,7 +71,7 @@ module GoodData
             ads_output_stage_prefix = segment_in.ads_output_stage_prefix
 
             # Create master project Postgres
-            version = get_project_version(params, domain_name, segment_id) + 1
+            version = get_project_version(params, domain_name, data_product, segment_id) + 1
 
             master_name = segment_in.master_name.gsub('#{version}', version.to_s)
 
@@ -113,6 +113,7 @@ module GoodData
               status = 'modified'
             end
 
+            segment_in[:data_product_id] = data_product.data_product_id
             segment_in[:master_pid] = project.pid
             segment_in[:version] = version
             segment_in[:timestamp] = Time.now.utc.iso8601
@@ -150,7 +151,7 @@ module GoodData
           }
         end
 
-        def get_project_version(params, domain_name, segment_id)
+        def get_project_version(params, domain_name, data_product, segment_id)
           if params.ads_client
             current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
               params.release_table_name,
@@ -158,7 +159,8 @@ module GoodData
               segment_id
             )
           else
-            current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, segment_id)
+            data_product_id = data_product.data_product_id # data_product was populated by CollectDataProduct action already
+            current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product_id, segment_id)
           end
           return 0 unless current_master
           current_master[:version].to_i

data/lib/gooddata/lcm/actions/set_master_project.rb

@@ -0,0 +1,76 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_action'
+
+module GoodData
+  module LCM2
+    class SetMasterProject < BaseAction
+      DESCRIPTION = 'Set master project'
+
+      PARAMS = define_params(self) do
+        description 'Organization Name'
+        param :organization, instance_of(Type::StringType), required: false
+
+        description 'Domain'
+        param :domain, instance_of(Type::StringType), required: false
+
+        description 'ADS Client'
+        param :ads_client, instance_of(Type::AdsClientType), required: false
+
+        description 'Table Name'
+        param :release_table_name, instance_of(Type::StringType), required: false
+
+        description 'Segments to manage'
+        param :segments, array_of(instance_of(Type::SegmentType)), required: true
+
+        description 'DataProduct to manage'
+        param :data_product, instance_of(Type::GDDataProductType), required: false
+
+        description 'Released master project should be used in next rollout'
+        param :set_master_project, instance_of(Type::StringType), required: false
+      end
+
+      class << self
+        def call(params)
+          results = []
+          domain_name = params.organization || params.domain
+          data_product = params.data_product
+          params.segments.each do |segment_in|
+            version = get_latest_version(params, domain_name, data_product.data_product_id, segment_in.segment_id) + 1
+            segment_in[:data_product_id] = data_product.data_product_id
+            segment_in[:master_pid] = params.set_master_project
+            segment_in[:version] = version
+            segment_in[:timestamp] = Time.now.utc.iso8601
+
+            results << {
+              data_product_id: data_product.data_product_id,
+              segment_id: segment_in.segment_id,
+              version: version
+            }
+          end
+          results
+        end
+
+        def get_latest_version(params, domain_name, data_product_id, segment_id)
+          if params.ads_client
+            current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
+              params.release_table_name,
+              params.ads_client,
+              segment_id
+            )
+          else
+            current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product_id, segment_id)
+          end
+          return 0 unless current_master
+
+          current_master[:version].to_i
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/lcm/actions/synchronize_clients.rb

@@ -68,7 +68,7 @@ module GoodData
                 segment.segment_id
               )
             else
-              current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, segment.segment_id)
+              current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
             end
 
             # TODO: Check res.first.nil? || res.first[:master_project_id].nil?

data/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb

@@ -80,8 +80,7 @@ module GoodData
             res = GoodData::Helpers.symbolize_keys(res)
 
             if res[:syncedResult][:errors]
-              params.gdc_logger.error "Error: #{res[:syncedResult][:errors].pretty_inspect}"
-              fail "Failed to sync processes/schedules for segment #{segment_id}"
+              fail "Failed to sync processes/schedules for segment #{segment_id}. Error: #{res[:syncedResult][:errors].pretty_inspect}"
             end
 
             if res[:syncedResult][:clients]

data/lib/gooddata/lcm/actions/synchronize_ldm.rb

@@ -44,6 +44,9 @@ module GoodData
 
         description 'Specifies how to synchronize LDM and resolve possible conflicts'
         param :synchronize_ldm, instance_of(Type::SynchronizeLDM), required: false, default: 'diff_against_master_with_fallback'
+
+        description 'Enables handling of deprecated objects in the logical data model.'
+        param :include_deprecated, instance_of(Type::BooleanType), required: false, default: false
       end
 
       class << self
@@ -70,6 +73,7 @@ module GoodData
           results = []
           client = params.gdc_gd_client
           exclude_fact_rule = params.exclude_fact_rule.to_b
+          include_deprecated = params.include_deprecated.to_b
           from_pid = segment_info[:from]
           from = params.development_client.projects(from_pid) || fail("Invalid 'from' project specified - '#{from_pid}'")
 
@@ -79,10 +83,19 @@ module GoodData
           previous_master = segment_info[:previous_master]
           diff_against_master = %w(diff_against_master_with_fallback diff_against_master)
             .include?(params[:synchronize_ldm].downcase)
+          GoodData.logger.info "Synchronize LDM mode: '#{params[:synchronize_ldm].downcase}'"
           if previous_master && diff_against_master
             maql_diff_params = [:includeGrain]
             maql_diff_params << :excludeFactRule if exclude_fact_rule
+            maql_diff_params << :includeDeprecated if include_deprecated
             maql_diff = previous_master.maql_diff(blueprint: blueprint, params: maql_diff_params)
+            chunks = maql_diff['projectModelDiff']['updateScripts']
+            if chunks.empty?
+              GoodData.logger.info "Synchronize LDM to clients will not proceed in mode \
+'#{params[:synchronize_ldm].downcase}' due to no LDM changes in the new master project. \
+If you had changed LDM of clients manually, please use mode 'diff_against_clients' \
+to force synchronize LDM to clients"
+            end
           end
 
           segment_info[:to] = segment_info[:to].pmap do |entry|
@@ -96,7 +109,8 @@ module GoodData
                 update_preference: params[:update_preference],
                 exclude_fact_rule: exclude_fact_rule,
                 execute_ca_scripts: false,
-                maql_diff: maql_diff
+                maql_diff: maql_diff,
+                include_deprecated: include_deprecated
               )
             rescue MaqlExecutionError => e
               GoodData.logger.info("Applying MAQL to project #{to_project.title} - #{pid} failed. Reason: #{e}")
@@ -106,7 +120,8 @@ module GoodData
                 blueprint,
                 update_preference: params[:update_preference],
                 exclude_fact_rule: exclude_fact_rule,
-                execute_ca_scripts: false
+                execute_ca_scripts: false,
+                include_deprecated: include_deprecated
               )
             end
 

data/lib/gooddata/lcm/actions/synchronize_user_filters.rb

@@ -121,6 +121,7 @@ module GoodData
             users_brick_input: params.users_brick_users
           }
           all_clients = domain.clients(:all, data_product).to_a
+          GoodData.gd_logger.info("Synchronizing in mode=#{mode}, number_of_clients=#{all_clients.size}, data_rows=#{user_filters.size}")
 
           GoodData.logger.info("Synchronizing in mode \"#{mode}\"")
           case mode
@@ -131,6 +132,8 @@ module GoodData
               filter = UserBricksHelper.resolve_client_id(domain, project, params.data_product)
             end
             user_filters = user_filters.select { |f| f[:pid] == filter } if filter
+
+            GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{user_filters.size}")
             sync_user_filters(project, user_filters, run_params, symbolized_config)
           when 'sync_multiple_projects_based_on_pid', 'sync_multiple_projects_based_on_custom_id'
             users_by_project = run_params[:users_brick_input].group_by { |u| u[:pid] }
@@ -144,6 +147,8 @@ module GoodData
               elsif mode == 'sync_multiple_projects_based_on_pid'
                 current_project = client.projects(id)
               end
+
+              GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{id}, data_rows=#{new_filters.size}")
               sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
             end
           when 'sync_domain_client_workspaces'
@@ -170,6 +175,8 @@ module GoodData
               fail "Client #{client_id} does not have project." unless current_project
 
               working_client_ids << client_id
+
+              GoodData.gd_logger.info("Synchronizing in mode=#{mode}, client_id=#{client_id}, data_rows=#{new_filters.size}")
               partial_results = sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
               results.concat(partial_results[:results])
             end
@@ -182,6 +189,8 @@ module GoodData
                   current_project = c.project
                   users = users_by_project[c.client_id]
                   params.gdc_logger.info "Delete all filters in project #{current_project.pid} of client #{c.client_id}"
+
+                  GoodData.gd_logger.info("Delete all filters in project_id=#{current_project.pid}, client_id=#{c.client_id}")
                   current_results = sync_user_filters(current_project, [], run_params.merge(users_brick_input: users), symbolized_config)
 
                   results.concat(current_results[:results])
@@ -214,10 +223,21 @@ module GoodData
           multiple_projects_column = params.multiple_projects_column
           data_source = GoodData::Helpers::DataSource.new(params.input_source)
 
-          without_check(PARAMS, params) do
-            CSV.foreach(File.open(data_source.realize(params), 'r:UTF-8'), headers: csv_with_headers, return_headers: false, encoding: 'utf-8') do |row|
-              filters << row.to_hash.merge(pid: row[multiple_projects_column])
+          tmp = without_check(PARAMS, params) do
+            File.open(data_source.realize(params), 'r:UTF-8')
+          end
+
+          begin
+            GoodData.logger.info('Start reading data')
+            row_count = 0
+            CSV.foreach(tmp, :headers => csv_with_headers, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
+              filters << row.to_hash.merge(pid: row[multiple_projects_column.downcase])
+              row_count += 1
+              GoodData.logger.info("Read #{row_count} rows") if (row_count % 50_000).zero?
             end
+            GoodData.logger.info("Done reading data, total #{row_count} rows")
+          rescue Exception => e # rubocop:disable RescueException
+            fail "There was an error during loading data. Message: #{e.message}. Error: #{e}"
           end
 
           if filters.empty? && %w(sync_multiple_projects_based_on_pid sync_multiple_projects_based_on_custom_id).include?(params.sync_mode)

data/lib/gooddata/lcm/actions/synchronize_users.rb

@@ -190,6 +190,9 @@ module GoodData
             create_non_existing_user_groups: create_non_existing_user_groups,
             user_groups_cache: nil
           }
+          GoodData.gd_logger.info("Synchronizing in mode=#{mode}, data_rows=#{new_users.size}")
+
+          GoodData.logger.info("Synchronizing in mode \"#{mode}\"")
           results = case mode
                     when 'add_to_organization'
                       domain.create_users(new_users.uniq { |u| u[:login] || u[:email] })
@@ -197,6 +200,8 @@ module GoodData
                       user_ids = new_users.uniq { |u| u[:login] || u[:email] }.map { |u| u[:login] || u[:email] }
                       users = user_ids.map { |u| domain.users(u, client: client) }
                       params.gdc_logger.warn "Deleting #{users.count} users from domain #{domain_name}"
+
+                      GoodData.gd_logger.info("Synchronizing in mode=#{mode}, domain=#{domain_name}, data_rows=#{users.count}")
                       users.map(&:delete)
                     when 'sync_project'
                       project.import_users(new_users, common_params)
@@ -204,6 +209,8 @@ module GoodData
                       new_users.group_by { |u| u[:pid] }.flat_map do |project_id, users|
                         begin
                           project = client.projects(project_id)
+
+                          GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project_id}, data_rows=#{users.count}")
                           project.import_users(users, common_params)
                         rescue RestClient::ResourceNotFound
                           fail "Project \"#{project_id}\" was not found. Please check your project ids in the source file"
@@ -215,6 +222,8 @@ module GoodData
                       end
                     when 'sync_one_project_based_on_pid'
                       filtered_users = new_users.select { |u| u[:pid] == project.pid }
+
+                      GoodData.gd_logger.info("Synchronizing in mode=#{mode}, data_rows=#{filtered_users.count}")
                       project.import_users(filtered_users, common_params)
                     when 'sync_one_project_based_on_custom_id'
                       filter_value = UserBricksHelper.resolve_client_id(domain, project, data_product)
@@ -235,6 +244,7 @@ module GoodData
                       end
 
                       GoodData.logger.info("Project #{project.pid} will receive #{filtered_users.count} from #{new_users.count} users")
+                      GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, filtered_users=#{filtered_users.count}, data_rows=#{new_users.count}")
                       project.import_users(filtered_users, common_params)
                     when 'sync_multiple_projects_based_on_custom_id'
                       all_clients = domain.clients(:all, data_product).to_a
@@ -248,6 +258,8 @@ module GoodData
                         fail "Client #{client_id} does not have project." unless project
 
                         GoodData.logger.info("Project #{project.pid} of client #{client_id} will receive #{users.count} users")
+
+                        GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{users.count}")
                         project.import_users(users, common_params)
                       end
                     when 'sync_domain_client_workspaces'
@@ -280,6 +292,8 @@ module GoodData
 
                         working_client_ids << client_id.to_s
                         GoodData.logger.info("Project #{project.pid} of client #{client_id} will receive #{users.count} users")
+
+                        GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{users.count}")
                         project.import_users(users, common_params)
                       end
 
@@ -303,13 +317,18 @@ module GoodData
                             next
                           end
                           GoodData.logger.info("Synchronizing all users in project #{project.pid} of client #{c.client_id}")
+
+                          GoodData.gd_logger.info("Synchronizing all users in project_id=#{project.pid}, client_id=#{c.client_id}")
                           res += project.import_users([], common_params)
                         end
                       end
 
                       res
                     when 'sync_domain_and_project'
+                      GoodData.gd_logger.info("Create users in mode=#{mode}, data_rows=#{new_users.count}")
                       domain.create_users(new_users, ignore_failures: ignore_failures)
+
+                      GoodData.gd_logger.info("Import users in mode=#{mode}, data_rows=#{new_users.count}")
                       project.import_users(new_users, common_params)
                     end
 
@@ -328,38 +347,33 @@ module GoodData
         end
 
         def load_data(params, data_source)
-          first_name_column           = params.first_name_column || 'first_name'
-          last_name_column            = params.last_name_column || 'last_name'
-          login_column                = params.login_column || 'login'
-          password_column             = params.password_column || 'password'
-          email_column                = params.email_column || 'email'
-          role_column                 = params.role_column || 'role'
-          sso_provider_column         = params.sso_provider_column || 'sso_provider'
-          authentication_modes_column = params.authentication_modes_column || 'authentication_modes'
-          user_groups_column          = params.user_groups_column || 'user_groups'
-          language_column             = params.language_column || 'language'
-          company_column              = params.company_column || 'company'
-          position_column             = params.position_column || 'position'
-          country_column              = params.country_column || 'country'
-          phone_column                = params.phone_column || 'phone'
-          ip_whitelist_column         = params.ip_whitelist_column || 'ip_whitelist'
+          first_name_column           = params.first_name_column&.downcase || 'first_name'
+          last_name_column            = params.last_name_column&.downcase || 'last_name'
+          login_column                = params.login_column&.downcase || 'login'
+          password_column             = params.password_column&.downcase || 'password'
+          email_column                = params.email_column&.downcase || 'email'
+          role_column                 = params.role_column&.downcase || 'role'
+          sso_provider_column         = params.sso_provider_column&.downcase || 'sso_provider'
+          authentication_modes_column = params.authentication_modes_column&.downcase || 'authentication_modes'
+          user_groups_column          = params.user_groups_column&.downcase || 'user_groups'
+          language_column             = params.language_column&.downcase || 'language'
+          company_column              = params.company_column&.downcase || 'company'
+          position_column             = params.position_column&.downcase || 'position'
+          country_column              = params.country_column&.downcase || 'country'
+          phone_column                = params.phone_column&.downcase || 'phone'
+          ip_whitelist_column         = params.ip_whitelist_column&.downcase || 'ip_whitelist'
 
           sso_provider = params.sso_provider
           authentication_modes = params.authentication_modes || []
 
-          dwh = params.ads_client
-          if dwh
-            data = dwh.execute_select(params.input_source.query)
-          else
-            tmp = without_check(PARAMS, params) do
-              File.open(data_source.realize(params), 'r:UTF-8')
-            end
+          tmp = without_check(PARAMS, params) do
+            File.open(data_source.realize(params), 'r:UTF-8')
+          end
 
-            begin
-              data = read_csv_file(tmp)
-            rescue Exception => e # rubocop:disable RescueException
-              fail "There was an error during loading users from csv file. Message: #{e.message}. Error: #{e}"
-            end
+          begin
+            data = read_csv_file(tmp)
+          rescue Exception => e # rubocop:disable RescueException
+            fail "There was an error during loading users from csv file. Message: #{e.message}. Error: #{e}"
           end
 
           data.map do |row|
@@ -379,12 +393,18 @@ module GoodData
             ip_whitelist = row[ip_whitelist_column] || row[ip_whitelist_column.to_sym]
             ip_whitelist = ip_whitelist.split(',').map(&:strip) if ip_whitelist
 
+            user_login = row[login_column] || row[login_column.to_sym]
+            user_login = user_login.strip unless user_login.nil?
+
+            user_email = row[email_column] || row[login_column] || row[email_column.to_sym] || row[login_column.to_sym]
+            user_email = user_email.strip unless user_email.nil?
+
             {
               :first_name => row[first_name_column] || row[first_name_column.to_sym],
               :last_name => row[last_name_column] || row[last_name_column.to_sym],
-              :login => row[login_column] || row[login_column.to_sym],
+              :login => user_login,
               :password => row[password_column] || row[password_column.to_sym],
-              :email => row[email_column] || row[login_column] || row[email_column.to_sym] || row[login_column.to_sym],
+              :email => user_email,
               :role => row[role_column] || row[role_column.to_sym],
               :sso_provider => sso_provider || row[sso_provider_column] || row[sso_provider_column.to_sym],
               :authentication_modes => modes,
@@ -405,7 +425,7 @@ module GoodData
           res = []
           row_count = 0
 
-          CSV.foreach(path, :headers => true) do |row|
+          CSV.foreach(path, :headers => true, :header_converters => :downcase, :encoding => 'utf-8') do |row|
             if block_given?
               data = yield row
             else

data/lib/gooddata/lcm/actions/update_release_table.rb

@@ -39,6 +39,7 @@ module GoodData
       class << self
         def call(params)
           client = params.gdc_gd_client
+          GoodData.gd_logger.info("Update release table: use_nfs=#{params.ads_client.nil?}")
 
           domain_name = params.organization || params.domain
           fail "Either organisation or domain has to be specified in params" unless domain_name
@@ -48,6 +49,7 @@ module GoodData
             segment_id = segment_in.segment_id
 
             placeholders = {
+              data_product_id: segment_in[:data_product_id],
               segment_id: segment_in[:segment_id],
               master_project_id: segment_in[:master_pid],
               version: segment_in[:version],
@@ -83,7 +85,11 @@ module GoodData
 
             params.ads_client.execute(query)
           else
-            GoodData::LCM2::Helpers.update_latest_master_to_nfs(domain_id, placeholders[:segment_id], placeholders[:master_project_id], placeholders[:version])
+            data_product_id = placeholders[:data_product_id]
+            segment_id = placeholders[:segment_id]
+            master_pid = placeholders[:master_project_id]
+            version = placeholders[:version]
+            GoodData::LCM2::Helpers.update_latest_master_to_nfs(domain_id, data_product_id, segment_id, master_pid, version)
           end
         end
       end