gooddata 2.1.8 → 2.1.13

data/lib/gooddata/lcm/actions/collect_dynamic_schedule_params.rb

@@ -38,11 +38,11 @@ module GoodData
         def call(params)
           return [] unless params.dynamic_params
 
-          schedule_title_column = params.schedule_title_column || 'schedule_title'
-          client_id_column = params.client_id_column || 'client_id'
-          param_name_column = params.param_name_column || 'param_name'
-          param_value_column = params.param_value_column || 'param_value'
-          param_secure_column = params.param_secure_column || 'param_secure'
+          schedule_title_column = params.schedule_title_column&.downcase || 'schedule_title'
+          client_id_column = params.client_id_column&.downcase || 'client_id'
+          param_name_column = params.param_name_column&.downcase || 'param_name'
+          param_value_column = params.param_value_column&.downcase || 'param_value'
+          param_secure_column = params.param_secure_column&.downcase || 'param_secure'
 
           encryption_key = params.dynamic_params_encryption_key || ''
           exist_encryption_key = encryption_key.blank? ? false : true
@@ -59,7 +59,7 @@ module GoodData
           schedule_hidden_params = {}
           exist_param_secure = false
 
-          CSV.foreach(input_data, :headers => true, :return_headers => false, encoding: 'utf-8') do |row|
+          CSV.foreach(input_data, :headers => true, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
             is_param_secure = row[param_secure_column] == 'true'
             is_decrypt_secure_value = is_param_secure && exist_encryption_key ? true : false
             exist_param_secure = true if is_param_secure

data/lib/gooddata/lcm/actions/collect_segment_clients.rb

@@ -65,8 +65,11 @@ module GoodData
                 segment.segment_id
               )
             else
-              latest_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, segment.segment_id)
+              data_product = params.data_product
+              data_product_id = data_product.data_product_id
+              latest_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product_id, segment.segment_id)
             end
+            raise 'Release table has no data' unless latest_master
 
             latest_master = client.projects(latest_master[:master_project_id])
 

data/lib/gooddata/lcm/actions/collect_segments.rb

@@ -51,8 +51,7 @@ module GoodData
             begin
               project = segment.master_project
             rescue RestClient::BadRequest => e
-              params.gdc_logger.error "Failed to retrieve master project for segment #{segment.id}. Error: #{e}"
-              raise
+              raise "Failed to retrieve master project for segment #{segment.id}. Error: #{e}"
             end
 
             raise "Master project for segment #{segment.id} doesn't exist." unless project

data/lib/gooddata/lcm/actions/collect_users_brick_users.rb

@@ -35,7 +35,7 @@ module GoodData
       class << self
         def call(params)
           users_brick_users = []
-          login_column = params.users_brick_config.login_column || 'login'
+          login_column = params.users_brick_config.login_column&.downcase || 'login'
           users_brick_data_source = GoodData::Helpers::DataSource.new(params.users_brick_config.input_source)
 
           users_brick_data_source_file = without_check(PARAMS, params) do
@@ -45,14 +45,15 @@ module GoodData
             )
           end
           CSV.foreach(users_brick_data_source_file,
-                      headers: true,
-                      return_headers: false,
-                      encoding: 'utf-8') do |row|
-            pid = row[params.multiple_projects_column]
+                      :headers => true,
+                      :return_headers => false,
+                      :header_converters => :downcase,
+                      :encoding => 'utf-8') do |row|
+            pid = row[params.multiple_projects_column&.downcase]
             fail "The set multiple_projects_column '#{params.multiple_projects_column}' of the users input is empty" if !pid && MULTIPLE_COLUMN_MODES.include?(params.sync_mode)
 
             users_brick_users << {
-              login: row[login_column].downcase,
+              login: row[login_column].nil? ? nil : row[login_column].strip.downcase,
               pid: pid
             }
           end

data/lib/gooddata/lcm/actions/create_segment_masters.rb

@@ -71,7 +71,7 @@ module GoodData
             ads_output_stage_prefix = segment_in.ads_output_stage_prefix
 
             # Create master project Postgres
-            version = get_project_version(params, domain_name, segment_id) + 1
+            version = get_project_version(params, domain_name, data_product, segment_id) + 1
 
             master_name = segment_in.master_name.gsub('#{version}', version.to_s)
 
@@ -113,6 +113,7 @@ module GoodData
               status = 'modified'
             end
 
+            segment_in[:data_product_id] = data_product.data_product_id
             segment_in[:master_pid] = project.pid
             segment_in[:version] = version
             segment_in[:timestamp] = Time.now.utc.iso8601
@@ -150,7 +151,7 @@ module GoodData
           }
         end
 
-        def get_project_version(params, domain_name, segment_id)
+        def get_project_version(params, domain_name, data_product, segment_id)
           if params.ads_client
             current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
               params.release_table_name,
@@ -158,7 +159,8 @@ module GoodData
               segment_id
             )
           else
-            current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, segment_id)
+            data_product_id = data_product.data_product_id # data_product was populated by CollectDataProduct action already
+            current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product_id, segment_id)
           end
           return 0 unless current_master
           current_master[:version].to_i

data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+# (C) 2019-2020 GoodData Corporation
+require_relative 'base_action'
+
+# Migrate date dimension urn:gooddata:date or urn:custom:date to urn:custom_v2:date
+module GoodData
+  module LCM2
+    class MigrateGdcDateDimension < BaseAction
+      DESCRIPTION = 'Migrate Gdc Date Dimension'
+      DATE_DIMENSION_CUSTOM_V2 = 'urn:custom_v2:date'
+      DATE_DIMENSION_OLD = %w[urn:gooddata:date urn:custom:date]
+
+      PARAMS = define_params(self) do
+        description 'Client Used for Connecting to GD'
+        param :gdc_gd_client, instance_of(Type::GdClientType), required: true
+
+        description 'Specifies how to synchronize LDM and resolve possible conflicts'
+        param :synchronize_ldm, instance_of(Type::SynchronizeLDM), required: false, default: 'diff_against_master_with_fallback'
+
+        description 'Synchronization Info'
+        param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
+      end
+
+      RESULT_HEADER = %i[from to status]
+
+      class << self
+        def call(params)
+          results = []
+          params.synchronize.map do |segment_info|
+            result = migrate_date_dimension(params, segment_info)
+            results.concat(result)
+          end
+
+          {
+            results: results
+          }
+        end
+
+        def migrate_date_dimension(params, segment_info)
+          results = []
+          client = params.gdc_gd_client
+          latest_blueprint = segment_info[:from_blueprint]
+          # don't migrate when latest master doesn't contain custom v2 date.
+          return results unless contain_v2?(latest_blueprint)
+
+          previous_blueprint = segment_info[:previous_master]&.blueprint
+          # check latest master and previous master
+          master_upgrade_datasets = get_upgrade_dates(latest_blueprint, previous_blueprint) if params[:synchronize_ldm].downcase == 'diff_against_master' && previous_blueprint
+          unless master_upgrade_datasets&.empty?
+            segment_info[:to].pmap do |entry|
+              pid = entry[:pid]
+              to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
+              to_blueprint = to_project.blueprint
+              upgrade_datasets = get_upgrade_dates(latest_blueprint, to_blueprint)
+              next if upgrade_datasets.empty?
+
+              message = get_upgrade_message(upgrade_datasets)
+
+              results << {
+                from: segment_info[:from],
+                to: pid,
+                status: to_project.upgrade_custom_v2(message)
+              }
+            end
+          end
+
+          results
+        end
+
+        def get_upgrade_dates(src_blueprint, dest_blueprint)
+          dest_dates = get_date_dimensions(dest_blueprint) if dest_blueprint
+          src_dates = get_date_dimensions(src_blueprint) if src_blueprint
+
+          return false if dest_dates.empty? || src_dates.empty?
+
+          upgrade_datasets = []
+          dest_dates.each do |dest|
+            src_dim = get_date_dimension(src_blueprint, dest[:id])
+            next unless src_dim
+
+            upgrade_datasets << src_dim[:identifier] if upgrade?(src_dim, dest) && src_dim[:identifier]
+          end
+
+          upgrade_datasets
+        end
+
+        def get_upgrade_message(upgrade_datasets)
+          {
+            upgrade: {
+              dateDatasets: {
+                upgrade: "exact",
+                datasets: upgrade_datasets
+              }
+            }
+          }
+        end
+
+        def upgrade?(src_dim, dest_dim)
+          src_dim[:urn] == DATE_DIMENSION_CUSTOM_V2 && DATE_DIMENSION_OLD.any? { |e| dest_dim[:urn] == e }
+        end
+
+        def contain_v2?(blueprint)
+          get_date_dimensions(blueprint).any? { |e| e[:urn] == DATE_DIMENSION_CUSTOM_V2 }
+        end
+
+        def get_date_dimension(blueprint, id)
+          GoodData::Model::ProjectBlueprint.find_date_dimension(blueprint, id)
+        end
+
+        def get_date_dimensions(blueprint)
+          GoodData::Model::ProjectBlueprint.date_dimensions(blueprint)
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/lcm/actions/set_master_project.rb

@@ -0,0 +1,76 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_action'
+
+module GoodData
+  module LCM2
+    class SetMasterProject < BaseAction
+      DESCRIPTION = 'Set master project'
+
+      PARAMS = define_params(self) do
+        description 'Organization Name'
+        param :organization, instance_of(Type::StringType), required: false
+
+        description 'Domain'
+        param :domain, instance_of(Type::StringType), required: false
+
+        description 'ADS Client'
+        param :ads_client, instance_of(Type::AdsClientType), required: false
+
+        description 'Table Name'
+        param :release_table_name, instance_of(Type::StringType), required: false
+
+        description 'Segments to manage'
+        param :segments, array_of(instance_of(Type::SegmentType)), required: true
+
+        description 'DataProduct to manage'
+        param :data_product, instance_of(Type::GDDataProductType), required: false
+
+        description 'Released master project should be used in next rollout'
+        param :set_master_project, instance_of(Type::StringType), required: false
+      end
+
+      class << self
+        def call(params)
+          results = []
+          domain_name = params.organization || params.domain
+          data_product = params.data_product
+          params.segments.each do |segment_in|
+            version = get_latest_version(params, domain_name, data_product.data_product_id, segment_in.segment_id) + 1
+            segment_in[:data_product_id] = data_product.data_product_id
+            segment_in[:master_pid] = params.set_master_project
+            segment_in[:version] = version
+            segment_in[:timestamp] = Time.now.utc.iso8601
+
+            results << {
+              data_product_id: data_product.data_product_id,
+              segment_id: segment_in.segment_id,
+              version: version
+            }
+          end
+          results
+        end
+
+        def get_latest_version(params, domain_name, data_product_id, segment_id)
+          if params.ads_client
+            current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
+              params.release_table_name,
+              params.ads_client,
+              segment_id
+            )
+          else
+            current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product_id, segment_id)
+          end
+          return 0 unless current_master
+
+          current_master[:version].to_i
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/lcm/actions/synchronize_clients.rb

@@ -68,7 +68,7 @@ module GoodData
                 segment.segment_id
               )
             else
-              current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, segment.segment_id)
+              current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
             end
 
             # TODO: Check res.first.nil? || res.first[:master_project_id].nil?

data/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb

@@ -80,8 +80,7 @@ module GoodData
             res = GoodData::Helpers.symbolize_keys(res)
 
             if res[:syncedResult][:errors]
-              params.gdc_logger.error "Error: #{res[:syncedResult][:errors].pretty_inspect}"
-              fail "Failed to sync processes/schedules for segment #{segment_id}"
+              fail "Failed to sync processes/schedules for segment #{segment_id}. Error: #{res[:syncedResult][:errors].pretty_inspect}"
             end
 
             if res[:syncedResult][:clients]

data/lib/gooddata/lcm/actions/synchronize_ldm.rb

@@ -44,8 +44,13 @@ module GoodData
 
         description 'Specifies how to synchronize LDM and resolve possible conflicts'
         param :synchronize_ldm, instance_of(Type::SynchronizeLDM), required: false, default: 'diff_against_master_with_fallback'
+
+        description 'Enables handling of deprecated objects in the logical data model.'
+        param :include_deprecated, instance_of(Type::BooleanType), required: false, default: false
       end
 
+      RESULT_HEADER = %i[from to status]
+
       class << self
         def call(params)
           results = []
@@ -70,19 +75,29 @@ module GoodData
           results = []
           client = params.gdc_gd_client
           exclude_fact_rule = params.exclude_fact_rule.to_b
+          include_deprecated = params.include_deprecated.to_b
           from_pid = segment_info[:from]
           from = params.development_client.projects(from_pid) || fail("Invalid 'from' project specified - '#{from_pid}'")
-
           GoodData.logger.info "Creating Blueprint, project: '#{from.title}', PID: #{from_pid}"
           blueprint = from.blueprint(include_ca: params.include_computed_attributes.to_b)
+          segment_info[:from_blueprint] = blueprint
           maql_diff = nil
           previous_master = segment_info[:previous_master]
           diff_against_master = %w(diff_against_master_with_fallback diff_against_master)
             .include?(params[:synchronize_ldm].downcase)
+          GoodData.logger.info "Synchronize LDM mode: '#{params[:synchronize_ldm].downcase}'"
           if previous_master && diff_against_master
             maql_diff_params = [:includeGrain]
             maql_diff_params << :excludeFactRule if exclude_fact_rule
+            maql_diff_params << :includeDeprecated if include_deprecated
             maql_diff = previous_master.maql_diff(blueprint: blueprint, params: maql_diff_params)
+            chunks = maql_diff['projectModelDiff']['updateScripts']
+            if chunks.empty?
+              GoodData.logger.info "Synchronize LDM to clients will not proceed in mode \
+'#{params[:synchronize_ldm].downcase}' due to no LDM changes in the new master project. \
+If you had changed LDM of clients manually, please use mode 'diff_against_clients' \
+to force synchronize LDM to clients"
+            end
           end
 
           segment_info[:to] = segment_info[:to].pmap do |entry|
@@ -96,7 +111,8 @@ module GoodData
                 update_preference: params[:update_preference],
                 exclude_fact_rule: exclude_fact_rule,
                 execute_ca_scripts: false,
-                maql_diff: maql_diff
+                maql_diff: maql_diff,
+                include_deprecated: include_deprecated
               )
             rescue MaqlExecutionError => e
               GoodData.logger.info("Applying MAQL to project #{to_project.title} - #{pid} failed. Reason: #{e}")
@@ -106,7 +122,8 @@ module GoodData
                 blueprint,
                 update_preference: params[:update_preference],
                 exclude_fact_rule: exclude_fact_rule,
-                execute_ca_scripts: false
+                execute_ca_scripts: false,
+                include_deprecated: include_deprecated
               )
             end
 

data/lib/gooddata/lcm/actions/synchronize_user_filters.rb

@@ -121,6 +121,7 @@ module GoodData
             users_brick_input: params.users_brick_users
           }
           all_clients = domain.clients(:all, data_product).to_a
+          GoodData.gd_logger.info("Synchronizing in mode=#{mode}, number_of_clients=#{all_clients.size}, data_rows=#{user_filters.size}")
 
           GoodData.logger.info("Synchronizing in mode \"#{mode}\"")
           case mode
@@ -131,6 +132,8 @@ module GoodData
               filter = UserBricksHelper.resolve_client_id(domain, project, params.data_product)
             end
             user_filters = user_filters.select { |f| f[:pid] == filter } if filter
+
+            GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{user_filters.size}")
             sync_user_filters(project, user_filters, run_params, symbolized_config)
           when 'sync_multiple_projects_based_on_pid', 'sync_multiple_projects_based_on_custom_id'
             users_by_project = run_params[:users_brick_input].group_by { |u| u[:pid] }
@@ -144,6 +147,8 @@ module GoodData
               elsif mode == 'sync_multiple_projects_based_on_pid'
                 current_project = client.projects(id)
               end
+
+              GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{id}, data_rows=#{new_filters.size}")
               sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
             end
           when 'sync_domain_client_workspaces'
@@ -170,6 +175,8 @@ module GoodData
               fail "Client #{client_id} does not have project." unless current_project
 
               working_client_ids << client_id
+
+              GoodData.gd_logger.info("Synchronizing in mode=#{mode}, client_id=#{client_id}, data_rows=#{new_filters.size}")
               partial_results = sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
               results.concat(partial_results[:results])
             end
@@ -182,6 +189,8 @@ module GoodData
                   current_project = c.project
                   users = users_by_project[c.client_id]
                   params.gdc_logger.info "Delete all filters in project #{current_project.pid} of client #{c.client_id}"
+
+                  GoodData.gd_logger.info("Delete all filters in project_id=#{current_project.pid}, client_id=#{c.client_id}")
                   current_results = sync_user_filters(current_project, [], run_params.merge(users_brick_input: users), symbolized_config)
 
                   results.concat(current_results[:results])
@@ -214,10 +223,21 @@ module GoodData
           multiple_projects_column = params.multiple_projects_column
           data_source = GoodData::Helpers::DataSource.new(params.input_source)
 
-          without_check(PARAMS, params) do
-            CSV.foreach(File.open(data_source.realize(params), 'r:UTF-8'), headers: csv_with_headers, return_headers: false, encoding: 'utf-8') do |row|
-              filters << row.to_hash.merge(pid: row[multiple_projects_column])
+          tmp = without_check(PARAMS, params) do
+            File.open(data_source.realize(params), 'r:UTF-8')
+          end
+
+          begin
+            GoodData.logger.info('Start reading data')
+            row_count = 0
+            CSV.foreach(tmp, :headers => csv_with_headers, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
+              filters << row.to_hash.merge(pid: row[multiple_projects_column.downcase])
+              row_count += 1
+              GoodData.logger.info("Read #{row_count} rows") if (row_count % 50_000).zero?
             end
+            GoodData.logger.info("Done reading data, total #{row_count} rows")
+          rescue Exception => e # rubocop:disable RescueException
+            fail "There was an error during loading data. Message: #{e.message}. Error: #{e}"
           end
 
           if filters.empty? && %w(sync_multiple_projects_based_on_pid sync_multiple_projects_based_on_custom_id).include?(params.sync_mode)