gooddata 2.1.19 → 2.3.0

data/lib/gooddata/lcm/actions/collect_projects_warning_status.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+# (C) 2019-2022 GoodData Corporation
+require_relative 'base_action'
+
+module GoodData
+  module LCM2
+    class CollectProjectsWarningStatus < BaseAction
+      DESCRIPTION = 'Collect synced projects status'
+
+      PARAMS = define_params(self) do
+        description 'Abort on error'
+        param :abort_on_error, instance_of(Type::StringType), required: false
+
+        description 'Logger'
+        param :gdc_logger, instance_of(Type::GdLogger), required: false
+
+        description 'Collect synced status'
+        param :collect_synced_status, instance_of(Type::BooleanType), required: false
+
+        description 'Sync failed list'
+        param :sync_failed_list, instance_of(Type::HashType), required: false
+      end
+
+      RESULT_HEADER = %i[segment client project_pid failed_action]
+
+      class << self
+        def call(params)
+          results = []
+          return results unless collect_synced_status(params)
+
+          sync_failed_list = params[SYNC_FAILED_LIST]
+          if sync_failed_list
+            failed_detailed_projects = sync_failed_list[:failed_detailed_projects]
+            failed_detailed_projects.each do |failed_detailed_project|
+              results << {
+                segment: failed_detailed_project[:segment],
+                client: failed_detailed_project[:client_id],
+                project_pid: failed_detailed_project[:project_id],
+                failed_action: failed_detailed_project[:action]
+              }
+            end
+          end
+
+          results
+        end
+
+        def print_result(params)
+          collect_synced_status(params)
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/lcm/actions/collect_segment_clients.rb

@@ -32,9 +32,22 @@ module GoodData
 
         description 'Domain'
         param :domain, instance_of(Type::StringType), required: false
+
+        description 'Abort on error'
+        param :abort_on_error, instance_of(Type::StringType), required: false
+
+        description 'Logger'
+        param :gdc_logger, instance_of(Type::GdLogger), required: false
+
+        description 'Collect synced status'
+        param :collect_synced_status, instance_of(Type::BooleanType), required: false
+
+        description 'Sync failed list'
+        param :sync_failed_list, instance_of(Type::HashType), required: false
       end
 
       RESULT_HEADER = [
+        :segment,
         :from_name,
         :from_pid,
         :to_name,
@@ -89,6 +102,7 @@ module GoodData
                 client_project = segment_client.project
                 to_pid = client_project.pid
                 results << {
+                  segment: segment.segment_id,
                   from_name: master_name,
                   from_pid: master_pid,
                   to_name: client_project.title,

data/lib/gooddata/lcm/actions/initialize_continue_on_error_option.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+# (C) 2019-2022 GoodData Corporation
+require_relative 'base_action'
+
+module GoodData
+  module LCM2
+    class InitializeContinueOnErrorOption < BaseAction
+      DESCRIPTION = 'Initialize continue on error option'
+
+      PARAMS = define_params(self) do
+        description 'Organization Name'
+        param :organization, instance_of(Type::StringType), required: false
+
+        description 'Domain'
+        param :domain, instance_of(Type::StringType), required: false
+
+        description 'DataProduct'
+        param :data_product, instance_of(Type::GDDataProductType), required: false
+
+        description 'Restricts synchronization to specified segments'
+        param :segments_filter, array_of(instance_of(Type::StringType)), required: false
+
+        description 'Abort on error'
+        param :abort_on_error, instance_of(Type::StringType), required: false
+
+        description 'Client Used for Connecting to GD'
+        param :gdc_gd_client, instance_of(Type::GdClientType), required: true
+      end
+
+      class << self
+        def call(params)
+          project_mappings = ThreadSafe::Hash.new
+          client_mappings = ThreadSafe::Hash.new
+          continue_on_error = continue_on_error(params)
+
+          if continue_on_error
+            client = params.gdc_gd_client
+            domain_name = params.organization || params.domain
+            domain = client.domain(domain_name)
+            data_product = params.data_product
+            data_product_segments = domain.segments(:all, data_product)
+
+            if params.segments_filter
+              data_product_segments.select! do |segment|
+                params.segments_filter.include?(segment.segment_id)
+              end
+            end
+
+            data_product_segments.pmap do |segment|
+              segment.clients.map do |segment_client|
+                project = segment_client.project
+                next unless project
+
+                project_mappings[project.pid.to_sym] = {
+                  client_id: segment_client.client_id,
+                  segment_id: segment.segment_id
+                }
+                client_mappings[segment_client.client_id.to_sym] = {
+                  project_id: project.pid,
+                  segment_id: segment.segment_id
+                }
+              end
+            end
+          end
+
+          {
+            params: {
+              collect_synced_status: continue_on_error,
+              sync_failed_list: {
+                project_client_mappings: project_mappings,
+                client_project_mappings: client_mappings,
+                failed_detailed_projects: [],
+                failed_projects: [],
+                failed_clients: [],
+                failed_segments: []
+              }
+            }
+          }
+        end
+
+        def print_result(_params)
+          false
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb

@@ -19,6 +19,18 @@ module GoodData
 
         description 'Synchronization Info'
         param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
+
+        description 'Abort on error'
+        param :abort_on_error, instance_of(Type::StringType), required: false
+
+        description 'Logger'
+        param :gdc_logger, instance_of(Type::GdLogger), required: false
+
+        description 'Collect synced status'
+        param :collect_synced_status, instance_of(Type::BooleanType), required: false
+
+        description 'Sync failed list'
+        param :sync_failed_list, instance_of(Type::HashType), required: false
       end
 
       RESULT_HEADER = %i[from to status]
@@ -27,6 +39,8 @@ module GoodData
         def call(params)
           results = []
           params.synchronize.map do |segment_info|
+            next if sync_failed_segment(segment_info[:segment_id], params)
+
             result = migrate_date_dimension(params, segment_info)
             results.concat(result)
           end
@@ -47,21 +61,32 @@ module GoodData
           # check latest master and previous master
           master_upgrade_datasets = get_upgrade_dates(latest_blueprint, previous_blueprint) if params[:synchronize_ldm].downcase == 'diff_against_master' && previous_blueprint
           unless master_upgrade_datasets&.empty?
+            collect_synced_status = collect_synced_status(params)
+            failed_projects = ThreadSafe::Array.new
+
             segment_info[:to].pmap do |entry|
               pid = entry[:pid]
+              next if sync_failed_project(pid, params)
+
               to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
+              GoodData.logger.info "Migrating date dimension, project: '#{to_project.title}', PID: #{pid}"
               to_blueprint = to_project.blueprint
               upgrade_datasets = get_upgrade_dates(latest_blueprint, to_blueprint)
               next if upgrade_datasets.empty?
 
               message = get_upgrade_message(upgrade_datasets)
+              failed_message = "Failed to migrate date dimension for project #{pid}"
+              update_status = to_project.upgrade_custom_v2(message)
+              process_failed_project(pid, failed_message, failed_projects, collect_synced_status) if collect_synced_status && update_status != 'OK'
 
               results << {
                 from: segment_info[:from],
                 to: pid,
-                status: to_project.upgrade_custom_v2(message)
+                status: update_status
               }
             end
+
+            process_failed_projects(failed_projects, short_name, params) if collect_synced_status
           end
 
           results
@@ -71,9 +96,9 @@ module GoodData
           dest_dates = get_date_dimensions(dest_blueprint) if dest_blueprint
           src_dates = get_date_dimensions(src_blueprint) if src_blueprint
 
-          return false if dest_dates.empty? || src_dates.empty?
-
           upgrade_datasets = []
+          return upgrade_datasets if dest_dates.empty? || src_dates.empty?
+
           dest_dates.each do |dest|
             src_dim = get_date_dimension(src_blueprint, dest[:id])
             next unless src_dim
@@ -103,8 +128,10 @@ module GoodData
           get_date_dimensions(blueprint).any? { |e| e[:urn] == DATE_DIMENSION_CUSTOM_V2 }
         end
 
+        # Get date dimension from blue print. Return nil if date dimension not existing
         def get_date_dimension(blueprint, id)
-          GoodData::Model::ProjectBlueprint.find_date_dimension(blueprint, id)
+          date_dimensions = get_date_dimensions(blueprint)
+          date_dimensions.find { |d| d[:id] == id }
         end
 
         def get_date_dimensions(blueprint)

data/lib/gooddata/lcm/actions/provision_clients.rb

@@ -29,7 +29,16 @@ module GoodData
         param :data_product, instance_of(Type::GDDataProductType), required: false
 
         description 'Logger'
-        param :gdc_logger, instance_of(Type::GdLogger), required: true
+        param :gdc_logger, instance_of(Type::GdLogger), required: false
+
+        description 'Abort on error'
+        param :abort_on_error, instance_of(Type::StringType), required: false
+
+        description 'Collect synced status'
+        param :collect_synced_status, instance_of(Type::BooleanType), required: false
+
+        description 'Sync failed list'
+        param :sync_failed_list, instance_of(Type::HashType), required: false
       end
 
       RESULT_HEADER = [
@@ -45,6 +54,8 @@ module GoodData
           synchronize_projects = []
           data_product = params.data_product
           client = params.gdc_gd_client
+          collect_synced_status = collect_synced_status(params)
+          continue_on_error = continue_on_error(params)
           domain_name = params.organization || params.domain
           fail "Either organisation or domain has to be specified in params" unless domain_name
           domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
@@ -52,6 +63,8 @@ module GoodData
           invalid_client_ids = []
           begin
             results = params.segments.map do |segment|
+              next if sync_failed_segment(segment.segment_id, params)
+
               segment_object = domain.segments(segment.segment_id, data_product)
               tmp = segment_object.provision_client_projects.map do |m|
                 Hash[m.each_pair.to_a].merge(type: :provision_result)
@@ -65,10 +78,21 @@ module GoodData
                     unless entry[:project_uri]
                       error_message = "There was error during provisioning clients: #{entry[:error]}" unless error_message
                       invalid_client_ids << entry[:id]
+                      if collect_synced_status
+                        failed_message = "Failed to provision client #{entry[:id]} in segment #{segment.segment_id}. Error: #{entry[:error]}"
+                        add_failed_client(entry[:id], failed_message, short_name, params)
+                      end
                       next
                     end
+
+                    project_id = entry[:project_uri].split('/').last
+                    if collect_synced_status && entry[:status] == 'CREATED' && entry[:id]
+                      # Update project client mappings when there are create new clients during provision clients of the segment
+                      add_new_clients_to_project_client_mapping(project_id, entry[:id], segment.segment_id, params)
+                    end
+
                     {
-                      pid: entry[:project_uri].split('/').last,
+                      pid: project_id,
                       client_id: entry[:id]
                     }
                   end.compact
@@ -93,15 +117,20 @@ module GoodData
                 end
 
                 params.gdc_logger.debug "Deleted clients: #{deleted_client_ids.join(', ')}"
-                raise error_message unless error_message['TooManyProjectsCreatedException'] || error_message['Max number registered projects']
+                unless error_message['TooManyProjectsCreatedException'] || error_message['Max number registered projects']
+                  raise error_message unless continue_on_error
+
+                  next
+                end
+
                 break tmp
               end
 
               tmp
             end
           rescue => e
-            params.gdc_logger.error "Problem occurs when provisioning clients."
-            raise e
+            params.gdc_logger.error "Problem occurs when provisioning clients. Error: #{e}"
+            raise e unless continue_on_error
           end
 
           results.flatten! if results

data/lib/gooddata/lcm/actions/synchronize_cas.rb

@@ -32,6 +32,15 @@ module GoodData
 
         description 'Additional Hidden Parameters'
         param :additional_hidden_params, instance_of(Type::HashType), required: false
+
+        description 'Abort on error'
+        param :abort_on_error, instance_of(Type::StringType), required: false
+
+        description 'Collect synced status'
+        param :collect_synced_status, instance_of(Type::BooleanType), required: false
+
+        description 'Sync failed list'
+        param :sync_failed_list, instance_of(Type::HashType), required: false
       end
 
       class << self
@@ -46,6 +55,8 @@ module GoodData
           return results unless include_ca
 
           client = params.gdc_gd_client
+          collect_synced_status = collect_synced_status(params)
+          failed_projects = ThreadSafe::Array.new
 
           params.synchronize.each do |info|
             from = info.from
@@ -58,24 +69,33 @@ module GoodData
               next unless ca_scripts
 
               pid = entry[:pid]
+              next if sync_failed_project(pid, params)
+
               ca_chunks = ca_scripts[:maqlDdlChunks]
-              to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
-              params.gdc_logger.info "Synchronizing Computed Attributes to project: '#{to_project.title}', PID: #{pid}"
+              to_project = client.projects(pid)
+              unless to_project
+                process_failed_project(pid, "Invalid 'to' project specified - '#{pid}'", failed_projects, collect_synced_status)
+                next
+              end
 
+              params.gdc_logger.info "Synchronizing Computed Attributes to project: '#{to_project.title}', PID: #{pid}"
+              error_message = nil
               begin
                 ca_chunks.each { |chunk| to_project.execute_maql(chunk) }
               rescue => e
-                raise "Error occured when executing MAQL, project: \"#{to_project.title}\" reason: \"#{e.message}\", chunks: #{ca_chunks.inspect}"
+                error_message = "Error occurred when executing MAQL for project: \"#{to_project.title}\" reason: \"#{e.message}\", chunks: #{ca_chunks.inspect}"
+                process_failed_project(pid, error_message, failed_projects, collect_synced_status)
               end
 
               results << {
                 from: from,
                 to: pid,
-                status: 'ok'
+                status: error_message.nil? ? 'ok' : 'failed'
               }
             end
           end
 
+          process_failed_projects(failed_projects, short_name, params) if collect_synced_status
           results
         end
       end

data/lib/gooddata/lcm/actions/synchronize_clients.rb

@@ -33,8 +33,23 @@ module GoodData
         description 'ADS Client'
         param :ads_client, instance_of(Type::AdsClientType), required: false
 
+        description 'Keep number of old master workspace excluding the latest one'
+        param :keep_only_previous_masters_count, instance_of(Type::StringType), required: false, default: '-1'
+
         description 'Additional Hidden Parameters'
         param :additional_hidden_params, instance_of(Type::HashType), required: false
+
+        description 'Abort on error'
+        param :abort_on_error, instance_of(Type::StringType), required: false
+
+        description 'Logger'
+        param :gdc_logger, instance_of(Type::GdLogger), required: false
+
+        description 'Collect synced status'
+        param :collect_synced_status, instance_of(Type::BooleanType), required: false
+
+        description 'Sync failed list'
+        param :sync_failed_list, instance_of(Type::HashType), required: false
       end
 
       RESULT_HEADER = [
@@ -53,6 +68,7 @@ module GoodData
           domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
           data_product = params.data_product
           domain_segments = domain.segments(:all, data_product)
+          keep_only_previous_masters_count = Integer(params.keep_only_previous_masters_count || "-1")
 
           segments = params.segments.map do |seg|
             domain_segments.find do |s|
@@ -61,19 +77,17 @@ module GoodData
           end
 
           results = segments.map do |segment|
+            next if sync_failed_segment(segment.segment_id, params)
+
             if params.ads_client
-              current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
-                params.release_table_name,
-                params.ads_client,
-                segment.segment_id
-              )
+              master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_ads(params.release_table_name, params.ads_client, segment.segment_id)
             else
-              current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
+              master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
             end
 
+            current_master = master_projects.last
             # TODO: Check res.first.nil? || res.first[:master_project_id].nil?
             master = client.projects(current_master[:master_project_id])
-
             segment.master_project = master
             segment.save
 
@@ -83,14 +97,26 @@ module GoodData
             failed_count = sync_result['failedClients']['count']
 
             if failed_count.to_i > 0
-              fail("#{failed_count} clients failed to synchronize. " \
-                   "Details: #{sync_result['links']['details']}")
+              error_handle(segment, res, collect_synced_status(params), params)
+            end
+
+            if keep_only_previous_masters_count >= 0
+              number_of_deleted_projects = master_projects.count - (keep_only_previous_masters_count + 1)
+
+              if number_of_deleted_projects.positive?
+                begin
+                  removal_master_project_ids = remove_multiple_workspace(params, segment.segment_id, master_projects, number_of_deleted_projects)
+                  remove_old_workspaces_from_release_table(params, domain_name, data_product.data_product_id, segment.segment_id, master_projects, removal_master_project_ids)
+                rescue Exception => e # rubocop:disable RescueException
+                  GoodData.logger.error "Problem occurs when removing old master workspace, reason: #{e.message}"
+                end
+              end
             end
 
             {
               segment: segment.id,
-              master_pid: master.pid,
-              master_name: master.title,
+              master_pid: master.nil? ? '' : master.pid,
+              master_name: master.nil? ? '' : master.title,
               successful_count: sync_result['successfulClients']['count']
             }
           end
@@ -98,6 +124,81 @@ module GoodData
           # Return results
           results
         end
+
+        def remove_multiple_workspace(params, segment_id, master_projects, number_of_deleted_projects)
+          removal_master_project_ids = []
+          need_to_delete_projects = master_projects.take(number_of_deleted_projects)
+
+          need_to_delete_projects.each do |project_wrapper|
+            master_project_id = project_wrapper[:master_project_id]
+            next if master_project_id.to_s.empty?
+
+            begin
+              project = params.gdc_gd_client.projects(master_project_id)
+              if project && !%w[deleted archived].include?(project.state.to_s)
+                GoodData.logger.info "Segment #{segment_id}: Deleting old master workspace, project: '#{project.title}', PID: (#{project.pid})."
+                project.delete
+              end
+              removal_master_project_ids << master_project_id
+              master_projects.delete_if { |p| p[:master_project_id] == master_project_id }
+            rescue Exception => ex # rubocop:disable RescueException
+              GoodData.logger.error "Unable to remove master workspace: '#{master_project_id}', Error: #{ex.message}"
+            end
+          end
+          removal_master_project_ids
+        end
+
+        # rubocop:disable Metrics/ParameterLists
+        def remove_old_workspaces_from_release_table(params, domain_id, data_product_id, segment_id, master_projects, removal_master_project_ids)
+          unless removal_master_project_ids.empty?
+            if params.ads_client
+              GoodData::LCM2::Helpers.delete_master_project_from_ads(params.release_table_name, params.ads_client, segment_id, removal_master_project_ids)
+            else
+              data = master_projects.sort_by { |master| master[:version] }
+              GoodData::LCM2::Helpers.update_master_project_to_nfs(domain_id, data_product_id, segment_id, data)
+            end
+          end
+        end
+        # rubocop:enable Metrics/ParameterLists
+
+        private
+
+        def error_handle(segment, error_result, continue_on_error, params)
+          sync_result = error_result.json['synchronizationResult']
+          success_count = sync_result['successfulClients']['count'].to_i
+          failed_count = sync_result['failedClients']['count'].to_i
+
+          # Synchronize failure for all clients in segment
+          if continue_on_error && success_count.zero? && failed_count.positive?
+            segment_warning_message = "Failed to synchronize clients for #{segment.segment_id} segment. Details: #{sync_result['links']['details']}"
+            add_failed_segment(segment.segment_id, segment_warning_message, short_name, params)
+            return
+          end
+
+          summary_error_message = "#{failed_count} clients failed to synchronize. Details: #{sync_result['links']['details']}"
+          fail(summary_error_message) unless continue_on_error
+
+          GoodData.logger.warn summary_error_message
+          if error_result.details # rubocop:disable Style/SafeNavigation
+            error_result.details.items.each do |item|
+              next if item['status'] == 'OK'
+
+              error_client_id = item['id']
+              error_message = error_message(item, segment)
+              add_failed_client(error_client_id, error_message, short_name, params)
+            end
+          end
+        end
+
+        def error_message(error_item, segment)
+          error_client_id = error_item['id']
+          error_message = "Failed to synchronize #{error_client_id} client in #{segment.segment_id} segment."
+          error_message = "#{error_message}. Detail: #{error_item['error']['message']}" if error_item['error'] && error_item['error']['message']
+
+          error_message = "#{error_message}. Error items: #{error_item['error']['parameters']}" if error_item['error'] && error_item['error']['parameters']
+
+          error_message
+        end
       end
     end
   end

data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb

@@ -0,0 +1,89 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_action'
+
+module GoodData
+  module LCM2
+    class SynchronizeDataSetMapping < BaseAction
+      DESCRIPTION = 'Synchronize Dataset Mappings'
+
+      PARAMS = define_params(self) do
+        description 'Client Used for Connecting to GD'
+        param :gdc_gd_client, instance_of(Type::GdClientType), required: true
+
+        description 'Client used to connecting to development domain'
+        param :development_client, instance_of(Type::GdClientType), required: true
+
+        description 'Synchronization Info'
+        param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
+
+        description 'Logger'
+        param :gdc_logger, instance_of(Type::GdLogger), required: true
+
+        description 'Abort on error'
+        param :abort_on_error, instance_of(Type::StringType), required: false
+
+        description 'Collect synced status'
+        param :collect_synced_status, instance_of(Type::BooleanType), required: false
+
+        description 'Sync failed list'
+        param :sync_failed_list, instance_of(Type::HashType), required: false
+      end
+
+      RESULT_HEADER = %i[from to count status]
+
+      class << self
+        def call(params)
+          results = []
+          collect_synced_status = collect_synced_status(params)
+          failed_projects = ThreadSafe::Array.new
+
+          client = params.gdc_gd_client
+          development_client = params.development_client
+
+          params.synchronize.peach do |info|
+            from_project = info.from
+            to_projects = info.to
+
+            from = development_client.projects(from_project)
+            unless from
+              process_failed_project(from_project, "Invalid 'from' project specified - '#{from_project}'", failed_projects, collect_synced_status)
+              next
+            end
+
+            dataset_mapping = from.dataset_mapping
+            if dataset_mapping&.dig('datasetMappings', 'items').nil? || dataset_mapping['datasetMappings']['items'].empty?
+              params.gdc_logger.info "Project: '#{from.title}', PID: '#{from.pid}' has no model mapping, skip synchronizing model mapping."
+            else
+              to_projects.peach do |to|
+                pid = to[:pid]
+                next if sync_failed_project(pid, params)
+
+                to_project = client.projects(pid)
+                process_failed_project(pid, "Invalid 'to' project specified - '#{pid}'", failed_projects, collect_synced_status) unless to_project
+
+                message_to_project = "to project: '#{to_project.title}', PID: '#{to_project.pid}'"
+                params.gdc_logger.info "Transferring model mapping, from project: '#{from.title}', PID: '#{from.pid}', #{message_to_project}"
+                res = to_project.update_dataset_mapping(dataset_mapping)
+                res[:from] = from.pid
+                results << res
+
+                failed_message = "Failed to transfer model mapping from project '#{from.pid}' to project '#{to_project.pid}'"
+                process_failed_project(pid, failed_message, failed_projects, collect_synced_status) if collect_synced_status && res[:status] != 'OK'
+              end
+            end
+          end
+
+          process_failed_projects(failed_projects, short_name, params) if collect_synced_status
+          # Return results
+          results.flatten
+        end
+      end
+    end
+  end
+end