gooddata 2.2.0 → 2.3.0

data/lib/gooddata/lcm/actions/update_metric_formats.rb

@@ -35,14 +35,26 @@ module GoodData
 
         description 'Localization query'
         param :localization_query, instance_of(Type::StringType), required: false
+
+        description 'Abort on error'
+        param :abort_on_error, instance_of(Type::StringType), required: false
+
+        description 'Collect synced status'
+        param :collect_synced_status, instance_of(Type::BooleanType), required: false
+
+        description 'Sync failed list'
+        param :sync_failed_list, instance_of(Type::HashType), required: false
       end
 
       RESULT_HEADER = %i[action ok_clients error_clients]
 
       class << self
         def load_metric_data(params)
+          collect_synced_status = collect_synced_status(params)
+
           if params&.dig(:input_source, :metric_format) && params[:input_source][:metric_format].present?
-            metric_input_source = validate_input_source(params[:input_source])
+            metric_input_source = validate_input_source(params[:input_source], collect_synced_status)
+            return nil unless metric_input_source
           else
             return nil
           end
@@ -68,10 +80,14 @@ module GoodData
           metrics_hash
         end
 
-        def validate_input_source(input_source)
+        def validate_input_source(input_source, continue_on_error)
           type = input_source[:type] if input_source&.dig(:type)
           metric_format = input_source[:metric_format]
-          raise "Incorrect configuration: 'type' of 'input_source' is required" if type.blank?
+          if type.blank?
+            raise "Incorrect configuration: 'type' of 'input_source' is required" unless continue_on_error
+
+            return nil
+          end
 
           modified_input_source = input_source
           case type
@@ -154,8 +170,9 @@ module GoodData
           data_product_clients = data_product.clients
           number_client_ok = 0
           number_client_error = 0
+          collect_synced_status = collect_synced_status(params)
           metric_group.each do |client_id, formats|
-            next unless updated_clients.include?(client_id)
+            next if !updated_clients.include?(client_id) || (collect_synced_status && sync_failed_client(client_id, params))
 
             client = data_product_clients.find { |c| c.id == client_id }
             begin

data/lib/gooddata/lcm/exceptions/lcm_execution_warning.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+# (C) 2019-2022 GoodData Corporation
+
+module GoodData
+  class LcmExecutionWarning < RuntimeError
+    DEFAULT_MSG = 'Existing errors during lcm execution'
+
+    attr_reader :summary_error
+
+    def initialize(summary_error, message = DEFAULT_MSG)
+      super(message)
+      @summary_error = summary_error
+    end
+  end
+end

data/lib/gooddata/lcm/helpers/check_helper.rb

@@ -10,6 +10,9 @@ module GoodData
   module LCM2
     class Helpers
       class << self
+        ABORT_ON_ERROR_PARAM = 'abort_on_error'.to_sym
+        COLLECT_SYNCED_STATUS = 'collect_synced_status'.to_sym
+
         def check_params(specification, params)
           specification.keys.each do |param_name|
             value = params.send(param_name)
@@ -39,6 +42,22 @@ module GoodData
             end
           end
         end
+
+        def continue_on_error(params)
+          params.include?(ABORT_ON_ERROR_PARAM) && !to_bool(ABORT_ON_ERROR_PARAM, params[ABORT_ON_ERROR_PARAM])
+        end
+
+        def collect_synced_status(params)
+          params.include?(COLLECT_SYNCED_STATUS) && to_bool(COLLECT_SYNCED_STATUS, params[COLLECT_SYNCED_STATUS])
+        end
+
+        def to_bool(key, value)
+          return value if value.is_a?(TrueClass) || value.is_a?(FalseClass)
+          return true if value =~ /^(true|t|yes|y|1)$/i
+          return false if value == '' || value =~ /^(false|f|no|n|0)$/i
+
+          raise ArgumentError, "Invalid '#{value}' boolean value for '#{key}' parameter"
+        end
       end
     end
   end

data/lib/gooddata/lcm/lcm2.rb

@@ -17,6 +17,7 @@ require_relative 'actions/actions'
 require_relative 'dsl/dsl'
 require_relative 'helpers/helpers'
 require_relative 'exceptions/lcm_execution_error'
+require_relative 'exceptions/lcm_execution_warning'
 
 using TrueExtensions
 using FalseExtensions
@@ -97,10 +98,13 @@ module GoodData
         ImportObjectCollections,
         SynchronizeComputedAttributes,
         SynchronizeDataSetMapping,
+        SynchronizeLdmLayout,
         SynchronizeProcesses,
         SynchronizeSchedules,
         SynchronizeColorPalette,
         SynchronizeUserGroups,
+        SynchronizePPDashboardPermissions,
+        SynchronizeKDDashboardPermissions,
         SynchronizeNewSegments,
         UpdateReleaseTable
       ],
@@ -122,13 +126,19 @@ module GoodData
         CollectClients,
         AssociateClients,
         RenameExistingClientProjects,
+        InitializeContinueOnErrorOption,
         ProvisionClients,
         UpdateMetricFormats,
         EnsureTechnicalUsersDomain,
         EnsureTechnicalUsersProject,
         CollectDymanicScheduleParams,
         SynchronizeDataSetMapping,
-        SynchronizeETLsInSegment
+        SynchronizeLdmLayout,
+        SynchronizeUserGroups,
+        SynchronizePPDashboardPermissions,
+        SynchronizeKDDashboardPermissions,
+        SynchronizeETLsInSegment,
+        CollectProjectsWarningStatus
       ],
 
       rollout: [
@@ -138,14 +148,20 @@ module GoodData
         CollectSegmentClients,
         EnsureTechnicalUsersDomain,
         EnsureTechnicalUsersProject,
+        InitializeContinueOnErrorOption,
         SynchronizeLdm,
         SynchronizeDataSetMapping,
+        SynchronizeLdmLayout,
         MigrateGdcDateDimension,
         SynchronizeClients,
+        SynchronizeUserGroups,
+        SynchronizePPDashboardPermissions,
+        SynchronizeKDDashboardPermissions,
         UpdateMetricFormats,
         SynchronizeComputedAttributes,
         CollectDymanicScheduleParams,
-        SynchronizeETLsInSegment
+        SynchronizeETLsInSegment,
+        CollectProjectsWarningStatus
       ],
 
       users: [
@@ -340,7 +356,7 @@ module GoodData
           # Invoke action
           begin
             out = run_action action, params
-          rescue Exception => e # rubocop:disable RescueException
+          rescue Exception => e # rubocop:disable Style/RescueException
             errors << {
               action: action,
               err: e,
@@ -362,7 +378,7 @@ module GoodData
           params.merge!(new_params)
 
           # Print action result
-          print_action_result(action, res)
+          print_action_result(action, res) if action.send(:print_result, params)
 
           # Store result for final summary
           results << res
@@ -389,9 +405,28 @@ module GoodData
           end
         end
 
+        process_sync_failed_projects(params) if GoodData::LCM2::Helpers.collect_synced_status(params) && strict_mode
+
         result
       end
 
+      def process_sync_failed_projects(params)
+        sync_failed_list = params[:sync_failed_list]
+        sync_project_list = sync_failed_list[:project_client_mappings]
+        sync_failed_project_list = sync_failed_list[:failed_detailed_projects]
+
+        if sync_project_list && sync_failed_project_list && sync_project_list.size.positive? && sync_failed_project_list.size.positive?
+          failed_project = sync_failed_project_list[0]
+          summary_message = "Existing errors during execution. See log for details"
+          error_message = failed_project[:message]
+          if sync_project_list.size == sync_failed_project_list.size
+            raise GoodData::LcmExecutionError.new(summary_message, error_message)
+          else
+            raise GoodData::LcmExecutionWarning.new(summary_message, error_message)
+          end
+        end
+      end
+
       def run_action(action, params)
         begin
           GoodData.gd_logger.start_action action, GoodData.gd_logger
@@ -401,6 +436,12 @@ module GoodData
           BaseAction.check_params(action.const_get('PARAMS'), params)
           params.setup_filters(action.const_get('PARAMS'))
           out = action.send(:call, params)
+        rescue Exception => e # rubocop:disable Style/RescueException
+          # Log to splunk
+          GoodData.gd_logger.error("action=#{action} status=failed message=#{e} exception=#{e.backtrace}")
+          # Log to execution log
+          GoodData.logger.error("Execution #{action} failed. Error: #{e}. Detail:#{e.backtrace}")
+          raise e
         ensure
           params.clear_filters
           GoodData.gd_logger.end_action GoodData.gd_logger

data/lib/gooddata/lcm/user_bricks_helper.rb

@@ -26,6 +26,15 @@ module GoodData
 
           goodot_id.empty? ? client.id : goodot_id
         end
+
+        def non_working_clients(domain_clients, working_client_ids)
+          non_working_clients = []
+          domain_clients.each do |c|
+            non_working_clients << c unless working_client_ids.include?(c.client_id.to_s)
+          end
+
+          non_working_clients
+        end
       end
     end
   end

data/lib/gooddata/mixins/inspector.rb

@@ -10,7 +10,7 @@ module GoodData
   module Mixin
     # When an RSpec test like this fails,
     #
-    #   @my_array.should == [@some_model, @some_model2]
+    #   expect(@my_array).to == [@some_model, @some_model2]
     #
     # RSpec will call inspect on each of the objects to "help" you figure out
     # what went wrong. Well, inspect will usually dump a TON OF SHIT and make trying

data/lib/gooddata/models/ldm_layout.rb

@@ -0,0 +1,38 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2022 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+module GoodData
+  class LdmLayout
+    DEFAULT_EMPTY_LDM_LAYOUT = {
+      "ldmLayout" => {
+        "layout" => []
+      }
+    }
+
+    LDM_LAYOUT_URI = '/gdc/dataload/internal/projects/%<project_id>s/ldmLayout'
+
+    class << self
+      def get(opts = { :client => GoodData.connection, :project => GoodData.project })
+        client, project = GoodData.get_client_and_project(opts)
+        get_uri = LDM_LAYOUT_URI % { project_id: project.pid }
+
+        client.get(get_uri)
+      end
+    end
+
+    def initialize(data)
+      @data = data
+    end
+
+    def save(opts)
+      client, project = GoodData.get_client_and_project(opts)
+      post_uri = LDM_LAYOUT_URI % { project_id: project.pid }
+
+      client.post(post_uri, @data, opts)
+    end
+  end
+end

data/lib/gooddata/models/project.rb

@@ -31,6 +31,7 @@ require_relative 'project_log_formatter'
 require_relative 'project_role'
 require_relative 'blueprint/blueprint'
 require_relative 'dataset_mapping'
+require_relative 'ldm_layout'
 
 require_relative 'metadata/scheduled_mail'
 require_relative 'metadata/scheduled_mail/dashboard_attachment'
@@ -272,6 +273,29 @@ module GoodData
         }
       end
 
+      def get_ldm_layout(from_project)
+        GoodData::LdmLayout.get(:client => from_project.client, :project => from_project)
+      rescue StandardError => e
+        GoodData.logger.warn "An unexpected error when get ldm layout. Error: #{e.message}"
+        GoodData::LdmLayout::DEFAULT_EMPTY_LDM_LAYOUT
+      end
+
+      def save_ldm_layout(ldm_layout_json, to_project)
+        ldm_layout = GoodData::LdmLayout.new(ldm_layout_json)
+        begin
+          ldm_layout.save(:client => to_project.client, :project => to_project)
+          status = "OK"
+        rescue StandardError => e
+          GoodData.logger.warn "An unexpected error when save ldm layout. Error: #{e.message}"
+          status = "Failed"
+        end
+
+        {
+          to: to_project.pid,
+          status: status
+        }
+      end
+
       # @param from_project The source project
       # @param to_project The target project
       # @param options Optional parameters
@@ -416,16 +440,6 @@ module GoodData
           new_group.project = to_project
           new_group.description = ug.description
           new_group.save
-          # migrate dashboard "grantees"
-          dashboards = from_project.dashboards
-          dashboards.each do |dashboard|
-            new_dashboard = to_project.dashboards.select { |dash| dash.title == dashboard.title }.first
-            next unless new_dashboard
-            grantee = dashboard.grantees['granteeURIs']['items'].select { |item| item['aclEntryURI']['grantee'].split('/').last == ug.links['self'].split('/').last }.first
-            next unless grantee
-            permission = grantee['aclEntryURI']['permission']
-            new_dashboard.grant(:member => new_group, :permission => permission)
-          end
 
           {
             from: from_project.pid,
@@ -436,6 +450,68 @@ module GoodData
         end
       end
 
+      def transfer_dashboard_permission(from_project, to_project, source_dashboards, target_dashboards)
+        source_user_groups = from_project.user_groups
+        target_user_groups = to_project.user_groups
+
+        source_dashboards.each do |source_dashboard|
+          target_dashboard = target_dashboards.select { |dash| dash.title == source_dashboard.title }.first
+          next unless target_dashboard
+
+          begin
+            source_group_dashboards = dashboard_user_groups(source_user_groups, source_dashboard)
+            target_group_dashboards = dashboard_user_groups(target_user_groups, target_dashboard)
+
+            common_group_names = source_group_dashboards.flat_map { |s| target_group_dashboards.select { |t| t[:name] == s[:name] } }.map { |x| [x[:name], true] }.to_h
+
+            remove_user_groups_from_dashboard(target_group_dashboards, target_dashboard, common_group_names)
+            add_user_groups_to_dashboard(source_group_dashboards, target_dashboard, common_group_names, target_user_groups)
+          rescue StandardError => e
+            GoodData.logger.warn "Failed to synchronize dashboard permission from project: '#{from_project.title}', PID: '#{from_project.pid}' to project: '#{to_project.title}', PID: '#{to_project.pid}', dashboard: '#{target_dashboard.title}', dashboard uri: '#{target_dashboard.uri}' . Error: #{e.message} - #{e}" # rubocop:disable Metrics/LineLength
+          end
+        end
+      end
+
+      def dashboard_user_groups(user_groups, dashboard)
+        group_dashboards = []
+        dashboard_grantees = dashboard.grantees['granteeURIs']['items'].select { |item| item['aclEntryURI']['grantee'].include?('/usergroups/') }
+
+        dashboard_grantees.each do |dashboard_grantee|
+          permission = dashboard_grantee['aclEntryURI']['permission']
+          group_id = dashboard_grantee['aclEntryURI']['grantee'].split('/').last
+          user_group = user_groups.select { |group| group.links['self'].split('/').last == group_id }.first
+          next unless user_group
+
+          group_dashboards << {
+            name: user_group.name,
+            user_group: user_group,
+            permission: permission
+          }
+        end
+        group_dashboards
+      end
+
+      def remove_user_groups_from_dashboard(group_dashboards, dashboard, common_group_names)
+        group_dashboards.each do |group_dashboard|
+          group_name = group_dashboard[:name]
+          next if common_group_names && common_group_names[group_name]
+
+          dashboard.revoke(:member => group_dashboard[:user_group], :permission => group_dashboard[:permission])
+        end
+      end
+
+      def add_user_groups_to_dashboard(group_dashboards, dashboard, common_group_names, target_user_groups)
+        group_dashboards.each do |group_dashboard|
+          group_name = group_dashboard[:name]
+          next if common_group_names && common_group_names[group_name]
+
+          target_user_group = target_user_groups.select { |group| group.name == group_name }.first
+          next unless target_user_group
+
+          dashboard.grant(:member => target_user_group, :permission => group_dashboard[:permission])
+        end
+      end
+
       # Clones project along with etl and schedules.
       #
       # @param client [GoodData::Rest::Client] GoodData client to be used for connection
@@ -443,7 +519,7 @@ module GoodData
       # Object to be cloned from. Can be either segment in which case we take
       # the master, client in which case we take its project, string in which
       # case we treat is as an project object or directly project.
-      def transfer_schedules(from_project, to_project)
+      def transfer_schedules(from_project, to_project, has_cycle_trigger = false)
         to_project_processes = to_project.processes.sort_by(&:name)
         from_project_processes = from_project.processes.sort_by(&:name)
         from_project_processes.reject!(&:add_v2_component?)
@@ -496,15 +572,23 @@ module GoodData
         end
 
         results = []
+        update_trigger_schedules = []
         loop do # rubocop:disable Metrics/BlockLength
           break if stack.empty?
           state, changed_schedule = stack.shift
+          lazy_update_trigger_info = false
           if state == :added
             schedule_spec = changed_schedule
             if schedule_spec[:after] && !schedule_cache[schedule_spec[:after]]
-              stack << [state, schedule_spec]
-              next
+              if has_cycle_trigger
+                # The schedule is triggered by another schedule
+                lazy_update_trigger_info = true
+              else
+                stack << [state, schedule_spec]
+                next
+              end
             end
+
             remote_process, process_spec = cache.find do |_remote, local, schedule|
               (schedule_spec[:process_id] == local.process_id) && (schedule.name == schedule_spec[:name])
             end
@@ -517,8 +601,21 @@ module GoodData
             if process_spec.type != :dataload
               executable = schedule_spec[:executable] || (process_spec.type == :ruby ? 'main.rb' : 'main.grf')
             end
+
             params = schedule_parameters(schedule_spec)
-            created_schedule = remote_process.create_schedule(schedule_spec[:cron] || schedule_cache[schedule_spec[:after]], executable, params)
+
+            if lazy_update_trigger_info
+              # Temporary update nil for trigger info. The trigger info will be update late after transfer all schedules
+              created_schedule = remote_process.create_schedule(nil, executable, params)
+              update_trigger_schedules << {
+                state: :added,
+                schedule: created_schedule,
+                after: schedule_spec[:after]
+              }
+            else
+              created_schedule = remote_process.create_schedule(schedule_spec[:cron] || schedule_cache[schedule_spec[:after]], executable, params)
+            end
+
             schedule_cache[created_schedule.name] = created_schedule
 
             results << {
@@ -529,8 +626,13 @@ module GoodData
           else
             schedule_spec = changed_schedule[:new_obj]
             if schedule_spec[:after] && !schedule_cache[schedule_spec[:after]]
-              stack << [state, schedule_spec]
-              next
+              if has_cycle_trigger
+                # The schedule is triggered by another schedule
+                lazy_update_trigger_info = true
+              else
+                stack << [state, schedule_spec]
+                next
+              end
             end
 
             remote_process, process_spec = cache.find do |i|
@@ -543,8 +645,12 @@ module GoodData
 
             schedule.params = (schedule_spec[:params] || {})
             schedule.cron = schedule_spec[:cron] if schedule_spec[:cron]
-            schedule.after = schedule_cache[schedule_spec[:after]] if schedule_spec[:after]
-            schedule.trigger_execution_status = schedule_cache[schedule_spec[:trigger_execution_status]] if schedule_spec[:after]
+
+            unless lazy_update_trigger_info
+              schedule.after = schedule_cache[schedule_spec[:after]] if schedule_spec[:after]
+              schedule.trigger_execution_status = schedule_cache[schedule_spec[:trigger_execution_status]] if schedule_spec[:after]
+            end
+
             schedule.hidden_params = schedule_spec[:hidden_params] || {}
             if process_spec.type != :dataload
               schedule.executable = schedule_spec[:executable] || (process_spec.type == :ruby ? 'main.rb' : 'main.grf')
@@ -556,6 +662,15 @@ module GoodData
             schedule.save
             schedule_cache[schedule.name] = schedule
 
+            if lazy_update_trigger_info
+              update_trigger_schedules << {
+                state: :changed,
+                schedule: schedule,
+                after: schedule_spec[:after],
+                trigger_execution_status: schedule_spec[:trigger_execution_status]
+              }
+            end
+
             results << {
               state: :changed,
               process: remote_process,
@@ -564,6 +679,22 @@ module GoodData
           end
         end
 
+        if has_cycle_trigger
+          update_trigger_schedules.each do |update_trigger_schedule|
+            working_schedule = update_trigger_schedule[:schedule]
+            working_schedule.after = schedule_cache[update_trigger_schedule[:after]]
+            working_schedule.trigger_execution_status = schedule_cache[update_trigger_schedule[:trigger_execution_status]] if update_trigger_schedule[:state] == :changed
+
+            # Update trigger info
+            working_schedule.save
+
+            # Update transfer result
+            results.each do |transfer_result|
+              transfer_result[:schedule] = working_schedule if transfer_result[:schedule].obj_id == working_schedule.obj_id
+            end
+          end
+        end
+
         diff[:removed].each do |removed_schedule|
           GoodData.logger.info("Removing schedule #{removed_schedule[:name]}")
 
@@ -871,6 +1002,14 @@ module GoodData
       GoodData::Dashboard[id, project: self, client: client]
     end
 
+    # Helper for getting analytical dashboards (KD dashboards) of a project
+    #
+    # @param id [String | Number | Object] Anything that you can pass to GoodData::Dashboard[id]
+    # @return [GoodData::AnalyticalDashboard | Array<GoodData::AnalyticalDashboard>] dashboard instance or list
+    def analytical_dashboards(id = :all)
+      GoodData::AnalyticalDashboard[id, project: self, client: client]
+    end
+
     def data_permissions(id = :all)
       GoodData::MandatoryUserFilter[id, client: client, project: self]
     end
@@ -1797,15 +1936,15 @@ module GoodData
       end
 
       # reassign to groups
+      removal_user_group_members = []
       mappings = new_users.map(&:to_hash).flat_map do |user|
+        removal_user_group_members << user[:login] if user[:user_group]&.empty?
         groups = user[:user_group] || []
         groups.map { |g| [user[:login], g] }
       end
+
       unless mappings.empty?
-        users_lookup = users.reduce({}) do |a, e|
-          a[e.login] = e
-          a
-        end
+        users_lookup = login_users
         mappings.group_by { |_, g| g }.each do |g, mapping|
           remote_users = mapping.map { |user, _| user }.map { |login| users_lookup[login] && users_lookup[login].uri }.reject(&:nil?)
           GoodData.logger.info("Assigning users #{remote_users} to group #{g}")
@@ -1819,14 +1958,42 @@ module GoodData
         end
         mentioned_groups = mappings.map(&:last).uniq
         groups_to_cleanup = user_groups_cache.reject { |g| mentioned_groups.include?(g.name) }
+
         # clean all groups not mentioned with exception of whitelisted users
         groups_to_cleanup.each do |g|
           g.set_members(whitelist_users(g.members.map(&:to_hash), [], options[:whitelists], :include).first.map { |x| x[:uri] })
         end
       end
+
+      remove_member_from_group(users_lookup, removal_user_group_members, user_groups_cache)
       GoodData::Helpers.join(results, diff_results, [:user], [:login_uri])
     end
 
+    def remove_member_from_group(users_lookup, removal_user_group_members, user_groups_cache)
+      unless removal_user_group_members.empty?
+        users_lookup ||= login_users
+        current_user_groups = user_groups_cache || user_groups
+        removal_user_group_members.uniq.each do |login|
+          user_uri = users_lookup[login]&.uri
+
+          # remove user from group if exists as group member
+          current_user_groups.each do |user_group|
+            if user_group.member?(user_uri)
+              GoodData.logger.info("Removing #{user_uri} user from group #{user_group.name}")
+              user_group.remove_members(user_uri)
+            end
+          end
+        end
+      end
+    end
+
+    def login_users
+      users.reduce({}) do |a, e|
+        a[e.login] = e
+        a
+      end
+    end
+
     def disable_users(list, options = {})
       list = list.map(&:to_hash)
       url = "#{uri}/users"
@@ -2047,6 +2214,14 @@ module GoodData
       GoodData::Project.update_dataset_mapping(model_mapping_json, self)
     end
 
+    def ldm_layout
+      GoodData::Project.get_ldm_layout(self)
+    end
+
+    def save_ldm_layout(ldm_layout_json)
+      GoodData::Project.save_ldm_layout(ldm_layout_json, self)
+    end
+
     def transfer_processes(target)
       GoodData::Project.transfer_processes(self, target)
     end