gooddata 2.1.19 → 2.2.0

data/lib/gooddata/helpers/data_helper.rb

@@ -44,10 +44,14 @@ module GoodData
           realize_link
         when 's3'
           realize_s3(params)
-        when 'redshift', 'snowflake', 'bigquery', 'postgresql'
+        when 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql'
           raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
           require_relative '../cloud_resources/cloud_resources'
           realize_cloud_resource(source, params)
+        when 'blobStorage'
+          require_relative '../cloud_resources/blobstorage/blobstorage_client'
+          blob_storage_client = GoodData::BlobStorageClient.new(params)
+          blob_storage_client.realize_blob(@options[:file], params)
         else
           raise "DataSource does not support type \"#{source}\""
         end
@@ -111,12 +115,13 @@ module GoodData
       end
 
       def realize_s3(params)
-        s3_client = params['aws_client'] && params['aws_client']['s3_client']
+        s3_client = params['s3_client'] && params['s3_client']['client']
         raise 'AWS client not present. Perhaps S3Middleware is missing in the brick definition?' if !s3_client || !s3_client.respond_to?(:bucket)
         bucket_name = @options[:bucket]
-        key = @options[:key]
+        key = @options[:key].present? ? @options[:key] : @options[:file]
         raise 'Key "bucket" is missing in S3 datasource' if bucket_name.blank?
-        raise 'Key "key" is missing in S3 datasource' if key.blank?
+        raise 'Key "key" or "file" is missing in S3 datasource' if key.blank?
+
         GoodData.logger.info("Realizing download from S3. Bucket #{bucket_name}, object with key #{key}.")
         filename = Digest::SHA256.new.hexdigest(@options.to_json)
         bucket = s3_client.bucket(bucket_name)

data/lib/gooddata/lcm/actions/collect_meta.rb

@@ -53,7 +53,9 @@ module GoodData
                 client: development_client
               )
               kpi_dashboards = MdObject.query('analyticalDashboard', MdObject, client: development_client, project: from_project)
-              objects = old_dashboards.to_a + kpi_dashboards.to_a
+              kpi_dashboard_plugin = MdObject.query('dashboardPlugin', MdObject, client: development_client, project: from_project)
+              kpi_date_filter_config = MdObject.query('dateFilterConfig', MdObject, client: development_client, project: from_project)
+              objects = old_dashboards.to_a + kpi_dashboards.to_a + kpi_dashboard_plugin.to_a + kpi_date_filter_config.to_a
             else
               objects = GoodData::Dashboard.find_by_tag(
                 production_tags,

data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb

@@ -50,6 +50,7 @@ module GoodData
             segment_info[:to].pmap do |entry|
               pid = entry[:pid]
               to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
+              GoodData.logger.info "Migrating date dimension, project: '#{to_project.title}', PID: #{pid}"
               to_blueprint = to_project.blueprint
               upgrade_datasets = get_upgrade_dates(latest_blueprint, to_blueprint)
               next if upgrade_datasets.empty?
@@ -71,9 +72,9 @@ module GoodData
           dest_dates = get_date_dimensions(dest_blueprint) if dest_blueprint
           src_dates = get_date_dimensions(src_blueprint) if src_blueprint
 
-          return false if dest_dates.empty? || src_dates.empty?
-
           upgrade_datasets = []
+          return upgrade_datasets if dest_dates.empty? || src_dates.empty?
+
           dest_dates.each do |dest|
             src_dim = get_date_dimension(src_blueprint, dest[:id])
             next unless src_dim

data/lib/gooddata/lcm/actions/synchronize_clients.rb

@@ -33,6 +33,9 @@ module GoodData
         description 'ADS Client'
         param :ads_client, instance_of(Type::AdsClientType), required: false
 
+        description 'Keep number of old master workspace excluding the latest one'
+        param :keep_only_previous_masters_count, instance_of(Type::StringType), required: false, default: '-1'
+
         description 'Additional Hidden Parameters'
         param :additional_hidden_params, instance_of(Type::HashType), required: false
       end
@@ -53,6 +56,7 @@ module GoodData
           domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
           data_product = params.data_product
           domain_segments = domain.segments(:all, data_product)
+          keep_only_previous_masters_count = Integer(params.keep_only_previous_masters_count || "-1")
 
           segments = params.segments.map do |seg|
             domain_segments.find do |s|
@@ -62,18 +66,14 @@ module GoodData
 
           results = segments.map do |segment|
             if params.ads_client
-              current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
-                params.release_table_name,
-                params.ads_client,
-                segment.segment_id
-              )
+              master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_ads(params.release_table_name, params.ads_client, segment.segment_id)
             else
-              current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
+              master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
             end
 
+            current_master = master_projects.last
             # TODO: Check res.first.nil? || res.first[:master_project_id].nil?
             master = client.projects(current_master[:master_project_id])
-
             segment.master_project = master
             segment.save
 
@@ -87,6 +87,19 @@ module GoodData
                    "Details: #{sync_result['links']['details']}")
             end
 
+            if keep_only_previous_masters_count >= 0
+              number_of_deleted_projects = master_projects.count - (keep_only_previous_masters_count + 1)
+
+              if number_of_deleted_projects.positive?
+                begin
+                  removal_master_project_ids = remove_multiple_workspace(params, segment.segment_id, master_projects, number_of_deleted_projects)
+                  remove_old_workspaces_from_release_table(params, domain_name, data_product.data_product_id, segment.segment_id, master_projects, removal_master_project_ids)
+                rescue Exception => e # rubocop:disable RescueException
+                  GoodData.logger.error "Problem occurs when removing old master workspace, reason: #{e.message}"
+                end
+              end
+            end
+
             {
               segment: segment.id,
               master_pid: master.pid,
@@ -98,6 +111,42 @@ module GoodData
           # Return results
           results
         end
+
+        def remove_multiple_workspace(params, segment_id, master_projects, number_of_deleted_projects)
+          removal_master_project_ids = []
+          need_to_delete_projects = master_projects.take(number_of_deleted_projects)
+
+          need_to_delete_projects.each do |project_wrapper|
+            master_project_id = project_wrapper[:master_project_id]
+            next if master_project_id.to_s.empty?
+
+            begin
+              project = params.gdc_gd_client.projects(master_project_id)
+              if project && !%w[deleted archived].include?(project.state.to_s)
+                GoodData.logger.info "Segment #{segment_id}: Deleting old master workspace, project: '#{project.title}', PID: (#{project.pid})."
+                project.delete
+              end
+              removal_master_project_ids << master_project_id
+              master_projects.delete_if { |p| p[:master_project_id] == master_project_id }
+            rescue Exception => ex # rubocop:disable RescueException
+              GoodData.logger.error "Unable to remove master workspace: '#{master_project_id}', Error: #{ex.message}"
+            end
+          end
+          removal_master_project_ids
+        end
+
+        # rubocop:disable Metrics/ParameterLists
+        def remove_old_workspaces_from_release_table(params, domain_id, data_product_id, segment_id, master_projects, removal_master_project_ids)
+          unless removal_master_project_ids.empty?
+            if params.ads_client
+              GoodData::LCM2::Helpers.delete_master_project_from_ads(params.release_table_name, params.ads_client, segment_id, removal_master_project_ids)
+            else
+              data = master_projects.sort_by { |master| master[:version] }
+              GoodData::LCM2::Helpers.update_master_project_to_nfs(domain_id, data_product_id, segment_id, data)
+            end
+          end
+        end
+        # rubocop:enable Metrics/ParameterLists
       end
     end
   end

data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb

@@ -0,0 +1,64 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_action'
+
+module GoodData
+  module LCM2
+    class SynchronizeDataSetMapping < BaseAction
+      DESCRIPTION = 'Synchronize Dataset Mappings'
+
+      PARAMS = define_params(self) do
+        description 'Client Used for Connecting to GD'
+        param :gdc_gd_client, instance_of(Type::GdClientType), required: true
+
+        description 'Client used to connecting to development domain'
+        param :development_client, instance_of(Type::GdClientType), required: true
+
+        description 'Synchronization Info'
+        param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
+
+        description 'Logger'
+        param :gdc_logger, instance_of(Type::GdLogger), required: true
+      end
+
+      RESULT_HEADER = %i[from to count status]
+
+      class << self
+        def call(params)
+          results = []
+
+          client = params.gdc_gd_client
+          development_client = params.development_client
+
+          params.synchronize.peach do |info|
+            from_project = info.from
+            to_projects = info.to
+
+            from = development_client.projects(from_project) || fail("Invalid 'from' project specified - '#{from_project}'")
+            dataset_mapping = from.dataset_mapping
+            if dataset_mapping&.dig('datasetMappings', 'items').nil? || dataset_mapping['datasetMappings']['items'].empty?
+              params.gdc_logger.info "Project: '#{from.title}', PID: '#{from.pid}' has no model mapping, skip synchronizing model mapping."
+            else
+              to_projects.peach do |to|
+                pid = to[:pid]
+                to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
+
+                params.gdc_logger.info "Transferring model mapping, from project: '#{from.title}', PID: '#{from.pid}', to project: '#{to_project.title}', PID: '#{to_project.pid}'"
+                res = to_project.update_dataset_mapping(dataset_mapping)
+                res[:from] = from.pid
+                results << res
+              end
+            end
+          end
+          # Return results
+          results.flatten
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/lcm/actions/synchronize_ldm.rb

@@ -83,18 +83,29 @@ module GoodData
           segment_info[:from_blueprint] = blueprint
           maql_diff = nil
           previous_master = segment_info[:previous_master]
+          synchronize_ldm_mode = params[:synchronize_ldm].downcase
           diff_against_master = %w(diff_against_master_with_fallback diff_against_master)
-            .include?(params[:synchronize_ldm].downcase)
-          GoodData.logger.info "Synchronize LDM mode: '#{params[:synchronize_ldm].downcase}'"
-          if previous_master && diff_against_master
-            maql_diff_params = [:includeGrain]
-            maql_diff_params << :excludeFactRule if exclude_fact_rule
-            maql_diff_params << :includeDeprecated if include_deprecated
-            maql_diff = previous_master.maql_diff(blueprint: blueprint, params: maql_diff_params)
+            .include?(synchronize_ldm_mode)
+          GoodData.logger.info "Synchronize LDM mode: '#{synchronize_ldm_mode}'"
+          if segment_info.key?(:previous_master) && diff_against_master
+            if previous_master
+              maql_diff_params = [:includeGrain]
+              maql_diff_params << :excludeFactRule if exclude_fact_rule
+              maql_diff_params << :includeDeprecated if include_deprecated
+              maql_diff = previous_master.maql_diff(blueprint: blueprint, params: maql_diff_params)
+            else
+              maql_diff = {
+                "projectModelDiff" =>
+                   {
+                     "updateOperations" => [],
+                     "updateScripts" => []
+                   }
+              }
+            end
             chunks = maql_diff['projectModelDiff']['updateScripts']
             if chunks.empty?
               GoodData.logger.info "Synchronize LDM to clients will not proceed in mode \
-'#{params[:synchronize_ldm].downcase}' due to no LDM changes in the new master project. \
+'#{synchronize_ldm_mode}' due to no LDM changes in the segment master project. \
 If you had changed LDM of clients manually, please use mode 'diff_against_clients' \
 to force synchronize LDM to clients"
             end

data/lib/gooddata/lcm/actions/synchronize_user_filters.rb

@@ -124,6 +124,7 @@ module GoodData
           GoodData.gd_logger.info("Synchronizing in mode=#{mode}, number_of_clients=#{all_clients.size}, data_rows=#{user_filters.size}")
 
           GoodData.logger.info("Synchronizing in mode \"#{mode}\"")
+          results = []
           case mode
           when 'sync_project', 'sync_one_project_based_on_pid', 'sync_one_project_based_on_custom_id'
             if mode == 'sync_one_project_based_on_pid'
@@ -134,7 +135,9 @@ module GoodData
             user_filters = user_filters.select { |f| f[:pid] == filter } if filter
 
             GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{user_filters.size}")
-            sync_user_filters(project, user_filters, run_params, symbolized_config)
+            current_results = sync_user_filters(project, user_filters, run_params, symbolized_config)
+
+            results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
           when 'sync_multiple_projects_based_on_pid', 'sync_multiple_projects_based_on_custom_id'
             users_by_project = run_params[:users_brick_input].group_by { |u| u[:pid] }
             user_filters.group_by { |u| u[:pid] }.flat_map.pmap do |id, new_filters|
@@ -149,7 +152,9 @@ module GoodData
               end
 
               GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{id}, data_rows=#{new_filters.size}")
-              sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
+              current_results = sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
+
+              results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
             end
           when 'sync_domain_client_workspaces'
             domain_clients = all_clients
@@ -161,7 +166,6 @@ module GoodData
             working_client_ids = []
 
             users_by_project = run_params[:users_brick_input].group_by { |u| u[:pid] }
-            results = []
             user_filters.group_by { |u| u[multiple_projects_column] }.flat_map.pmap do |client_id, new_filters|
               users = users_by_project[client_id]
               fail "Client id cannot be empty" if client_id.blank?
@@ -182,7 +186,7 @@ module GoodData
 
               GoodData.gd_logger.info("Synchronizing in mode=#{mode}, client_id=#{client_id}, data_rows=#{new_filters.size}")
               partial_results = sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
-              results.concat(partial_results[:results])
+              results.concat(partial_results[:results]) unless partial_results.nil? || partial_results[:results].empty?
             end
 
             unless run_params[:do_not_touch_filters_that_are_not_mentioned]
@@ -197,17 +201,16 @@ module GoodData
                   GoodData.gd_logger.info("Delete all filters in project_id=#{current_project.pid}, client_id=#{c.client_id}")
                   current_results = sync_user_filters(current_project, [], run_params.merge(users_brick_input: users), symbolized_config)
 
-                  results.concat(current_results[:results])
+                  results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
                 rescue StandardError => e
                   params.gdc_logger.error "Failed to clear filters of  #{c.client_id} due to: #{e.inspect}"
                 end
               end
             end
-
-            {
-              results: results
-            }
           end
+          {
+            results: results
+          }
         end
 
         def sync_user_filters(project, filters, params, filters_config)

data/lib/gooddata/lcm/actions/update_metric_formats.rb

@@ -0,0 +1,185 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_action'
+
+module GoodData
+  module LCM2
+    class UpdateMetricFormats < BaseAction
+      DESCRIPTION = 'Localize Metric Formats'
+
+      PARAMS = define_params(self) do
+        description 'Synchronization Info'
+        param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
+
+        description 'Client Used for Connecting to GD'
+        param :gdc_gd_client, instance_of(Type::GdClientType), required: true
+
+        description 'Organization Name'
+        param :organization, instance_of(Type::StringType), required: false
+
+        description 'DataProduct to manage'
+        param :data_product, instance_of(Type::GDDataProductType), required: false
+
+        description 'Logger'
+        param :gdc_logger, instance_of(Type::GdLogger), required: true
+
+        description 'ADS Client'
+        param :ads_client, instance_of(Type::AdsClientType), required: false
+
+        description 'Input Source'
+        param :input_source, instance_of(Type::HashType), required: false
+
+        description 'Localization query'
+        param :localization_query, instance_of(Type::StringType), required: false
+      end
+
+      RESULT_HEADER = %i[action ok_clients error_clients]
+
+      class << self
+        def load_metric_data(params)
+          if params&.dig(:input_source, :metric_format) && params[:input_source][:metric_format].present?
+            metric_input_source = validate_input_source(params[:input_source])
+          else
+            return nil
+          end
+
+          metric_data_source = GoodData::Helpers::DataSource.new(metric_input_source)
+          begin
+            temp_csv = without_check(PARAMS, params) do
+              File.open(metric_data_source.realize(params), 'r:UTF-8')
+            end
+          rescue StandardError => e
+            GoodData.logger.warn("Unable to get metric input source, skip updating metric formats. Error: #{e.message} - #{e}")
+            return nil
+          end
+
+          metrics_hash = GoodData::Helpers::Csv.read_as_hash temp_csv
+          return nil if metrics_hash.empty?
+
+          expected_keys = %w[tag client_id format]
+          unless expected_keys.map(&:to_sym).all? { |s| metrics_hash.first.key? s }
+            GoodData.logger.warn("The input metric data is incorrect, expecting the following fields: #{expected_keys}")
+            return nil
+          end
+          metrics_hash
+        end
+
+        def validate_input_source(input_source)
+          type = input_source[:type] if input_source&.dig(:type)
+          metric_format = input_source[:metric_format]
+          raise "Incorrect configuration: 'type' of 'input_source' is required" if type.blank?
+
+          modified_input_source = input_source
+          case type
+          when 'ads', 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql'
+            if metric_format[:query].blank?
+              GoodData.logger.warn("The metric input_source '#{type}' is missing property 'query'")
+              return nil
+            end
+
+            modified_input_source[:query] = metric_format[:query]
+            return modified_input_source
+          when 's3'
+            if metric_format[:file].blank?
+              GoodData.logger.warn("The metric input_source '#{type}' is missing property 'file'")
+              return nil
+            end
+
+            if modified_input_source.key?(:key)
+              modified_input_source[:key] = metric_format[:file]
+            else
+              modified_input_source[:file] = metric_format[:file]
+            end
+            return modified_input_source
+          when 'blobStorage'
+            if metric_format[:file].blank?
+              GoodData.logger.warn("The metric input_source '#{type}' is missing property 'file'")
+              return nil
+            end
+
+            modified_input_source[:file] = metric_format[:file]
+            return modified_input_source
+          when 'staging'
+            if metric_format[:file].blank?
+              GoodData.logger.warn("The metric input_source '#{type}' is missing property 'file'")
+              return nil
+            end
+
+            modified_input_source[:path] = metric_format[:file]
+            return modified_input_source
+          when 'web'
+            if metric_format[:url].blank?
+              GoodData.logger.warn("The metric input_source '#{type}' is missing property 'url'")
+              return nil
+            end
+
+            modified_input_source[:url] = metric_format[:url]
+            return modified_input_source
+          else
+            return nil
+          end
+        end
+
+        def get_clients_metrics(metric_data)
+          return {} if metric_data.nil?
+
+          metric_groups = {}
+          clients = metric_data.map { |row| row[:client_id] }.uniq
+          clients.each do |client|
+            next if client.blank?
+
+            formats = {}
+            metric_data.select { |row| row[:client_id] == client && row[:tag].present? && row[:format].present? }.each { |row| formats[row[:tag]] = row[:format] }
+            metric_groups[client.to_s] ||= formats
+          end
+          metric_groups
+        end
+
+        def call(params)
+          data = load_metric_data(params)
+          result = []
+          return result if data.nil?
+
+          metric_group = get_clients_metrics(data)
+          return result if metric_group.empty?
+
+          GoodData.logger.debug("Clients have metrics which will be modified: #{metric_group.keys}")
+          updated_clients = params.synchronize.map { |segment| segment.to.map { |client| client[:client_id] } }.flatten.uniq
+          GoodData.logger.debug("Updating clients: #{updated_clients}")
+          data_product = params.data_product
+          data_product_clients = data_product.clients
+          number_client_ok = 0
+          number_client_error = 0
+          metric_group.each do |client_id, formats|
+            next unless updated_clients.include?(client_id)
+
+            client = data_product_clients.find { |c| c.id == client_id }
+            begin
+              GoodData.logger.info("Start updating metric format for client: '#{client_id}'")
+              metrics = client.project.metrics.to_a
+              formats.each do |tag, format|
+                next if tag.blank? || format.blank?
+
+                metrics_to_be_updated = metrics.select { |metric| metric.tags.include?(tag) }
+                metrics_to_be_updated.each do |metric|
+                  metric.format = format
+                  metric.save
+                end
+              end
+              number_client_ok += 1
+              GoodData.logger.info("Finished updating metric format for client: '#{client_id}'")
+            rescue StandardError => e
+              number_client_error += 1
+              GoodData.logger.warn("Failed to update metric format for client: '#{client_id}'. Error: #{e.message} - #{e}")
+            end
+          end
+          [{ :action => 'Update metric format', :ok_clients => number_client_ok, :error_clients => number_client_error }]
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb

@@ -0,0 +1,5 @@
+DELETE FROM "<%= table_name || 'LCM_RELEASE' %>"
+WHERE
+  segment_id = '<%= segment_id %>'
+  AND master_project_id IN (<%= master_project_ids %>)
+;

data/lib/gooddata/lcm/helpers/release_table_helper.rb

@@ -12,6 +12,21 @@ module GoodData
 
       class << self
         def latest_master_project_from_ads(release_table_name, ads_client, segment_id)
+          sorted = get_master_project_list_from_ads(release_table_name, ads_client, segment_id)
+          sorted.last
+        end
+
+        def latest_master_project_from_nfs(domain_id, data_product_id, segment_id)
+          file_path = path_to_release_table_file(domain_id, data_product_id, segment_id)
+          sorted = get_master_project_list_from_nfs(domain_id, data_product_id, segment_id)
+          latest_master_project = sorted.last
+
+          version_info = latest_master_project ? "master_pid=#{latest_master_project[:master_project_id]} version=#{latest_master_project[:version]}" : ""
+          GoodData.gd_logger.info "Getting latest master project: file=#{file_path} domain=#{domain_id} data_product=#{data_product_id} segment=#{segment_id} #{version_info}"
+          latest_master_project
+        end
+
+        def get_master_project_list_from_ads(release_table_name, ads_client, segment_id)
           replacements = {
             table_name: release_table_name || DEFAULT_TABLE_NAME,
             segment_id: segment_id
@@ -22,18 +37,27 @@ module GoodData
 
           res = ads_client.execute_select(query)
           sorted = res.sort_by { |row| row[:version] }
-          sorted.last
+          sorted
         end
 
-        def latest_master_project_from_nfs(domain_id, data_product_id, segment_id)
+        def delete_master_project_from_ads(release_table_name, ads_client, segment_id, removal_master_project_ids)
+          replacements = {
+            table_name: release_table_name || DEFAULT_TABLE_NAME,
+            segment_id: segment_id,
+            master_project_ids: removal_master_project_ids.map { |x| "'#{x}'" } * ', '
+          }
+
+          path = File.expand_path('../data/delete_from_lcm_release.sql.erb', __dir__)
+          query = GoodData::Helpers::ErbHelper.template_file(path, replacements)
+
+          ads_client.execute(query)
+        end
+
+        def get_master_project_list_from_nfs(domain_id, data_product_id, segment_id)
           file_path = path_to_release_table_file(domain_id, data_product_id, segment_id)
           data = GoodData::Helpers::Csv.read_as_hash(file_path)
-          latest_master_project = data.sort_by { |master| master[:version] }
-              .reverse.first
-
-          version_info = latest_master_project ? "master_pid=#{latest_master_project[:master_project_id]} version=#{latest_master_project[:version]}" : ""
-          GoodData.gd_logger.info "Getting latest master project: file=#{file_path} domain=#{domain_id} data_product=#{data_product_id} segment=#{segment_id} #{version_info}"
-          latest_master_project
+          sorted = data.sort_by { |master| master[:version] }
+          sorted
         end
 
         def update_latest_master_to_nfs(domain_id, data_product_id, segment_id, master_pid, version)
@@ -46,6 +70,16 @@ module GoodData
           )
         end
 
+        def update_master_project_to_nfs(domain_id, data_product_id, segment_id, data)
+          file_path = path_to_release_table_file(domain_id, data_product_id, segment_id)
+          FileUtils.mkpath(file_path.split('/')[0...-1].join('/'))
+          CSV.open(file_path, 'w', write_headers: true, headers: data.first.keys) do |csv|
+            data.each do |r|
+              csv << r.values
+            end
+          end
+        end
+
         def path_to_release_table_file(domain_id, data_prod_id, segment_id)
           nsf_directory = ENV['RELEASE_TABLE_NFS_DIRECTORY'] || DEFAULT_NFS_DIRECTORY
           [nsf_directory, domain_id, data_prod_id + '-' + segment_id + '.csv'].join('/')

data/lib/gooddata/lcm/lcm2.rb

@@ -96,6 +96,7 @@ module GoodData
         CollectComputedAttributeMetrics,
         ImportObjectCollections,
         SynchronizeComputedAttributes,
+        SynchronizeDataSetMapping,
         SynchronizeProcesses,
         SynchronizeSchedules,
         SynchronizeColorPalette,
@@ -122,9 +123,11 @@ module GoodData
         AssociateClients,
         RenameExistingClientProjects,
         ProvisionClients,
+        UpdateMetricFormats,
         EnsureTechnicalUsersDomain,
         EnsureTechnicalUsersProject,
         CollectDymanicScheduleParams,
+        SynchronizeDataSetMapping,
         SynchronizeETLsInSegment
       ],
 
@@ -136,8 +139,10 @@ module GoodData
         EnsureTechnicalUsersDomain,
         EnsureTechnicalUsersProject,
         SynchronizeLdm,
+        SynchronizeDataSetMapping,
         MigrateGdcDateDimension,
         SynchronizeClients,
+        UpdateMetricFormats,
         SynchronizeComputedAttributes,
         CollectDymanicScheduleParams,
         SynchronizeETLsInSegment

data/lib/gooddata/mixins/md_object_query.rb

@@ -54,6 +54,7 @@ module GoodData
               y << (klass ? client.create(klass, item, project: project) : item)
             end
             break if result['objects']['paging']['count'] < page_limit
+
             offset += page_limit
           end
         end

data/lib/gooddata/models/data_source.rb

@@ -34,7 +34,7 @@ module GoodData
             c.create(DataSource, ds_data)
           end
         else
-          c.create(DataSource, c.get("#{DATA_SOURCES_URL}/#{id}"))
+          c.create(DataSource, c.get(DATA_SOURCES_URL + '/' + id))
         end
       end
 
@@ -177,6 +177,10 @@ module GoodData
       @json['dataSource']['connectionInfo'][type]
     end
 
+    def type
+      @json['dataSource']['connectionInfo'].first[0].upcase
+    end
+
     private
 
     def build_connection_info