gooddata 2.1.8-java → 2.1.13-java

data/lib/gooddata/lcm/actions/synchronize_users.rb

@@ -190,6 +190,9 @@ module GoodData
             create_non_existing_user_groups: create_non_existing_user_groups,
             user_groups_cache: nil
           }
+          GoodData.gd_logger.info("Synchronizing in mode=#{mode}, data_rows=#{new_users.size}")
+
+          GoodData.logger.info("Synchronizing in mode \"#{mode}\"")
           results = case mode
                     when 'add_to_organization'
                       domain.create_users(new_users.uniq { |u| u[:login] || u[:email] })
@@ -197,6 +200,8 @@ module GoodData
                       user_ids = new_users.uniq { |u| u[:login] || u[:email] }.map { |u| u[:login] || u[:email] }
                       users = user_ids.map { |u| domain.users(u, client: client) }
                       params.gdc_logger.warn "Deleting #{users.count} users from domain #{domain_name}"
+
+                      GoodData.gd_logger.info("Synchronizing in mode=#{mode}, domain=#{domain_name}, data_rows=#{users.count}")
                       users.map(&:delete)
                     when 'sync_project'
                       project.import_users(new_users, common_params)
@@ -204,6 +209,8 @@ module GoodData
                       new_users.group_by { |u| u[:pid] }.flat_map do |project_id, users|
                         begin
                           project = client.projects(project_id)
+
+                          GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project_id}, data_rows=#{users.count}")
                           project.import_users(users, common_params)
                         rescue RestClient::ResourceNotFound
                           fail "Project \"#{project_id}\" was not found. Please check your project ids in the source file"
@@ -215,6 +222,8 @@ module GoodData
                       end
                     when 'sync_one_project_based_on_pid'
                       filtered_users = new_users.select { |u| u[:pid] == project.pid }
+
+                      GoodData.gd_logger.info("Synchronizing in mode=#{mode}, data_rows=#{filtered_users.count}")
                       project.import_users(filtered_users, common_params)
                     when 'sync_one_project_based_on_custom_id'
                       filter_value = UserBricksHelper.resolve_client_id(domain, project, data_product)
@@ -235,6 +244,7 @@ module GoodData
                       end
 
                       GoodData.logger.info("Project #{project.pid} will receive #{filtered_users.count} from #{new_users.count} users")
+                      GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, filtered_users=#{filtered_users.count}, data_rows=#{new_users.count}")
                       project.import_users(filtered_users, common_params)
                     when 'sync_multiple_projects_based_on_custom_id'
                       all_clients = domain.clients(:all, data_product).to_a
@@ -248,6 +258,8 @@ module GoodData
                         fail "Client #{client_id} does not have project." unless project
 
                         GoodData.logger.info("Project #{project.pid} of client #{client_id} will receive #{users.count} users")
+
+                        GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{users.count}")
                         project.import_users(users, common_params)
                       end
                     when 'sync_domain_client_workspaces'
@@ -280,6 +292,8 @@ module GoodData
 
                         working_client_ids << client_id.to_s
                         GoodData.logger.info("Project #{project.pid} of client #{client_id} will receive #{users.count} users")
+
+                        GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{users.count}")
                         project.import_users(users, common_params)
                       end
 
@@ -303,13 +317,18 @@ module GoodData
                             next
                           end
                           GoodData.logger.info("Synchronizing all users in project #{project.pid} of client #{c.client_id}")
+
+                          GoodData.gd_logger.info("Synchronizing all users in project_id=#{project.pid}, client_id=#{c.client_id}")
                           res += project.import_users([], common_params)
                         end
                       end
 
                       res
                     when 'sync_domain_and_project'
+                      GoodData.gd_logger.info("Create users in mode=#{mode}, data_rows=#{new_users.count}")
                       domain.create_users(new_users, ignore_failures: ignore_failures)
+
+                      GoodData.gd_logger.info("Import users in mode=#{mode}, data_rows=#{new_users.count}")
                       project.import_users(new_users, common_params)
                     end
 
@@ -328,38 +347,33 @@ module GoodData
         end
 
         def load_data(params, data_source)
-          first_name_column           = params.first_name_column || 'first_name'
-          last_name_column            = params.last_name_column || 'last_name'
-          login_column                = params.login_column || 'login'
-          password_column             = params.password_column || 'password'
-          email_column                = params.email_column || 'email'
-          role_column                 = params.role_column || 'role'
-          sso_provider_column         = params.sso_provider_column || 'sso_provider'
-          authentication_modes_column = params.authentication_modes_column || 'authentication_modes'
-          user_groups_column          = params.user_groups_column || 'user_groups'
-          language_column             = params.language_column || 'language'
-          company_column              = params.company_column || 'company'
-          position_column             = params.position_column || 'position'
-          country_column              = params.country_column || 'country'
-          phone_column                = params.phone_column || 'phone'
-          ip_whitelist_column         = params.ip_whitelist_column || 'ip_whitelist'
+          first_name_column           = params.first_name_column&.downcase || 'first_name'
+          last_name_column            = params.last_name_column&.downcase || 'last_name'
+          login_column                = params.login_column&.downcase || 'login'
+          password_column             = params.password_column&.downcase || 'password'
+          email_column                = params.email_column&.downcase || 'email'
+          role_column                 = params.role_column&.downcase || 'role'
+          sso_provider_column         = params.sso_provider_column&.downcase || 'sso_provider'
+          authentication_modes_column = params.authentication_modes_column&.downcase || 'authentication_modes'
+          user_groups_column          = params.user_groups_column&.downcase || 'user_groups'
+          language_column             = params.language_column&.downcase || 'language'
+          company_column              = params.company_column&.downcase || 'company'
+          position_column             = params.position_column&.downcase || 'position'
+          country_column              = params.country_column&.downcase || 'country'
+          phone_column                = params.phone_column&.downcase || 'phone'
+          ip_whitelist_column         = params.ip_whitelist_column&.downcase || 'ip_whitelist'
 
           sso_provider = params.sso_provider
           authentication_modes = params.authentication_modes || []
 
-          dwh = params.ads_client
-          if dwh
-            data = dwh.execute_select(params.input_source.query)
-          else
-            tmp = without_check(PARAMS, params) do
-              File.open(data_source.realize(params), 'r:UTF-8')
-            end
+          tmp = without_check(PARAMS, params) do
+            File.open(data_source.realize(params), 'r:UTF-8')
+          end
 
-            begin
-              data = read_csv_file(tmp)
-            rescue Exception => e # rubocop:disable RescueException
-              fail "There was an error during loading users from csv file. Message: #{e.message}. Error: #{e}"
-            end
+          begin
+            data = read_csv_file(tmp)
+          rescue Exception => e # rubocop:disable RescueException
+            fail "There was an error during loading users from csv file. Message: #{e.message}. Error: #{e}"
           end
 
           data.map do |row|
@@ -379,12 +393,18 @@ module GoodData
             ip_whitelist = row[ip_whitelist_column] || row[ip_whitelist_column.to_sym]
             ip_whitelist = ip_whitelist.split(',').map(&:strip) if ip_whitelist
 
+            user_login = row[login_column] || row[login_column.to_sym]
+            user_login = user_login.strip unless user_login.nil?
+
+            user_email = row[email_column] || row[login_column] || row[email_column.to_sym] || row[login_column.to_sym]
+            user_email = user_email.strip unless user_email.nil?
+
             {
               :first_name => row[first_name_column] || row[first_name_column.to_sym],
               :last_name => row[last_name_column] || row[last_name_column.to_sym],
-              :login => row[login_column] || row[login_column.to_sym],
+              :login => user_login,
               :password => row[password_column] || row[password_column.to_sym],
-              :email => row[email_column] || row[login_column] || row[email_column.to_sym] || row[login_column.to_sym],
+              :email => user_email,
               :role => row[role_column] || row[role_column.to_sym],
               :sso_provider => sso_provider || row[sso_provider_column] || row[sso_provider_column.to_sym],
               :authentication_modes => modes,
@@ -405,7 +425,7 @@ module GoodData
           res = []
           row_count = 0
 
-          CSV.foreach(path, :headers => true) do |row|
+          CSV.foreach(path, :headers => true, :header_converters => :downcase, :encoding => 'utf-8') do |row|
             if block_given?
               data = yield row
             else

data/lib/gooddata/lcm/actions/update_release_table.rb

@@ -39,6 +39,7 @@ module GoodData
       class << self
         def call(params)
           client = params.gdc_gd_client
+          GoodData.gd_logger.info("Update release table: use_nfs=#{params.ads_client.nil?}")
 
           domain_name = params.organization || params.domain
           fail "Either organisation or domain has to be specified in params" unless domain_name
@@ -48,6 +49,7 @@ module GoodData
             segment_id = segment_in.segment_id
 
             placeholders = {
+              data_product_id: segment_in[:data_product_id],
               segment_id: segment_in[:segment_id],
               master_project_id: segment_in[:master_pid],
               version: segment_in[:version],
@@ -83,7 +85,11 @@ module GoodData
 
             params.ads_client.execute(query)
           else
-            GoodData::LCM2::Helpers.update_latest_master_to_nfs(domain_id, placeholders[:segment_id], placeholders[:master_project_id], placeholders[:version])
+            data_product_id = placeholders[:data_product_id]
+            segment_id = placeholders[:segment_id]
+            master_pid = placeholders[:master_project_id]
+            version = placeholders[:version]
+            GoodData::LCM2::Helpers.update_latest_master_to_nfs(domain_id, data_product_id, segment_id, master_pid, version)
           end
         end
       end

data/lib/gooddata/lcm/exceptions/lcm_execution_error.rb

@@ -0,0 +1,16 @@
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+module GoodData
+  class LcmExecutionError < RuntimeError
+    DEFAULT_MSG = 'Error during lcm execution'
+
+    attr_reader :summary_error
+
+    def initialize(summary_error, message = DEFAULT_MSG)
+      super(message)
+      @summary_error = summary_error
+    end
+  end
+end

data/lib/gooddata/lcm/helpers/release_table_helper.rb

@@ -8,7 +8,7 @@ module GoodData
   module LCM2
     class Helpers
       DEFAULT_TABLE_NAME = 'LCM_RELEASE'
-      DEFAULT_NFS_DIRECTORY = 'release-tables'
+      DEFAULT_NFS_DIRECTORY = '/release-tables'
 
       class << self
         def latest_master_project_from_ads(release_table_name, ads_client, segment_id)
@@ -25,22 +25,30 @@ module GoodData
           sorted.last
         end
 
-        def latest_master_project_from_nfs(domain_id, segment_id)
-          data = GoodData::Helpers::Csv.read_as_hash(path_to_release_table_file(domain_id, segment_id))
-          data.sort_by { |master| master[:version] }
+        def latest_master_project_from_nfs(domain_id, data_product_id, segment_id)
+          file_path = path_to_release_table_file(domain_id, data_product_id, segment_id)
+          data = GoodData::Helpers::Csv.read_as_hash(file_path)
+          latest_master_project = data.sort_by { |master| master[:version] }
               .reverse.first
+
+          version_info = latest_master_project ? "master_pid=#{latest_master_project[:master_project_id]} version=#{latest_master_project[:version]}" : ""
+          GoodData.gd_logger.info "Getting latest master project: file=#{file_path} domain=#{domain_id} data_product=#{data_product_id} segment=#{segment_id} #{version_info}"
+          latest_master_project
         end
 
-        def update_latest_master_to_nfs(domain_id, segment_id, master_pid, version)
+        def update_latest_master_to_nfs(domain_id, data_product_id, segment_id, master_pid, version)
+          file_path = path_to_release_table_file(domain_id, data_product_id, segment_id)
+          GoodData.gd_logger.info "Updating release table: file=#{file_path} domain=#{domain_id} data_product=#{data_product_id} segment=#{segment_id} master_pid=#{master_pid} version=#{version}" # rubocop:disable Metrics/LineLength
           GoodData::Helpers::Csv.ammend_line(
-            path_to_release_table_file(domain_id, segment_id),
+            file_path,
             master_project_id: master_pid,
             version: version
           )
         end
 
-        def path_to_release_table_file(domain_id, segment_id)
-          [DEFAULT_NFS_DIRECTORY, domain_id, segment_id + '.csv'].join('/')
+        def path_to_release_table_file(domain_id, data_prod_id, segment_id)
+          nsf_directory = ENV['RELEASE_TABLE_NFS_DIRECTORY'] || DEFAULT_NFS_DIRECTORY
+          [nsf_directory, domain_id, data_prod_id + '-' + segment_id + '.csv'].join('/')
         end
       end
     end

data/lib/gooddata/lcm/lcm2.rb

@@ -18,6 +18,7 @@ require 'active_support/core_ext/hash/compact'
 require_relative 'actions/actions'
 require_relative 'dsl/dsl'
 require_relative 'helpers/helpers'
+require_relative 'exceptions/lcm_execution_error'
 
 using TrueExtensions
 using FalseExtensions
@@ -105,6 +106,14 @@ module GoodData
         UpdateReleaseTable
       ],
 
+      release_set_master_project: [
+        EnsureReleaseTable,
+        CollectDataProduct,
+        SegmentsFilter,
+        SetMasterProject,
+        UpdateReleaseTable
+      ],
+
       provision: [
         EnsureReleaseTable,
         CollectDataProduct,
@@ -129,6 +138,7 @@ module GoodData
         EnsureTechnicalUsersDomain,
         EnsureTechnicalUsersProject,
         SynchronizeLdm,
+        MigrateGdcDateDimension,
         SynchronizeClients,
         SynchronizeComputedAttributes,
         CollectDymanicScheduleParams,
@@ -270,8 +280,14 @@ module GoodData
 
         GoodData.gd_logger.brick = mode
 
+        final_mode = if params.set_master_project && mode == 'release'
+                       'release_set_master_project'
+                     else
+                       mode
+                     end
+
         # Get actions for mode specified
-        actions = get_mode_actions(mode)
+        actions = get_mode_actions(final_mode)
 
         if params.actions
           actions = params.actions.map do |action|
@@ -304,6 +320,12 @@ module GoodData
           skip_actions.include?(action.name.split('::').last)
         end
 
+        sync_mode = params.fetch(:sync_mode, nil)
+        if mode == 'users' && %w[add_to_organization remove_from_organization].include?(sync_mode)
+          actions = actions.reject do |action|
+            %w[CollectDataProduct CollectSegments].include?(action.name.split('::').last)
+          end
+        end
         check_unused_params(actions, params)
         print_action_names(mode, actions)
 
@@ -357,10 +379,11 @@ module GoodData
 
         if errors.any?
           error_message = JSON.pretty_generate(errors)
-          GoodData.logger.error(error_message)
-
-          # Fail whole execution if there is any failed action
-          fail(error_message) if strict_mode
+          if strict_mode
+            raise GoodData::LcmExecutionError.new(errors[0][:err], error_message)
+          else
+            GoodData.logger.error(error_message)
+          end
         end
 
         result

data/lib/gooddata/models/domain.rb

@@ -222,24 +222,26 @@ module GoodData
         domain = client.domain(domain)
         if id == :all
           GoodData.logger.warn("Retrieving all users from domain #{domain.name}")
-          Enumerator.new do |y|
-            page_limit = opts[:page_limit] || 1000
-            offset = opts[:offset] || 0
-            loop do
-              begin
-                tmp = client(opts).get("#{domain.uri}/users", params: { offset: offset, limit: page_limit })
-              end
-
-              tmp['accountSettings']['items'].each do |user_data|
-                user = client.create(GoodData::Profile, user_data)
-                y << user if user
-              end
-              break if tmp['accountSettings']['items'].count < page_limit
-              offset += page_limit
+          all_users = []
+          page_limit = opts[:page_limit] || 1000
+          offset = opts[:offset] || 0
+          loop do
+            begin
+              tmp = client(opts).get("#{domain.uri}/users", params: { offset: offset, limit: page_limit })
             end
+
+            tmp['accountSettings']['items'].each do |user_data|
+              user = client.create(GoodData::Profile, user_data)
+              all_users << user if user
+            end
+            break if tmp['accountSettings']['items'].count < page_limit
+
+            offset += page_limit
           end
+
+          all_users
         else
-          find_user_by_login(domain, id)
+          find_user_by_login(domain, id, opts)
         end
       end
 

data/lib/gooddata/models/execution.rb

@@ -5,7 +5,6 @@
 # LICENSE file in the root directory of this source tree.
 
 require_relative '../rest/resource'
-require_relative '../extensions/hash'
 
 module GoodData
   class Execution < Rest::Resource

data/lib/gooddata/models/execution_detail.rb

@@ -5,7 +5,6 @@
 # LICENSE file in the root directory of this source tree.
 
 require_relative '../rest/resource'
-require_relative '../extensions/hash'
 
 module GoodData
   class ExecutionDetail < Rest::Resource

data/lib/gooddata/models/from_wire.rb

@@ -105,6 +105,7 @@ module GoodData
           d[:title] = date_dim['dateDimension']['title']
           d[:urn] = date_dim['dateDimension']['urn']
           d[:identifier_prefix] = date_dim['dateDimension']['identifierPrefix']
+          d[:identifier] = date_dim['dateDimension']['identifier'] if date_dim['dateDimension']['identifier']
           d[:columns] = parse_bridges(date_dim)
         end
       end

data/lib/gooddata/models/process.rb

@@ -118,11 +118,13 @@ module GoodData
         GoodData.logger.info("Deploying #{path}") if verbose
 
         deployed_path = Process.upload_package(path, files_to_exclude, client: client, project: project)
+        data_sources = options[:data_sources] || []
         data = {
           :process => {
             :name => deploy_name,
             :path => "/uploads/#{File.basename(deployed_path)}",
-            :type => type
+            :type => type,
+            :dataSources => data_sources
           }
         }
 
@@ -171,10 +173,12 @@ module GoodData
         verbose = options[:verbose] || false
         GoodData.logger.info("Deploying #{path}") if verbose
 
+        data_sources = options[:data_sources] || []
         data = {
           process: {
             name: deploy_name,
             path: path,
+            dataSources: data_sources,
             type: 'RUBY'
           }
         }
@@ -185,7 +189,7 @@ module GoodData
       def deploy_component(data, options = { client: GoodData.client, project: GoodData.project })
         client, project = GoodData.get_client_and_project(options)
         data = { process: data } unless data[:process]
-        data[:process] = GoodData::Helpers.symbolize_keys(data[:process]).select { |k| %i[type name component].include? k }
+        data[:process] = GoodData::Helpers.symbolize_keys(data[:process]).select { |k| %i[type name component dataSources].include? k }
         data[:process][:component] = GoodData::Helpers.symbolize_keys(data[:process][:component]).select { |k| %i[name version configLocation config].include? k }
 
         save(data, options)
@@ -266,7 +270,7 @@ module GoodData
     # @option options [String] :name Readable name of the process
     # @option options [Boolean] :verbose (false) Switch on verbose mode for detailed logging
     def deploy(path, options = {})
-      Process.deploy(path, { client: client, process_id: process_id, :project => project, :name => name, :type => type }.merge(options))
+      Process.deploy(path, { client: client, process_id: process_id, :project => project, :name => name, :type => type, :data_sources => data_sources }.merge(options))
     end
 
     # Downloads the process from S3 in a zipped form.
@@ -326,6 +330,10 @@ module GoodData
       process['component']
     end
 
+    def data_sources
+      process['dataSources']
+    end
+
     # Determines whether the process is an ADDv2 component.
     # @return [Bool] True if the process is an ADDv2 component.
     def add_v2_component?