gooddata 2.1.8-java → 2.1.9-java

data/lib/gooddata/cloud_resources/cloud_resouce_factory.rb

@@ -0,0 +1,28 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'active_support/core_ext/string/inflections'
+require_relative 'cloud_resource_client'
+
+module GoodData
+  module CloudResources
+    class CloudResourceFactory
+      class << self
+        def create(type, data = {}, opts = {})
+          clients = CloudResourceClient.descendants.select { |c| c.respond_to?("accept?") && c.send("accept?", type) }
+          raise "DataSource does not support type \"#{type}\"" if clients.empty?
+
+          res = clients[0].new(data)
+          opts.each do |key, value|
+            method = "#{key}="
+            res.send(method, value) if res.respond_to?(method)
+          end
+          res
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/cloud_resources/cloud_resource_client.rb

@@ -0,0 +1,24 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+module GoodData
+  module CloudResources
+    class CloudResourceClient
+      def self.inherited(klass)
+        @descendants ||= []
+        @descendants << klass
+      end
+
+      def self.descendants
+        @descendants || []
+      end
+
+      def realize_query(_query, _params)
+        raise NotImplementedError, 'Must be implemented in subclass'
+      end
+    end
+  end
+end

data/lib/gooddata/cloud_resources/cloud_resources.rb

@@ -0,0 +1,12 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'pathname'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + '**/*.rb').each do |file|
+  require file
+end

data/lib/gooddata/cloud_resources/redshift/drivers/log4j.properties

@@ -0,0 +1,15 @@
+#
+# Copyright (C) 2007-2019, GoodData(R) Corporation. All rights reserved.
+#
+
+#=======================================================================================================================
+# Root Logger
+#=======================================================================================================================
+#log4j.rootCategory=INFO, Syslog, Console
+log4j.rootCategory=INFO
+
+#=======================================================================================================================
+# Logger with Higher Verbosity
+#=======================================================================================================================
+log4j.logger.com.amazonaws=INFO
+

data/lib/gooddata/cloud_resources/redshift/redshift_client.rb

@@ -0,0 +1,100 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'securerandom'
+require 'java'
+require 'pathname'
+require_relative '../cloud_resource_client'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + 'drivers/*.jar').each do |file|
+  require file unless file.start_with?('lcm-redshift-driver')
+end
+
+module GoodData
+  module CloudResources
+    class RedshiftClient < CloudResourceClient
+      class << self
+        def accept?(type)
+          type == 'redshift'
+        end
+      end
+
+      def initialize(options = {})
+        raise("Data Source needs a client to Redshift to be able to query the storage but 'redshift_client' is empty.") unless options['redshift_client']
+
+        if options['redshift_client']['connection'].is_a?(Hash)
+          @database = options['redshift_client']['connection']['database']
+          @schema = options['redshift_client']['connection']['schema'] || 'public'
+          @url = options['redshift_client']['connection']['url']
+          @authentication = options['redshift_client']['connection']['authentication']
+        else
+          raise('Missing connection info for Redshift client')
+
+        end
+        @debug = options['debug'] == true || options['debug'] == 'true'
+
+        Java.com.amazon.redshift.jdbc42.Driver
+        base = Pathname(__FILE__).dirname
+        org.apache.log4j.PropertyConfigurator.configure("#{base}/drivers/log4j.properties")
+      end
+
+      def realize_query(query, _params)
+        GoodData.gd_logger.info("Realize SQL query: type=redshift status=started")
+
+        connect
+        filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
+        measure = Benchmark.measure do
+          statement = @connection.create_statement
+          schema_sql = "set search_path to #{@schema}"
+          statement.execute(schema_sql)
+
+          has_result = statement.execute(query)
+          if has_result
+            result = statement.get_result_set
+            metadata = result.get_meta_data
+            col_count = metadata.column_count
+            CSV.open(filename, 'wb', :force_quotes => true) do |csv|
+              csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
+              csv << Array(1..col_count).map { |i| result.get_string(i) } while result.next
+            end
+          end
+        end
+        GoodData.gd_logger.info("Realize SQL query: type=redshift status=finished duration=#{measure.real}")
+        filename
+      ensure
+        @connection.close unless @connection.nil?
+        @connection = nil
+      end
+
+      def connect
+        full_url = build_url(@url, @database)
+        GoodData.logger.info "Setting up connection to Redshift #{full_url}"
+
+        prop = java.util.Properties.new
+        if @authentication['basic']
+          prop.setProperty('UID', @authentication['basic']['userName'])
+          prop.setProperty('PWD', @authentication['basic']['password'])
+        else
+          prop.setProperty('AccessKeyID', @authentication['iam']['accessKeyId'])
+          prop.setProperty('SecretAccessKey', @authentication['iam']['secretAccessKey'])
+          prop.setProperty('DbUser', @authentication['iam']['dbUser'])
+        end
+
+        @connection = java.sql.DriverManager.getConnection(full_url, prop)
+      end
+
+      private
+
+      def build_url(url, database)
+        url_parts = url.split('?')
+        url_path = url_parts[0].chomp('/')
+        url_path += "/#{database}" if database && !url_path.end_with?("/#{database}")
+        url_parts.length > 1 ? url_path + '?' + url_parts[1] : url_path
+      end
+    end
+  end
+end

data/lib/gooddata/exceptions/invalid_env_error.rb

@@ -0,0 +1,15 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+module GoodData
+  class InvalidEnvError < RuntimeError
+    DEFAULT_MSG = 'Invalid environment: It must be JAVA platform'
+
+    def initialize(msg = DEFAULT_MSG)
+      super(msg)
+    end
+  end
+end

data/lib/gooddata/helpers/data_helper.rb

@@ -44,6 +44,11 @@ module GoodData
           realize_link
         when 's3'
           realize_s3(params)
+        when 'redshift'
+          raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
+
+          require_relative '../cloud_resources/cloud_resources'
+          realize_cloud_resource(source, params)
         else
           raise "DataSource does not support type \"#{source}\""
         end
@@ -55,6 +60,11 @@ module GoodData
 
       private
 
+      def realize_cloud_resource(type, params)
+        cloud_resource_client = GoodData::CloudResources::CloudResourceFactory.create(type, params)
+        cloud_resource_client.realize_query(@options[:query], params)
+      end
+
       def realize_query(params)
         query = DataSource.interpolate_sql_params(@options[:query], params)
         dwh = params['ads_client'] || params[:ads_client] || raise("Data Source needs a client to ads to be able to query the storage but 'ads_client' is empty.")

data/lib/gooddata/helpers/global_helpers.rb

@@ -183,6 +183,10 @@ module GoodData
         end
       end
 
+      def deep_merge(source, target)
+        GoodData::Helpers::DeepMergeableHash[source].deep_merge(target)
+      end
+
       def undot(params)
         # for each key-value config given
         params.map do |k, v|

data/lib/gooddata/helpers/global_helpers_params.rb

@@ -3,10 +3,6 @@
 # LICENSE file in the root directory of this source tree.
 require 'active_support/core_ext/hash/slice'
 
-require 'gooddata/extensions/hash'
-
-using HashExtensions
-
 module GoodData
   module Helpers
     ENCODED_PARAMS_KEY = 'gd_encoded_params'
@@ -102,7 +98,8 @@ module GoodData
 
         params.delete(key)
         params.delete(hidden_key)
-        params = params.deep_merge(parsed_data_params).deep_merge(parsed_hidden_data_params)
+        params = GoodData::Helpers.deep_merge(params, parsed_data_params)
+        params = GoodData::Helpers.deep_merge(params, parsed_hidden_data_params)
 
         if options[:convert_pipe_delimited_params]
           convert_pipe_delimited_params = lambda do |args|
@@ -121,7 +118,7 @@ module GoodData
             end
 
             lines.reduce({}) do |a, e|
-              a.deep_merge(e)
+              GoodData::Helpers.deep_merge(a, e)
             end
           end
 
@@ -129,7 +126,7 @@ module GoodData
           params.delete_if do |k, _|
             k.include?('|')
           end
-          params = params.deep_merge(pipe_delimited_params)
+          params = GoodData::Helpers.deep_merge(params, pipe_delimited_params)
         end
 
         params

data/lib/gooddata/lcm/actions/collect_segment_clients.rb

@@ -65,8 +65,11 @@ module GoodData
                 segment.segment_id
               )
             else
-              latest_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, segment.segment_id)
+              data_product = params.data_product
+              data_product_id = data_product.data_product_id
+              latest_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product_id, segment.segment_id)
             end
+            raise 'Release table has no data' unless latest_master
 
             latest_master = client.projects(latest_master[:master_project_id])
 

data/lib/gooddata/lcm/actions/collect_segments.rb

@@ -51,8 +51,7 @@ module GoodData
             begin
               project = segment.master_project
             rescue RestClient::BadRequest => e
-              params.gdc_logger.error "Failed to retrieve master project for segment #{segment.id}. Error: #{e}"
-              raise
+              raise "Failed to retrieve master project for segment #{segment.id}. Error: #{e}"
             end
 
             raise "Master project for segment #{segment.id} doesn't exist." unless project

data/lib/gooddata/lcm/actions/create_segment_masters.rb

@@ -71,7 +71,7 @@ module GoodData
             ads_output_stage_prefix = segment_in.ads_output_stage_prefix
 
             # Create master project Postgres
-            version = get_project_version(params, domain_name, segment_id) + 1
+            version = get_project_version(params, domain_name, data_product, segment_id) + 1
 
             master_name = segment_in.master_name.gsub('#{version}', version.to_s)
 
@@ -113,6 +113,7 @@ module GoodData
               status = 'modified'
             end
 
+            segment_in[:data_product_id] = data_product.data_product_id
             segment_in[:master_pid] = project.pid
             segment_in[:version] = version
             segment_in[:timestamp] = Time.now.utc.iso8601
@@ -150,7 +151,7 @@ module GoodData
           }
         end
 
-        def get_project_version(params, domain_name, segment_id)
+        def get_project_version(params, domain_name, data_product, segment_id)
           if params.ads_client
             current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
               params.release_table_name,
@@ -158,7 +159,8 @@ module GoodData
               segment_id
             )
           else
-            current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, segment_id)
+            data_product_id = data_product.data_product_id # data_product was populated by CollectDataProduct action already
+            current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product_id, segment_id)
           end
           return 0 unless current_master
           current_master[:version].to_i

data/lib/gooddata/lcm/actions/synchronize_clients.rb

@@ -68,7 +68,7 @@ module GoodData
                 segment.segment_id
               )
             else
-              current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, segment.segment_id)
+              current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
             end
 
             # TODO: Check res.first.nil? || res.first[:master_project_id].nil?

data/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb

@@ -80,8 +80,7 @@ module GoodData
             res = GoodData::Helpers.symbolize_keys(res)
 
             if res[:syncedResult][:errors]
-              params.gdc_logger.error "Error: #{res[:syncedResult][:errors].pretty_inspect}"
-              fail "Failed to sync processes/schedules for segment #{segment_id}"
+              fail "Failed to sync processes/schedules for segment #{segment_id}. Error: #{res[:syncedResult][:errors].pretty_inspect}"
             end
 
             if res[:syncedResult][:clients]

data/lib/gooddata/lcm/actions/synchronize_ldm.rb

@@ -44,6 +44,9 @@ module GoodData
 
         description 'Specifies how to synchronize LDM and resolve possible conflicts'
         param :synchronize_ldm, instance_of(Type::SynchronizeLDM), required: false, default: 'diff_against_master_with_fallback'
+
+        description 'Enables handling of deprecated objects in the logical data model.'
+        param :include_deprecated, instance_of(Type::BooleanType), required: false, default: false
       end
 
       class << self
@@ -70,6 +73,7 @@ module GoodData
           results = []
           client = params.gdc_gd_client
           exclude_fact_rule = params.exclude_fact_rule.to_b
+          include_deprecated = params.include_deprecated.to_b
           from_pid = segment_info[:from]
           from = params.development_client.projects(from_pid) || fail("Invalid 'from' project specified - '#{from_pid}'")
 
@@ -79,9 +83,11 @@ module GoodData
           previous_master = segment_info[:previous_master]
           diff_against_master = %w(diff_against_master_with_fallback diff_against_master)
             .include?(params[:synchronize_ldm].downcase)
+          GoodData.logger.info "Synchronize LDM mode: '#{params[:synchronize_ldm].downcase}'"
           if previous_master && diff_against_master
             maql_diff_params = [:includeGrain]
             maql_diff_params << :excludeFactRule if exclude_fact_rule
+            maql_diff_params << :includeDeprecated if include_deprecated
             maql_diff = previous_master.maql_diff(blueprint: blueprint, params: maql_diff_params)
           end
 
@@ -96,7 +102,8 @@ module GoodData
                 update_preference: params[:update_preference],
                 exclude_fact_rule: exclude_fact_rule,
                 execute_ca_scripts: false,
-                maql_diff: maql_diff
+                maql_diff: maql_diff,
+                include_deprecated: include_deprecated
               )
             rescue MaqlExecutionError => e
               GoodData.logger.info("Applying MAQL to project #{to_project.title} - #{pid} failed. Reason: #{e}")
@@ -106,7 +113,8 @@ module GoodData
                 blueprint,
                 update_preference: params[:update_preference],
                 exclude_fact_rule: exclude_fact_rule,
-                execute_ca_scripts: false
+                execute_ca_scripts: false,
+                include_deprecated: include_deprecated
               )
             end
 

data/lib/gooddata/lcm/actions/synchronize_user_filters.rb

@@ -121,6 +121,7 @@ module GoodData
             users_brick_input: params.users_brick_users
           }
           all_clients = domain.clients(:all, data_product).to_a
+          GoodData.gd_logger.info("Synchronizing in mode=#{mode}, number_of_clients=#{all_clients.size}, data_rows=#{user_filters.size}")
 
           GoodData.logger.info("Synchronizing in mode \"#{mode}\"")
           case mode
@@ -131,6 +132,8 @@ module GoodData
               filter = UserBricksHelper.resolve_client_id(domain, project, params.data_product)
             end
             user_filters = user_filters.select { |f| f[:pid] == filter } if filter
+
+            GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{user_filters.size}")
             sync_user_filters(project, user_filters, run_params, symbolized_config)
           when 'sync_multiple_projects_based_on_pid', 'sync_multiple_projects_based_on_custom_id'
             users_by_project = run_params[:users_brick_input].group_by { |u| u[:pid] }
@@ -144,6 +147,8 @@ module GoodData
               elsif mode == 'sync_multiple_projects_based_on_pid'
                 current_project = client.projects(id)
               end
+
+              GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{id}, data_rows=#{new_filters.size}")
               sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
             end
           when 'sync_domain_client_workspaces'
@@ -170,6 +175,8 @@ module GoodData
               fail "Client #{client_id} does not have project." unless current_project
 
               working_client_ids << client_id
+
+              GoodData.gd_logger.info("Synchronizing in mode=#{mode}, client_id=#{client_id}, data_rows=#{new_filters.size}")
               partial_results = sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
               results.concat(partial_results[:results])
             end
@@ -182,6 +189,8 @@ module GoodData
                   current_project = c.project
                   users = users_by_project[c.client_id]
                   params.gdc_logger.info "Delete all filters in project #{current_project.pid} of client #{c.client_id}"
+
+                  GoodData.gd_logger.info("Delete all filters in project_id=#{current_project.pid}, client_id=#{c.client_id}")
                   current_results = sync_user_filters(current_project, [], run_params.merge(users_brick_input: users), symbolized_config)
 
                   results.concat(current_results[:results])
@@ -214,10 +223,21 @@ module GoodData
           multiple_projects_column = params.multiple_projects_column
           data_source = GoodData::Helpers::DataSource.new(params.input_source)
 
-          without_check(PARAMS, params) do
-            CSV.foreach(File.open(data_source.realize(params), 'r:UTF-8'), headers: csv_with_headers, return_headers: false, encoding: 'utf-8') do |row|
+          tmp = without_check(PARAMS, params) do
+            File.open(data_source.realize(params), 'r:UTF-8')
+          end
+
+          begin
+            GoodData.logger.info('Start reading data')
+            row_count = 0
+            CSV.foreach(tmp, headers: csv_with_headers, return_headers: false, encoding: 'utf-8') do |row|
               filters << row.to_hash.merge(pid: row[multiple_projects_column])
+              row_count += 1
+              GoodData.logger.info("Read #{row_count} rows") if (row_count % 50_000).zero?
             end
+            GoodData.logger.info("Done reading data, total #{row_count} rows")
+          rescue Exception => e # rubocop:disable RescueException
+            fail "There was an error during loading data. Message: #{e.message}. Error: #{e}"
           end
 
           if filters.empty? && %w(sync_multiple_projects_based_on_pid sync_multiple_projects_based_on_custom_id).include?(params.sync_mode)