gooddata 2.1.7-java → 2.1.12-java

data/lib/gooddata/cloud_resources/cloud_resouce_factory.rb

@@ -0,0 +1,28 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'active_support/core_ext/string/inflections'
+require_relative 'cloud_resource_client'
+
+module GoodData
+  module CloudResources
+    class CloudResourceFactory
+      class << self
+        def create(type, data = {}, opts = {})
+          clients = CloudResourceClient.descendants.select { |c| c.respond_to?("accept?") && c.send("accept?", type) }
+          raise "DataSource does not support type \"#{type}\"" if clients.empty?
+
+          res = clients[0].new(data)
+          opts.each do |key, value|
+            method = "#{key}="
+            res.send(method, value) if res.respond_to?(method)
+          end
+          res
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/cloud_resources/cloud_resource_client.rb

@@ -0,0 +1,24 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+module GoodData
+  module CloudResources
+    class CloudResourceClient
+      def self.inherited(klass)
+        @descendants ||= []
+        @descendants << klass
+      end
+
+      def self.descendants
+        @descendants || []
+      end
+
+      def realize_query(_query, _params)
+        raise NotImplementedError, 'Must be implemented in subclass'
+      end
+    end
+  end
+end

data/lib/gooddata/cloud_resources/cloud_resources.rb

@@ -0,0 +1,12 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'pathname'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + '**/*.rb').each do |file|
+  require file
+end

data/lib/gooddata/cloud_resources/redshift/drivers/log4j.properties

@@ -0,0 +1,15 @@
+#
+# Copyright (C) 2007-2019, GoodData(R) Corporation. All rights reserved.
+#
+
+#=======================================================================================================================
+# Root Logger
+#=======================================================================================================================
+#log4j.rootCategory=INFO, Syslog, Console
+log4j.rootCategory=INFO
+
+#=======================================================================================================================
+# Logger with Higher Verbosity
+#=======================================================================================================================
+log4j.logger.com.amazonaws=INFO
+

data/lib/gooddata/cloud_resources/redshift/redshift_client.rb

@@ -0,0 +1,101 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'securerandom'
+require 'java'
+require 'pathname'
+require_relative '../cloud_resource_client'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + 'drivers/*.jar').each do |file|
+  require file unless file.start_with?('lcm-redshift-driver')
+end
+
+module GoodData
+  module CloudResources
+    class RedshiftClient < CloudResourceClient
+      class << self
+        def accept?(type)
+          type == 'redshift'
+        end
+      end
+
+      def initialize(options = {})
+        raise("Data Source needs a client to Redshift to be able to query the storage but 'redshift_client' is empty.") unless options['redshift_client']
+
+        if options['redshift_client']['connection'].is_a?(Hash)
+          @database = options['redshift_client']['connection']['database']
+          @schema = options['redshift_client']['connection']['schema'] || 'public'
+          @url = options['redshift_client']['connection']['url']
+          @authentication = options['redshift_client']['connection']['authentication']
+        else
+          raise('Missing connection info for Redshift client')
+
+        end
+        @debug = options['debug'] == true || options['debug'] == 'true'
+
+        Java.com.amazon.redshift.jdbc42.Driver
+        base = Pathname(__FILE__).dirname
+        org.apache.log4j.PropertyConfigurator.configure("#{base}/drivers/log4j.properties")
+      end
+
+      def realize_query(query, _params)
+        GoodData.gd_logger.info("Realize SQL query: type=redshift status=started")
+
+        connect
+        filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
+        measure = Benchmark.measure do
+          statement = @connection.create_statement
+          schema_sql = "set search_path to #{@schema}"
+          statement.execute(schema_sql)
+
+          has_result = statement.execute(query)
+          if has_result
+            result = statement.get_result_set
+            metadata = result.get_meta_data
+            col_count = metadata.column_count
+            CSV.open(filename, 'wb') do |csv|
+              csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
+              csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
+            end
+          end
+        end
+        GoodData.gd_logger.info("Realize SQL query: type=redshift status=finished duration=#{measure.real}")
+        filename
+      ensure
+        @connection.close unless @connection.nil?
+        @connection = nil
+      end
+
+      def connect
+        full_url = build_url(@url, @database)
+        GoodData.logger.info "Setting up connection to Redshift #{full_url}"
+
+        prop = java.util.Properties.new
+        if @authentication['basic']
+          prop.setProperty('UID', @authentication['basic']['userName'])
+          prop.setProperty('PWD', @authentication['basic']['password'])
+        else
+          prop.setProperty('AccessKeyID', @authentication['iam']['accessKeyId'])
+          prop.setProperty('SecretAccessKey', @authentication['iam']['secretAccessKey'])
+          prop.setProperty('DbUser', @authentication['iam']['dbUser'])
+        end
+
+        @connection = java.sql.DriverManager.getConnection(full_url, prop)
+      end
+
+      private
+
+      def build_url(url, database)
+        url_parts = url.split('?')
+        url_path = url_parts[0].chomp('/')
+        url_path += "/#{database}" if database && !url_path.end_with?("/#{database}")
+        url_parts.length > 1 ? url_path + '?' + url_parts[1] : url_path
+      end
+    end
+  end
+end

data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb

@@ -0,0 +1,84 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'securerandom'
+require 'java'
+require 'pathname'
+require_relative '../cloud_resource_client'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + 'drivers/*.jar').each do |file|
+  require file unless file.start_with?('lcm-snowflake-driver')
+end
+
+module GoodData
+  module CloudResources
+    class SnowflakeClient < CloudResourceClient
+      class << self
+        def accept?(type)
+          type == 'snowflake'
+        end
+      end
+
+      def initialize(options = {})
+        raise("Data Source needs a client to Snowflake to be able to query the storage but 'snowflake_client' is empty.") unless options['snowflake_client']
+
+        if options['snowflake_client']['connection'].is_a?(Hash)
+          @database = options['snowflake_client']['connection']['database']
+          @schema = options['snowflake_client']['connection']['schema'] || 'public'
+          @warehouse = options['snowflake_client']['connection']['warehouse']
+          @url = options['snowflake_client']['connection']['url']
+          @authentication = options['snowflake_client']['connection']['authentication']
+        else
+          raise('Missing connection info for Snowflake client')
+
+        end
+
+        Java.net.snowflake.client.jdbc.SnowflakeDriver
+      end
+
+      def realize_query(query, _params)
+        GoodData.gd_logger.info("Realize SQL query: type=snowflake status=started")
+
+        connect
+        filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
+        measure = Benchmark.measure do
+          statement = @connection.create_statement
+
+          has_result = statement.execute(query)
+          if has_result
+            result = statement.get_result_set
+            metadata = result.get_meta_data
+            col_count = metadata.column_count
+            CSV.open(filename, 'wb') do |csv|
+              csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
+              csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
+            end
+          end
+        end
+        GoodData.gd_logger.info("Realize SQL query: type=snowflake status=finished duration=#{measure.real}")
+        filename
+      ensure
+        @connection&.close
+        @connection = nil
+      end
+
+      def connect
+        GoodData.logger.info "Setting up connection to Snowflake #{@url}"
+
+        prop = java.util.Properties.new
+        prop.setProperty('user', @authentication['basic']['userName'])
+        prop.setProperty('password', @authentication['basic']['password'])
+        prop.setProperty('schema', @schema)
+        prop.setProperty('warehouse', @warehouse)
+        prop.setProperty('db', @database)
+
+        @connection = java.sql.DriverManager.getConnection(@url, prop)
+      end
+    end
+  end
+end

data/lib/gooddata/exceptions/invalid_env_error.rb

@@ -0,0 +1,15 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+module GoodData
+  class InvalidEnvError < RuntimeError
+    DEFAULT_MSG = 'Invalid environment: It must be JAVA platform'
+
+    def initialize(msg = DEFAULT_MSG)
+      super(msg)
+    end
+  end
+end

data/lib/gooddata/helpers/data_helper.rb

@@ -44,6 +44,11 @@ module GoodData
           realize_link
         when 's3'
           realize_s3(params)
+        when 'redshift', 'snowflake', 'bigquery'
+          raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
+
+          require_relative '../cloud_resources/cloud_resources'
+          realize_cloud_resource(source, params)
         else
           raise "DataSource does not support type \"#{source}\""
         end
@@ -55,6 +60,11 @@ module GoodData
 
       private
 
+      def realize_cloud_resource(type, params)
+        cloud_resource_client = GoodData::CloudResources::CloudResourceFactory.create(type, params)
+        cloud_resource_client.realize_query(@options[:query], params)
+      end
+
       def realize_query(params)
         query = DataSource.interpolate_sql_params(@options[:query], params)
         dwh = params['ads_client'] || params[:ads_client] || raise("Data Source needs a client to ads to be able to query the storage but 'ads_client' is empty.")

data/lib/gooddata/helpers/global_helpers.rb

@@ -183,6 +183,10 @@ module GoodData
         end
       end
 
+      def deep_merge(source, target)
+        GoodData::Helpers::DeepMergeableHash[source].deep_merge(target)
+      end
+
       def undot(params)
         # for each key-value config given
         params.map do |k, v|

data/lib/gooddata/helpers/global_helpers_params.rb

@@ -3,10 +3,6 @@
 # LICENSE file in the root directory of this source tree.
 require 'active_support/core_ext/hash/slice'
 
-require 'gooddata/extensions/hash'
-
-using HashExtensions
-
 module GoodData
   module Helpers
     ENCODED_PARAMS_KEY = 'gd_encoded_params'
@@ -102,7 +98,8 @@ module GoodData
 
         params.delete(key)
         params.delete(hidden_key)
-        params = params.deep_merge(parsed_data_params).deep_merge(parsed_hidden_data_params)
+        params = GoodData::Helpers.deep_merge(params, parsed_data_params)
+        params = GoodData::Helpers.deep_merge(params, parsed_hidden_data_params)
 
         if options[:convert_pipe_delimited_params]
           convert_pipe_delimited_params = lambda do |args|
@@ -121,7 +118,7 @@ module GoodData
             end
 
             lines.reduce({}) do |a, e|
-              a.deep_merge(e)
+              GoodData::Helpers.deep_merge(a, e)
             end
           end
 
@@ -129,7 +126,7 @@ module GoodData
           params.delete_if do |k, _|
             k.include?('|')
           end
-          params = params.deep_merge(pipe_delimited_params)
+          params = GoodData::Helpers.deep_merge(params, pipe_delimited_params)
         end
 
         params

data/lib/gooddata/lcm/actions/collect_clients.rb

@@ -67,11 +67,11 @@ module GoodData
         end
 
         def collect_clients(params, segment_names = nil)
-          client_id_column = params.client_id_column || 'client_id'
-          segment_id_column = params.segment_id_column || 'segment_id'
-          project_id_column = params.project_id_column || 'project_id'
-          project_title_column = params.project_title_column || 'project_title'
-          project_token_column = params.project_token_column || 'project_token'
+          client_id_column = params.client_id_column&.downcase || 'client_id'
+          segment_id_column = params.segment_id_column&.downcase || 'segment_id'
+          project_id_column = params.project_id_column&.downcase || 'project_id'
+          project_title_column = params.project_title_column&.downcase || 'project_title'
+          project_token_column = params.project_token_column&.downcase || 'project_token'
           client = params.gdc_gd_client
 
           clients = []
@@ -82,7 +82,7 @@ module GoodData
           end
           GoodData.logger.debug("Input data: #{input_data.read}")
           GoodData.logger.debug("Segment names: #{segment_names}")
-          CSV.foreach(input_data, :headers => true, :return_headers => false, encoding: 'utf-8') do |row|
+          CSV.foreach(input_data, :headers => true, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
             GoodData.logger.debug("Processing row: #{row}")
             segment_name = row[segment_id_column]
             GoodData.logger.debug("Segment name: #{segment_name}")

data/lib/gooddata/lcm/actions/collect_dynamic_schedule_params.rb

@@ -38,11 +38,11 @@ module GoodData
         def call(params)
           return [] unless params.dynamic_params
 
-          schedule_title_column = params.schedule_title_column || 'schedule_title'
-          client_id_column = params.client_id_column || 'client_id'
-          param_name_column = params.param_name_column || 'param_name'
-          param_value_column = params.param_value_column || 'param_value'
-          param_secure_column = params.param_secure_column || 'param_secure'
+          schedule_title_column = params.schedule_title_column&.downcase || 'schedule_title'
+          client_id_column = params.client_id_column&.downcase || 'client_id'
+          param_name_column = params.param_name_column&.downcase || 'param_name'
+          param_value_column = params.param_value_column&.downcase || 'param_value'
+          param_secure_column = params.param_secure_column&.downcase || 'param_secure'
 
           encryption_key = params.dynamic_params_encryption_key || ''
           exist_encryption_key = encryption_key.blank? ? false : true
@@ -59,7 +59,7 @@ module GoodData
           schedule_hidden_params = {}
           exist_param_secure = false
 
-          CSV.foreach(input_data, :headers => true, :return_headers => false, encoding: 'utf-8') do |row|
+          CSV.foreach(input_data, :headers => true, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
             is_param_secure = row[param_secure_column] == 'true'
             is_decrypt_secure_value = is_param_secure && exist_encryption_key ? true : false
             exist_param_secure = true if is_param_secure

data/lib/gooddata/lcm/actions/collect_segment_clients.rb

@@ -65,8 +65,11 @@ module GoodData
                 segment.segment_id
               )
             else
-              latest_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, segment.segment_id)
+              data_product = params.data_product
+              data_product_id = data_product.data_product_id
+              latest_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product_id, segment.segment_id)
             end
+            raise 'Release table has no data' unless latest_master
 
             latest_master = client.projects(latest_master[:master_project_id])
 

data/lib/gooddata/lcm/actions/collect_segments.rb

@@ -51,8 +51,7 @@ module GoodData
             begin
               project = segment.master_project
             rescue RestClient::BadRequest => e
-              params.gdc_logger.error "Failed to retrieve master project for segment #{segment.id}. Error: #{e}"
-              raise
+              raise "Failed to retrieve master project for segment #{segment.id}. Error: #{e}"
             end
 
             raise "Master project for segment #{segment.id} doesn't exist." unless project

data/lib/gooddata/lcm/actions/collect_users_brick_users.rb

@@ -35,7 +35,7 @@ module GoodData
       class << self
         def call(params)
           users_brick_users = []
-          login_column = params.users_brick_config.login_column || 'login'
+          login_column = params.users_brick_config.login_column&.downcase || 'login'
           users_brick_data_source = GoodData::Helpers::DataSource.new(params.users_brick_config.input_source)
 
           users_brick_data_source_file = without_check(PARAMS, params) do
@@ -45,14 +45,15 @@ module GoodData
             )
           end
           CSV.foreach(users_brick_data_source_file,
-                      headers: true,
-                      return_headers: false,
-                      encoding: 'utf-8') do |row|
-            pid = row[params.multiple_projects_column]
+                      :headers => true,
+                      :return_headers => false,
+                      :header_converters => :downcase,
+                      :encoding => 'utf-8') do |row|
+            pid = row[params.multiple_projects_column&.downcase]
             fail "The set multiple_projects_column '#{params.multiple_projects_column}' of the users input is empty" if !pid && MULTIPLE_COLUMN_MODES.include?(params.sync_mode)
 
             users_brick_users << {
-              login: row[login_column].downcase,
+              login: row[login_column].nil? ? nil : row[login_column].strip.downcase,
               pid: pid
             }
           end