gooddata 2.1.8 → 2.1.13

data/lib/gooddata/cloud_resources/cloud_resouce_factory.rb

@@ -0,0 +1,28 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'active_support/core_ext/string/inflections'
+require_relative 'cloud_resource_client'
+
+module GoodData
+  module CloudResources
+    class CloudResourceFactory
+      class << self
+        def create(type, data = {}, opts = {})
+          clients = CloudResourceClient.descendants.select { |c| c.respond_to?("accept?") && c.send("accept?", type) }
+          raise "DataSource does not support type \"#{type}\"" if clients.empty?
+
+          res = clients[0].new(data)
+          opts.each do |key, value|
+            method = "#{key}="
+            res.send(method, value) if res.respond_to?(method)
+          end
+          res
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/cloud_resources/cloud_resource_client.rb

@@ -0,0 +1,24 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+module GoodData
+  module CloudResources
+    class CloudResourceClient
+      def self.inherited(klass)
+        @descendants ||= []
+        @descendants << klass
+      end
+
+      def self.descendants
+        @descendants || []
+      end
+
+      def realize_query(_query, _params)
+        raise NotImplementedError, 'Must be implemented in subclass'
+      end
+    end
+  end
+end

data/lib/gooddata/cloud_resources/cloud_resources.rb

@@ -0,0 +1,12 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'pathname'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + '**/*.rb').each do |file|
+  require file
+end

data/lib/gooddata/cloud_resources/redshift/drivers/log4j.properties

@@ -0,0 +1,15 @@
+#
+# Copyright (C) 2007-2019, GoodData(R) Corporation. All rights reserved.
+#
+
+#=======================================================================================================================
+# Root Logger
+#=======================================================================================================================
+#log4j.rootCategory=INFO, Syslog, Console
+log4j.rootCategory=INFO
+
+#=======================================================================================================================
+# Logger with Higher Verbosity
+#=======================================================================================================================
+log4j.logger.com.amazonaws=INFO
+

data/lib/gooddata/cloud_resources/redshift/redshift_client.rb

@@ -0,0 +1,101 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'securerandom'
+require 'java'
+require 'pathname'
+require_relative '../cloud_resource_client'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + 'drivers/*.jar').each do |file|
+  require file unless file.start_with?('lcm-redshift-driver')
+end
+
+module GoodData
+  module CloudResources
+    class RedshiftClient < CloudResourceClient
+      class << self
+        def accept?(type)
+          type == 'redshift'
+        end
+      end
+
+      def initialize(options = {})
+        raise("Data Source needs a client to Redshift to be able to query the storage but 'redshift_client' is empty.") unless options['redshift_client']
+
+        if options['redshift_client']['connection'].is_a?(Hash)
+          @database = options['redshift_client']['connection']['database']
+          @schema = options['redshift_client']['connection']['schema'] || 'public'
+          @url = options['redshift_client']['connection']['url']
+          @authentication = options['redshift_client']['connection']['authentication']
+        else
+          raise('Missing connection info for Redshift client')
+
+        end
+        @debug = options['debug'] == true || options['debug'] == 'true'
+
+        Java.com.amazon.redshift.jdbc42.Driver
+        base = Pathname(__FILE__).dirname
+        org.apache.log4j.PropertyConfigurator.configure("#{base}/drivers/log4j.properties")
+      end
+
+      def realize_query(query, _params)
+        GoodData.gd_logger.info("Realize SQL query: type=redshift status=started")
+
+        connect
+        filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
+        measure = Benchmark.measure do
+          statement = @connection.create_statement
+          schema_sql = "set search_path to #{@schema}"
+          statement.execute(schema_sql)
+
+          has_result = statement.execute(query)
+          if has_result
+            result = statement.get_result_set
+            metadata = result.get_meta_data
+            col_count = metadata.column_count
+            CSV.open(filename, 'wb') do |csv|
+              csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
+              csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
+            end
+          end
+        end
+        GoodData.gd_logger.info("Realize SQL query: type=redshift status=finished duration=#{measure.real}")
+        filename
+      ensure
+        @connection.close unless @connection.nil?
+        @connection = nil
+      end
+
+      def connect
+        full_url = build_url(@url, @database)
+        GoodData.logger.info "Setting up connection to Redshift #{full_url}"
+
+        prop = java.util.Properties.new
+        if @authentication['basic']
+          prop.setProperty('UID', @authentication['basic']['userName'])
+          prop.setProperty('PWD', @authentication['basic']['password'])
+        else
+          prop.setProperty('AccessKeyID', @authentication['iam']['accessKeyId'])
+          prop.setProperty('SecretAccessKey', @authentication['iam']['secretAccessKey'])
+          prop.setProperty('DbUser', @authentication['iam']['dbUser'])
+        end
+
+        @connection = java.sql.DriverManager.getConnection(full_url, prop)
+      end
+
+      private
+
+      def build_url(url, database)
+        url_parts = url.split('?')
+        url_path = url_parts[0].chomp('/')
+        url_path += "/#{database}" if database && !url_path.end_with?("/#{database}")
+        url_parts.length > 1 ? url_path + '?' + url_parts[1] : url_path
+      end
+    end
+  end
+end

data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb

@@ -0,0 +1,84 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'securerandom'
+require 'java'
+require 'pathname'
+require_relative '../cloud_resource_client'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + 'drivers/*.jar').each do |file|
+  require file unless file.start_with?('lcm-snowflake-driver')
+end
+
+module GoodData
+  module CloudResources
+    class SnowflakeClient < CloudResourceClient
+      class << self
+        def accept?(type)
+          type == 'snowflake'
+        end
+      end
+
+      def initialize(options = {})
+        raise("Data Source needs a client to Snowflake to be able to query the storage but 'snowflake_client' is empty.") unless options['snowflake_client']
+
+        if options['snowflake_client']['connection'].is_a?(Hash)
+          @database = options['snowflake_client']['connection']['database']
+          @schema = options['snowflake_client']['connection']['schema'] || 'public'
+          @warehouse = options['snowflake_client']['connection']['warehouse']
+          @url = options['snowflake_client']['connection']['url']
+          @authentication = options['snowflake_client']['connection']['authentication']
+        else
+          raise('Missing connection info for Snowflake client')
+
+        end
+
+        Java.net.snowflake.client.jdbc.SnowflakeDriver
+      end
+
+      def realize_query(query, _params)
+        GoodData.gd_logger.info("Realize SQL query: type=snowflake status=started")
+
+        connect
+        filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
+        measure = Benchmark.measure do
+          statement = @connection.create_statement
+
+          has_result = statement.execute(query)
+          if has_result
+            result = statement.get_result_set
+            metadata = result.get_meta_data
+            col_count = metadata.column_count
+            CSV.open(filename, 'wb') do |csv|
+              csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
+              csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
+            end
+          end
+        end
+        GoodData.gd_logger.info("Realize SQL query: type=snowflake status=finished duration=#{measure.real}")
+        filename
+      ensure
+        @connection&.close
+        @connection = nil
+      end
+
+      def connect
+        GoodData.logger.info "Setting up connection to Snowflake #{@url}"
+
+        prop = java.util.Properties.new
+        prop.setProperty('user', @authentication['basic']['userName'])
+        prop.setProperty('password', @authentication['basic']['password'])
+        prop.setProperty('schema', @schema)
+        prop.setProperty('warehouse', @warehouse)
+        prop.setProperty('db', @database)
+
+        @connection = java.sql.DriverManager.getConnection(@url, prop)
+      end
+    end
+  end
+end

data/lib/gooddata/exceptions/invalid_env_error.rb

@@ -0,0 +1,15 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+module GoodData
+  class InvalidEnvError < RuntimeError
+    DEFAULT_MSG = 'Invalid environment: It must be JAVA platform'
+
+    def initialize(msg = DEFAULT_MSG)
+      super(msg)
+    end
+  end
+end

data/lib/gooddata/helpers/data_helper.rb

@@ -44,6 +44,11 @@ module GoodData
           realize_link
         when 's3'
           realize_s3(params)
+        when 'redshift', 'snowflake', 'bigquery'
+          raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
+
+          require_relative '../cloud_resources/cloud_resources'
+          realize_cloud_resource(source, params)
         else
           raise "DataSource does not support type \"#{source}\""
         end
@@ -55,6 +60,11 @@ module GoodData
 
       private
 
+      def realize_cloud_resource(type, params)
+        cloud_resource_client = GoodData::CloudResources::CloudResourceFactory.create(type, params)
+        cloud_resource_client.realize_query(@options[:query], params)
+      end
+
       def realize_query(params)
         query = DataSource.interpolate_sql_params(@options[:query], params)
         dwh = params['ads_client'] || params[:ads_client] || raise("Data Source needs a client to ads to be able to query the storage but 'ads_client' is empty.")

data/lib/gooddata/helpers/data_source_helpers.rb

@@ -0,0 +1,47 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2010-2020 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+# frozen_string_literal: false
+
+module GoodData
+  module Helpers
+    class << self
+      # Get a data source information from server by id
+      #
+      # @param [String] data_source_id The data source ID
+      # @param [Object] client The Rest Client object
+      # @return [Hash] Returns Data source
+      def get_data_source_by_id(data_source_id, client)
+        unless data_source_id.blank?
+          uri = "/gdc/dataload/dataSources/#{data_source_id}"
+          client.get(uri)
+        end
+      end
+
+      # Verify to see if the data source exists in the domain using its alias
+      #
+      # @param [String] ds_alias The data source's alias
+      # @param [Object] client The Rest Client object
+      # @return [String] Id of the data source or failed with the reason
+      def verify_data_source_alias(ds_alias, client)
+        domain = client.connection.server.url
+        fail "The data source alias is empty, check your data source configuration." unless ds_alias
+
+        uri = "/gdc/dataload/dataSources/internal/availableAlias?alias=#{ds_alias[:alias]}"
+        res = client.get(uri)
+        fail "Unable to get information about the Data Source '#{ds_alias[:alias]}' in the domain '#{domain}'" unless res
+        fail "Unable to find the #{ds_alias[:type]} Data Source '#{ds_alias[:alias]}' in the domain '#{domain}'" if res['availableAlias']['available']
+
+        ds_type = res['availableAlias']['existingDataSource']['type']
+        if ds_type && ds_type != ds_alias[:type]
+          fail "Wrong Data Source type - the '#{ds_type}' type is expected but the Data Source '#{ds_alias[:alias]}' in the domain '#{domain}' has the '#{ds_alias[:type]}' type"
+        else
+          res['availableAlias']['existingDataSource']['id']
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/helpers/global_helpers.rb

@@ -183,6 +183,10 @@ module GoodData
         end
       end
 
+      def deep_merge(source, target)
+        GoodData::Helpers::DeepMergeableHash[source].deep_merge(target)
+      end
+
       def undot(params)
         # for each key-value config given
         params.map do |k, v|

data/lib/gooddata/helpers/global_helpers_params.rb

@@ -3,10 +3,6 @@
 # LICENSE file in the root directory of this source tree.
 require 'active_support/core_ext/hash/slice'
 
-require 'gooddata/extensions/hash'
-
-using HashExtensions
-
 module GoodData
   module Helpers
     ENCODED_PARAMS_KEY = 'gd_encoded_params'
@@ -102,7 +98,8 @@ module GoodData
 
         params.delete(key)
         params.delete(hidden_key)
-        params = params.deep_merge(parsed_data_params).deep_merge(parsed_hidden_data_params)
+        params = GoodData::Helpers.deep_merge(params, parsed_data_params)
+        params = GoodData::Helpers.deep_merge(params, parsed_hidden_data_params)
 
         if options[:convert_pipe_delimited_params]
           convert_pipe_delimited_params = lambda do |args|
@@ -121,7 +118,7 @@ module GoodData
             end
 
             lines.reduce({}) do |a, e|
-              a.deep_merge(e)
+              GoodData::Helpers.deep_merge(a, e)
             end
           end
 
@@ -129,7 +126,7 @@ module GoodData
           params.delete_if do |k, _|
             k.include?('|')
           end
-          params = params.deep_merge(pipe_delimited_params)
+          params = GoodData::Helpers.deep_merge(params, pipe_delimited_params)
         end
 
         params
@@ -245,7 +242,7 @@ module GoodData
 
       def resolve_reference_params(data_params, params)
         reference_values = []
-        regexps = Regexp.union(/\\\\/, /\\\$/, /\$\{(\w+)\}/)
+        regexps = Regexp.union(/\\\\/, /\\\$/, /\$\{([\w\s\.]+)\}/)
         resolve_reference = lambda do |v|
           if v.is_a? Hash
             Hash[
@@ -265,7 +262,7 @@ module GoodData
                 data_params.is_a?(Hash) ? '\\' : '\\\\' # rubocop: disable Metrics/BlockNesting
               elsif match =~ /\\\$/
                 '$'
-              elsif match =~ /\$\{(\w+)\}/
+              elsif match =~ /\$\{([\w\s\.]+)\}/
                 val = params["#{$1}"]
                 if val
                   reference_values << val

data/lib/gooddata/lcm/actions/collect_clients.rb

@@ -67,11 +67,11 @@ module GoodData
         end
 
         def collect_clients(params, segment_names = nil)
-          client_id_column = params.client_id_column || 'client_id'
-          segment_id_column = params.segment_id_column || 'segment_id'
-          project_id_column = params.project_id_column || 'project_id'
-          project_title_column = params.project_title_column || 'project_title'
-          project_token_column = params.project_token_column || 'project_token'
+          client_id_column = params.client_id_column&.downcase || 'client_id'
+          segment_id_column = params.segment_id_column&.downcase || 'segment_id'
+          project_id_column = params.project_id_column&.downcase || 'project_id'
+          project_title_column = params.project_title_column&.downcase || 'project_title'
+          project_token_column = params.project_token_column&.downcase || 'project_token'
           client = params.gdc_gd_client
 
           clients = []
@@ -82,7 +82,7 @@ module GoodData
           end
           GoodData.logger.debug("Input data: #{input_data.read}")
           GoodData.logger.debug("Segment names: #{segment_names}")
-          CSV.foreach(input_data, :headers => true, :return_headers => false, encoding: 'utf-8') do |row|
+          CSV.foreach(input_data, :headers => true, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
             GoodData.logger.debug("Processing row: #{row}")
             segment_name = row[segment_id_column]
             GoodData.logger.debug("Segment name: #{segment_name}")