gooddata 2.1.19-java → 2.3.0-java

data/lib/gooddata/cloud_resources/mssql/mssql_client.rb

@@ -0,0 +1,122 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2021 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'securerandom'
+require 'java'
+require 'pathname'
+require_relative '../cloud_resource_client'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + 'drivers/*.jar').each do |file|
+  require file unless file.start_with?('lcm-mssql-driver')
+end
+
+module GoodData
+  module CloudResources
+    class MSSQLClient < CloudResourceClient
+      MSSQL_SEPARATOR_PARAM = ";"
+      MSSQL_URL_PATTERN = %r{jdbc:sqlserver://([^:]+)(:([0-9]+))?(/)?}
+      MSSQL_DEFAULT_PORT = 1433
+      LOGIN_TIME_OUT = 30
+      MSSQL_FETCH_SIZE = 10_000
+      VERIFY_FULL = 'verify-full'
+      PREFER = 'prefer'
+      REQUIRE = 'require'
+
+      class << self
+        def accept?(type)
+          type == 'mssql'
+        end
+      end
+
+      def initialize(options = {})
+        raise("Data Source needs a client to MSSQL to be able to query the storage but 'mssql_client' is empty.") unless options['mssql_client']
+
+        connection = options['mssql_client']['connection']
+        if connection.is_a?(Hash)
+          @database = connection['database']
+          @schema = connection['schema']
+          @authentication = connection['authentication']
+          @ssl_mode = connection['sslMode']
+          @url = connection['url']
+
+          validate
+        else
+          raise('Missing connection info for MSSQL client')
+        end
+
+        Java.com.microsoft.sqlserver.jdbc.SQLServerDriver
+      end
+
+      def realize_query(query, _params)
+        GoodData.gd_logger.info("Realize SQL query: type=mssql status=started")
+
+        connect
+
+        filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
+        measure = Benchmark.measure do
+          statement = @connection.create_statement
+          statement.set_fetch_size(MSSQL_FETCH_SIZE)
+          has_result = statement.execute(query)
+          if has_result
+            result = statement.get_result_set
+            metadata = result.get_meta_data
+            col_count = metadata.column_count
+            CSV.open(filename, 'wb') do |csv|
+              csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
+              csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
+            end
+          end
+        end
+
+        GoodData.gd_logger.info("Realize SQL query: type=mssql status=finished duration=#{measure.real}")
+        filename
+      ensure
+        @connection&.close
+        @connection = nil
+      end
+
+      def connect
+        connection_string = build_connection_string
+        GoodData.logger.info "Setting up connection to MSSQL #{connection_string}"
+
+        authentication = @authentication['basic'] || @authentication['activeDirectoryPassword']
+
+        prop = java.util.Properties.new
+        prop.setProperty('userName', authentication['userName'])
+        prop.setProperty('password', authentication['password'])
+
+        @connection = java.sql.DriverManager.getConnection(connection_string, prop)
+      end
+
+      def validate
+        raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
+
+        raise "Instance name is not supported" if @url !~ /^[^\\]*$/
+
+        raise "The connection url is invalid. Parameter is not supported." if @url.include? MSSQL_SEPARATOR_PARAM
+
+        url_matches = @url.scan(MSSQL_URL_PATTERN)
+        raise "Cannot reach the url" if url_matches.nil? || url_matches.length.zero?
+
+        raise "The authentication method is not supported." unless @authentication['basic'] || @authentication['activeDirectoryPassword']
+      end
+
+      def build_connection_string
+        encrypt = @ssl_mode != PREFER
+        trust_server_certificate = @ssl_mode == REQUIRE
+
+        "#{@url};" \
+          "database=#{@database};" \
+          "encrypt=#{encrypt};" \
+          "trustServerCertificate=#{trust_server_certificate};" \
+          "loginTimeout=#{LOGIN_TIME_OUT};" \
+          "#{'authentication=ActiveDirectoryPassword;' if @authentication['activeDirectoryPassword']}"
+      end
+    end
+  end
+end

data/lib/gooddata/cloud_resources/mysql/mysql_client.rb

@@ -0,0 +1,121 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2021 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'securerandom'
+require 'java'
+require 'pathname'
+require_relative '../cloud_resource_client'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + 'drivers/*.jar').each do |file|
+  require file unless file.start_with?('lcm-mysql-driver')
+end
+
+module GoodData
+  module CloudResources
+    class MysqlClient < CloudResourceClient
+      JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
+      MYSQL_DEFAULT_PORT = 3306
+      JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://'
+      VERIFY_FULL = '&useSSL=true&verifyServerCertificate=true'
+      PREFER = '&useSSL=true&requireSSL=false&verifyServerCertificate=false'
+      REQUIRE = '&useSSL=true&requireSSL=true&verifyServerCertificate=false'
+      MYSQL_FETCH_SIZE = 1000
+      MONGO_BI_FETCH_SIZE = 1
+      MONGO_BI_TYPE = 'MongoDBConnector'
+
+      class << self
+        def accept?(type)
+          type == 'mysql'
+        end
+      end
+
+      def initialize(options = {})
+        raise("Data Source needs a client to Mysql to be able to query the storage but 'mysql_client' is empty.") unless options['mysql_client']
+
+        if options['mysql_client']['connection'].is_a?(Hash)
+          @database = options['mysql_client']['connection']['database']
+          @authentication = options['mysql_client']['connection']['authentication']
+          @ssl_mode = options['mysql_client']['connection']['sslMode']
+          @database_type = options['mysql_client']['connection']['databaseType']
+          raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
+
+          @url = build_url(options['mysql_client']['connection']['url'])
+        else
+          raise('Missing connection info for Mysql client')
+        end
+
+        Java.com.mysql.jdbc.Driver
+      end
+
+      def realize_query(query, _params)
+        GoodData.gd_logger.info("Realize SQL query: type=mysql status=started")
+
+        connect
+        filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
+        measure = Benchmark.measure do
+          statement = @connection.create_statement
+          statement.set_fetch_size(fetch_size)
+          has_result = statement.execute(query)
+          if has_result
+            result = statement.get_result_set
+            metadata = result.get_meta_data
+            col_count = metadata.column_count
+            CSV.open(filename, 'wb') do |csv|
+              csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
+              csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
+            end
+          end
+        end
+        GoodData.gd_logger.info("Realize SQL query: type=mysql status=finished duration=#{measure.real}")
+        filename
+      ensure
+        @connection&.close
+        @connection = nil
+      end
+
+      def connect
+        GoodData.logger.info "Setting up connection to Mysql #{@database_type} #{@url} "
+
+        prop = java.util.Properties.new
+        prop.setProperty('user', @authentication['basic']['userName'])
+        prop.setProperty('password', @authentication['basic']['password'])
+        @connection = java.sql.DriverManager.getConnection(@url, prop)
+        @connection.set_auto_commit(false)
+      end
+
+      def build_url(url)
+        matches = url.scan(JDBC_MYSQL_PATTERN)
+        raise 'Cannot reach the url' unless matches
+
+        host = matches[0][0]
+        port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT
+
+        "#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?#{get_ssl_mode(@ssl_mode)}#{add_extended}&useCursorFetch=true&enabledTLSProtocols=TLSv1.2"
+      end
+
+      def fetch_size
+        @database_type == MONGO_BI_TYPE ? MONGO_BI_FETCH_SIZE : MYSQL_FETCH_SIZE
+      end
+
+      def get_ssl_mode(ssl_mode)
+        mode = PREFER
+        if ssl_mode == 'verify-full'
+          mode =  VERIFY_FULL
+        elsif ssl_mode == 'require'
+          mode =  REQUIRE
+        end
+
+        mode
+      end
+
+      def add_extended
+        @database_type == MONGO_BI_TYPE ? '&authenticationPlugins=org.mongodb.mongosql.auth.plugin.MongoSqlAuthenticationPlugin&useLocalTransactionState=true' : ''
+      end
+    end
+  end
+end

data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb

@@ -98,7 +98,6 @@ module GoodData
 
         host = matches[0][0]
         port = matches[0][2]&.to_i || POSTGRES_DEFAULT_PORT
-        raise "Custom port #{port} is not supported. Remove it or use the default port '5432'" if POSTGRES_DEFAULT_PORT != port
 
         "#{JDBC_POSTGRES_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{@ssl_mode}#{VERIFY_FULL == @ssl_mode ? SSL_JAVA_FACTORY : ''}"
       end

data/lib/gooddata/cloud_resources/redshift/redshift_client.rb

@@ -39,8 +39,6 @@ module GoodData
         @debug = options['debug'] == true || options['debug'] == 'true'
 
         Java.com.amazon.redshift.jdbc42.Driver
-        base = Pathname(__FILE__).dirname
-        org.apache.log4j.PropertyConfigurator.configure("#{base}/drivers/log4j.properties")
       end
 
       def realize_query(query, _params)

data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb

@@ -18,6 +18,9 @@ end
 module GoodData
   module CloudResources
     class SnowflakeClient < CloudResourceClient
+      SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=gooddata_platform'
+      SNOWFLAKE_SEPARATOR_PARAM = '?'
+
       class << self
         def accept?(type)
           type == 'snowflake'
@@ -31,7 +34,7 @@ module GoodData
           @database = options['snowflake_client']['connection']['database']
           @schema = options['snowflake_client']['connection']['schema'] || 'public'
           @warehouse = options['snowflake_client']['connection']['warehouse']
-          @url = options['snowflake_client']['connection']['url']
+          @url = build_url(options['snowflake_client']['connection']['url'])
           @authentication = options['snowflake_client']['connection']['authentication']
         else
           raise('Missing connection info for Snowflake client')
@@ -79,6 +82,20 @@ module GoodData
 
         @connection = java.sql.DriverManager.getConnection(@url, prop)
       end
+
+      def build_url(url)
+        is_contain = url.include?(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
+        unless is_contain
+          if url.include?(SNOWFLAKE_SEPARATOR_PARAM)
+            url.concat("&")
+          else
+            url.concat(SNOWFLAKE_SEPARATOR_PARAM)
+          end
+          url.concat(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
+        end
+
+        url
+      end
     end
   end
 end

data/lib/gooddata/helpers/data_helper.rb

@@ -44,10 +44,14 @@ module GoodData
           realize_link
         when 's3'
           realize_s3(params)
-        when 'redshift', 'snowflake', 'bigquery', 'postgresql'
+        when 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql'
           raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
           require_relative '../cloud_resources/cloud_resources'
           realize_cloud_resource(source, params)
+        when 'blobStorage'
+          require_relative '../cloud_resources/blobstorage/blobstorage_client'
+          blob_storage_client = GoodData::BlobStorageClient.new(params)
+          blob_storage_client.realize_blob(@options[:file], params)
         else
           raise "DataSource does not support type \"#{source}\""
         end
@@ -111,12 +115,13 @@ module GoodData
       end
 
       def realize_s3(params)
-        s3_client = params['aws_client'] && params['aws_client']['s3_client']
+        s3_client = params['s3_client'] && params['s3_client']['client']
         raise 'AWS client not present. Perhaps S3Middleware is missing in the brick definition?' if !s3_client || !s3_client.respond_to?(:bucket)
         bucket_name = @options[:bucket]
-        key = @options[:key]
+        key = @options[:key].present? ? @options[:key] : @options[:file]
         raise 'Key "bucket" is missing in S3 datasource' if bucket_name.blank?
-        raise 'Key "key" is missing in S3 datasource' if key.blank?
+        raise 'Key "key" or "file" is missing in S3 datasource' if key.blank?
+
         GoodData.logger.info("Realizing download from S3. Bucket #{bucket_name}, object with key #{key}.")
         filename = Digest::SHA256.new.hexdigest(@options.to_json)
         bucket = s3_client.bucket(bucket_name)

data/lib/gooddata/lcm/actions/base_action.rb

@@ -18,6 +18,11 @@ module GoodData
       class << self
         include Dsl::Dsl
 
+        SYNC_FAILED_LIST = 'sync_failed_list'.to_sym
+        FAILED_PROJECTS = 'failed_projects'.to_sym
+        FAILED_CLIENTS = 'failed_clients'.to_sym
+        FAILED_SEGMENTS = 'failed_segments'.to_sym
+
         def check_params(specification, params)
           Helpers.check_params(specification, params)
         end
@@ -31,6 +36,158 @@ module GoodData
           params.setup_filters(specification) # enables params validation
           result
         end
+
+        def print_result(_params)
+          true
+        end
+
+        def continue_on_error(params)
+          Helpers.continue_on_error(params)
+        end
+
+        def collect_synced_status(params)
+          Helpers.collect_synced_status(params)
+        end
+
+        def add_failed_project(project_id, message, failed_action, params)
+          if collect_synced_status(params) && !sync_failed_project(project_id, params)
+            sync_failed_list = sync_failed_list(params)
+            project_client_mappings = sync_failed_list[:project_client_mappings]
+            project_client_mapping = project_client_mappings ? project_client_mappings[project_id.to_sym] : nil
+            client_id = project_client_mapping ? project_client_mapping[:client_id] : nil
+            segment_id = project_client_mapping ? project_client_mapping[:segment_id] : nil
+
+            failed_detailed_project = {
+              project_id: project_id,
+              client_id: client_id,
+              segment: segment_id,
+              message: message,
+              action: failed_action
+            }
+            add_failed_detail(params, failed_detailed_project, sync_failed_list)
+          end
+        end
+
+        def add_failed_client(client_id, message, error_action, params)
+          if collect_synced_status(params) && !sync_failed_client(client_id, params)
+            sync_failed_list = sync_failed_list(params)
+            client_project_mappings = sync_failed_list[:client_project_mappings]
+            client_project_mapping = client_project_mappings ? client_project_mappings[client_id.to_sym] : nil
+            project_id = client_project_mapping ? client_project_mapping[:project_id] : nil
+            segment_id = client_project_mapping ? client_project_mapping[:segment_id] : nil
+
+            failed_detailed_client = {
+              project_id: project_id,
+              client_id: client_id,
+              segment: segment_id,
+              message: message,
+              action: error_action
+            }
+            add_failed_detail(params, failed_detailed_client, sync_failed_list)
+          end
+        end
+
+        def add_failed_segment(segment_id, message, error_action, params)
+          if collect_synced_status(params) && !sync_failed_segment(segment_id, params)
+            sync_failed_list = sync_failed_list(params)
+            failed_detailed_segment = {
+              project_id: nil,
+              client_id: nil,
+              segment: segment_id,
+              message: message,
+              action: error_action
+            }
+            add_failed_detail(params, failed_detailed_segment, sync_failed_list, true)
+          end
+        end
+
+        # Add new clients to project client mapping
+        #
+        # @param [String] project_id project identify will be added to mapping
+        # @param [String] client_id client identify will be added to mapping
+        # @param [String] segment_id segment identify will be added to mapping
+        # @param [Hash] params the hash contains list of parameters and values
+        def add_new_clients_to_project_client_mapping(project_id, client_id, segment_id, params)
+          if collect_synced_status(params)
+            sync_failed_list = sync_failed_list(params)
+            client_project_mappings = sync_failed_list[:client_project_mappings]
+            project_client_mappings = sync_failed_list[:project_client_mappings]
+            client_project_mappings[client_id.to_sym] = {
+              project_id: project_id,
+              segment_id: segment_id
+            } if client_project_mappings
+
+            project_client_mappings[project_id.to_sym] = {
+              client_id: client_id,
+              segment_id: segment_id
+            } if project_client_mappings
+          end
+        end
+
+        def process_failed_project(project_id, failed_message, failed_projects, continue_on_error)
+          fail(failed_message) unless continue_on_error
+
+          failed_projects << {
+            project_id: project_id,
+            message: failed_message
+          }
+        end
+
+        def process_failed_projects(failed_projects, failed_action, params)
+          failed_projects.each do |failed_project|
+            add_failed_project(failed_project[:project_id], failed_project[:message], failed_action, params)
+          end
+        end
+
+        def sync_failed_project(project_id, params)
+          collect_synced_status(params) && params[SYNC_FAILED_LIST][FAILED_PROJECTS].include?(project_id)
+        end
+
+        def sync_failed_client(client_id, params)
+          collect_synced_status(params) && params[SYNC_FAILED_LIST][FAILED_CLIENTS].include?(client_id)
+        end
+
+        def sync_failed_segment(segment_id, params)
+          collect_synced_status(params) && params[SYNC_FAILED_LIST][FAILED_SEGMENTS].include?(segment_id)
+        end
+
+        private
+
+        def add_failed_detail(params, failed_detailed_project, sync_failed_list, ignore_segment = false)
+          params.gdc_logger&.warn failed_detailed_project[:message]
+          sync_failed_list[:failed_detailed_projects] << failed_detailed_project
+
+          if ignore_segment
+            add_failed_detail_segment(failed_detailed_project[:segment_id], sync_failed_list)
+          else
+            add_failed_detail_client(failed_detailed_project[:client_id], failed_detailed_project[:project_id], sync_failed_list)
+          end
+        end
+
+        def sync_failed_list(params)
+          if params.include?(SYNC_FAILED_LIST)
+            params[SYNC_FAILED_LIST]
+          else
+            nil
+          end
+        end
+
+        def add_failed_detail_client(client_id, project_id, sync_failed_list)
+          sync_failed_list[FAILED_CLIENTS] << client_id if client_id
+
+          sync_failed_list[FAILED_PROJECTS] << project_id if project_id
+        end
+
+        def add_failed_detail_segment(segment_id, sync_failed_list)
+          if segment_id
+            sync_failed_list[FAILED_SEGMENTS] << segment_id
+
+            client_project_mappings = sync_failed_list[:client_project_mappings]
+            client_project_mappings.each do |client_id, client_project_mapping|
+              add_failed_detail_client(client_id, client_project_mapping[:project_id], sync_failed_list) if client_project_mapping[:segment_id] == segment_id
+            end
+          end
+        end
       end
     end
   end

data/lib/gooddata/lcm/actions/collect_data_product.rb

@@ -47,8 +47,9 @@ module GoodData
           end
 
           begin
+            params.gdc_logger.info "Starting to find DataProduct ID: #{data_product_id}"
             data_product = domain.data_products(data_product_id)
-          rescue RestClient::BadRequest
+          rescue RestClient::BadRequest, RestClient::NotFound
             params.gdc_logger.info "Can not find DataProduct #{params.data_product}, creating it instead"
             data_product = domain.create_data_product(id: params.data_product)
           end

data/lib/gooddata/lcm/actions/collect_meta.rb

@@ -53,7 +53,9 @@ module GoodData
                 client: development_client
               )
               kpi_dashboards = MdObject.query('analyticalDashboard', MdObject, client: development_client, project: from_project)
-              objects = old_dashboards.to_a + kpi_dashboards.to_a
+              kpi_dashboard_plugin = MdObject.query('dashboardPlugin', MdObject, client: development_client, project: from_project)
+              kpi_date_filter_config = MdObject.query('dateFilterConfig', MdObject, client: development_client, project: from_project)
+              objects = old_dashboards.to_a + kpi_dashboards.to_a + kpi_dashboard_plugin.to_a + kpi_date_filter_config.to_a
             else
               objects = GoodData::Dashboard.find_by_tag(
                 production_tags,

data/lib/gooddata/lcm/actions/collect_projects_warning_status.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+# (C) 2019-2022 GoodData Corporation
+require_relative 'base_action'
+
+module GoodData
+  module LCM2
+    class CollectProjectsWarningStatus < BaseAction
+      DESCRIPTION = 'Collect synced projects status'
+
+      PARAMS = define_params(self) do
+        description 'Abort on error'
+        param :abort_on_error, instance_of(Type::StringType), required: false
+
+        description 'Logger'
+        param :gdc_logger, instance_of(Type::GdLogger), required: false
+
+        description 'Collect synced status'
+        param :collect_synced_status, instance_of(Type::BooleanType), required: false
+
+        description 'Sync failed list'
+        param :sync_failed_list, instance_of(Type::HashType), required: false
+      end
+
+      RESULT_HEADER = %i[segment client project_pid failed_action]
+
+      class << self
+        def call(params)
+          results = []
+          return results unless collect_synced_status(params)
+
+          sync_failed_list = params[SYNC_FAILED_LIST]
+          if sync_failed_list
+            failed_detailed_projects = sync_failed_list[:failed_detailed_projects]
+            failed_detailed_projects.each do |failed_detailed_project|
+              results << {
+                segment: failed_detailed_project[:segment],
+                client: failed_detailed_project[:client_id],
+                project_pid: failed_detailed_project[:project_id],
+                failed_action: failed_detailed_project[:action]
+              }
+            end
+          end
+
+          results
+        end
+
+        def print_result(params)
+          collect_synced_status(params)
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/lcm/actions/collect_segment_clients.rb

@@ -32,9 +32,22 @@ module GoodData
 
         description 'Domain'
         param :domain, instance_of(Type::StringType), required: false
+
+        description 'Abort on error'
+        param :abort_on_error, instance_of(Type::StringType), required: false
+
+        description 'Logger'
+        param :gdc_logger, instance_of(Type::GdLogger), required: false
+
+        description 'Collect synced status'
+        param :collect_synced_status, instance_of(Type::BooleanType), required: false
+
+        description 'Sync failed list'
+        param :sync_failed_list, instance_of(Type::HashType), required: false
       end
 
       RESULT_HEADER = [
+        :segment,
         :from_name,
         :from_pid,
         :to_name,
@@ -89,6 +102,7 @@ module GoodData
                 client_project = segment_client.project
                 to_pid = client_project.pid
                 results << {
+                  segment: segment.segment_id,
                   from_name: master_name,
                   from_pid: master_pid,
                   to_name: client_project.title,