gooddata 2.1.9-java → 2.1.14-java

data/gdc_fossa_lcm.yaml

@@ -0,0 +1,2 @@
+---
+fossa_project: "gooddata-ruby-lcm"

data/gdc_fossa_ruby_sdk.yaml

@@ -0,0 +1,4 @@
+---
+fossa_project: "gooddata-ruby-sdk"
+ignored_paths:
+  - 'ci/.*'

data/gooddata.gemspec

@@ -49,15 +49,15 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'pronto', '~> 0.10' if RUBY_PLATFORM != 'java'
   s.add_development_dependency 'pronto-rubocop', '~> 0.9' if RUBY_PLATFORM != 'java'
   s.add_development_dependency 'pronto-reek', '~> 0.9' if RUBY_PLATFORM != 'java'
-  s.add_development_dependency 'vcr'
+  s.add_development_dependency 'vcr', '5.0.0'
   s.add_development_dependency 'hashdiff', '~> 0.4'
 
   s.add_development_dependency 'sqlite3' if RUBY_PLATFORM != 'java'
 
   if RUBY_VERSION >= '2.5'
-    s.add_dependency 'activesupport', '> 4.2.9', '< 6.1'
+    s.add_dependency 'activesupport', '>= 6.0.3.1', '< 6.1'
   else
-    s.add_dependency 'activesupport', '> 4.2.9', '< 6.0'
+    s.add_dependency 'activesupport', '>= 5.2.4.3', '< 6.0'
   end
 
   s.add_dependency 'aws-sdk-s3', '~> 1.16'

data/k8s/charts/lcm-bricks/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 name: lcm-bricks
 description: LCM Bricks
-version: 2.0.1
+version: 2.0.3

data/k8s/charts/lcm-bricks/templates/prometheus/alertingRules.yaml

@@ -20,7 +20,7 @@ data:
         expr: container_pod:lcm_pod_container_status_restarts:increase10m >= 1
         labels:
           severity: warning
-          team: lcm # switch to msf in production
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "There is more than 0 restarts of {{`{{ $labels.pod }}`}} pod in the last 10 minutes"
@@ -28,8 +28,8 @@ data:
       - alert: "[LCM] Pod has too many restarts on cluster={{ .Values.clusterId }}"
         expr: container_pod:lcm_pod_container_status_restarts:increase10m >= 2
         labels:
-          severity: critical
-          team: lcm # switch to msf in production
+          severity: warning
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "There is more than 1 restart of {{`{{ $labels.pod }}`}} pod in the last 10 minutes"
@@ -40,7 +40,7 @@ data:
         expr: container_pod:lcm_pod_container_status_oomkilled:increase10m >= 1
         labels:
           severity: warning
-          team: lcm # switch to msf in production
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "{{`{{ $labels.pod }}`}} was OOMKilled in the last 30 minutes. Investigate and/or increase memoryRequest or memoryLimit."
@@ -48,8 +48,8 @@ data:
       - alert: "[LCM] OOMKill occured on cluster={{ .Values.clusterId }}"
         expr: container_pod:lcm_pod_container_status_oomkilled:increase10m >= 2
         labels:
-          severity: critical
-          team: lcm # switch to msf in production
+          severity: warning
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "{{`{{ $labels.pod }}`}} was OOMKilled in the last 10 minutes. Investigate and/or increase memoryRequest or memoryLimit."
@@ -58,8 +58,8 @@ data:
         expr: rate(container_cpu_cfs_throttled_seconds_total{namespace='{{ .Release.Namespace }}'}[1m]) > 1
         for: 5m
         labels:
-          severity: critical
-          team: lcm # switch to msf in production
+          severity: warning
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "{{`{{ $labels.pod_name }}`}} container is beeing throttled and probably hit CPU limit. Investigate root cause and increase limit and/or number of replicas if necessary."
@@ -68,8 +68,8 @@ data:
         expr: rate(jvm_gc_pause_seconds_sum{kubernetes_namespace='{{ .Release.Namespace }}'}[1m]) > 1
         for: 5m
         labels:
-          severity: critical
-          team: lcm # switch to msf in production
+          severity: warning
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "{{`{{ $labels.kubernetes_pod_name }}`}} container is spending too much time in pause garbage collector. Investigate root cause and increase heap size and/or number of replicas if necessary."
@@ -77,9 +77,19 @@ data:
       - alert: "[LCM] there is more than 100 jobs on cluster={{ .Values.clusterId }}"
         expr: count(kube_job_info{namespace="lcm"}) > 100
         labels:
-          severity: critical
-          team: lcm # switch to msf in production
+          severity: warning
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "There is more than 100 jobs in LCM namespace. They are likely not deleted."
           summary: "There is more than 100 jobs in LCM namespace."
+      - alert: "[LCM] Resource quotas hit CPU limit on cluster={{ .Values.clusterId }}"
+        expr: kube_resourcequota{namespace='{{ .Release.Namespace }}',resource="limits.cpu",type="hard"} - ignoring(type) kube_resourcequota{namespace='{{ .Release.Namespace }}',resource="limits.cpu",type="used"} == 0
+        labels:
+          severity: warning
+          team: lcm
+          cluster_id: {{ .Values.clusterId }}
+        annotations:
+          description: "We are hitting CPU limit in LCM namespace."
+          summary: "We are hitting CPU limit in LCM namespace."
+

data/lcm.rake

@@ -118,7 +118,7 @@ end
 namespace :docker do
   desc 'Build Docker image'
   task :build do
-    Rake::Task["maven:build_redshift"].invoke
+    Rake::Task["maven:build_dependencies"].invoke
     system('docker build -f Dockerfile.jruby -t gooddata/appstore .')
   end
 
@@ -129,11 +129,15 @@ namespace :docker do
 end
 
 namespace :maven do
-  task :build_redshift do
-    system("cp -rf spec/lcm/redshift_driver_pom.xml tmp/pom.xml")
-    system('mvn -f tmp/pom.xml clean install -P binary-packaging')
-    system('cp -rf tmp/target/*.jar lib/gooddata/cloud_resources/redshift/drivers/')
-    system('rm -rf lib/gooddata/cloud_resources/redshift/drivers/lcm-redshift-driver*.jar')
+  task :build_dependencies do
+    system('mvn -f ci/snowflake/pom.xml clean install -P binary-packaging')
+    system('cp -rf ci/snowflake/target/*.jar lib/gooddata/cloud_resources/snowflake/drivers/')
+
+    system('mvn -f ci/bigquery/pom.xml clean install -P binary-packaging')
+    system('cp -rf ci/bigquery/target/*.jar lib/gooddata/cloud_resources/bigquery/drivers/')
+
+    system('mvn -f ci/redshift/pom.xml clean install -P binary-packaging')
+    system('cp -rf ci/redshift/target/*.jar lib/gooddata/cloud_resources/redshift/drivers/')
   end
 end
 

data/lib/gooddata/cloud_resources/bigquery/bigquery_client.rb

@@ -0,0 +1,86 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'securerandom'
+require 'java'
+require 'pathname'
+require_relative '../cloud_resource_client'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + 'drivers/*.jar').each do |file|
+  require file unless file.start_with?('lcm-bigquery-driver')
+end
+
+java_import 'com.google.auth.oauth2.ServiceAccountCredentials'
+java_import 'com.google.cloud.bigquery.BigQuery'
+java_import 'com.google.cloud.bigquery.BigQueryOptions'
+java_import 'com.google.cloud.bigquery.FieldList'
+java_import 'com.google.cloud.bigquery.FieldValue'
+java_import 'com.google.cloud.bigquery.FieldValueList'
+java_import 'com.google.cloud.bigquery.QueryJobConfiguration'
+java_import 'com.google.cloud.bigquery.TableResult'
+java_import 'org.apache.commons.text.StringEscapeUtils'
+
+module GoodData
+  module CloudResources
+    class BigQueryClient < CloudResourceClient
+      class << self
+        def accept?(type)
+          type == 'bigquery'
+        end
+      end
+
+      def initialize(options = {})
+        raise("Data Source needs a client to BigQuery to be able to query the storage but 'bigquery_client' is empty.") unless options['bigquery_client']
+
+        if options['bigquery_client']['connection'].is_a?(Hash)
+          @project = options['bigquery_client']['connection']['project']
+          @schema = options['bigquery_client']['connection']['schema'] || 'public'
+          @authentication = options['bigquery_client']['connection']['authentication']
+        else
+          raise('Missing connection info for BigQuery client')
+
+        end
+      end
+
+      def realize_query(query, _params)
+        GoodData.gd_logger.info("Realize SQL query: type=bigquery status=started")
+
+        client = create_client
+        filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
+        measure = Benchmark.measure do
+          query_config = QueryJobConfiguration.newBuilder(query).setDefaultDataset(@schema).build
+          table_result = client.query(query_config)
+
+          if table_result.getTotalRows.positive?
+            result = table_result.iterateAll
+            field_list = table_result.getSchema.getFields
+            col_count = field_list.size
+            CSV.open(filename, 'wb') do |csv|
+              csv << Array(1..col_count).map { |i| field_list.get(i - 1).getName } # build the header
+              result.each do |row|
+                csv << Array(1..col_count).map { |i| row.get(i - 1).getValue&.to_s }
+              end
+            end
+          end
+        end
+        GoodData.gd_logger.info("Realize SQL query: type=bigquery status=finished duration=#{measure.real}")
+        filename
+      end
+
+      private
+
+      def create_client
+        GoodData.logger.info "Setting up connection to BigQuery"
+        client_email = @authentication['serviceAccount']['clientEmail']
+        private_key = @authentication['serviceAccount']['privateKey']
+        credentials = ServiceAccountCredentials.fromPkcs8(nil, client_email, StringEscapeUtils.unescapeJson(private_key), nil, nil)
+        BigQueryOptions.newBuilder.setProjectId(@project).setCredentials(credentials).build.getService
+      end
+    end
+  end
+end

data/lib/gooddata/cloud_resources/redshift/redshift_client.rb

@@ -1,4 +1,5 @@
 # encoding: UTF-8
+# frozen_string_literal: true
 #
 # Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
 # This source code is licensed under the BSD-style license found in the
@@ -57,9 +58,9 @@ module GoodData
             result = statement.get_result_set
             metadata = result.get_meta_data
             col_count = metadata.column_count
-            CSV.open(filename, 'wb', :force_quotes => true) do |csv|
+            CSV.open(filename, 'wb') do |csv|
               csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
-              csv << Array(1..col_count).map { |i| result.get_string(i) } while result.next
+              csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
             end
           end
         end

data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb

@@ -0,0 +1,84 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'securerandom'
+require 'java'
+require 'pathname'
+require_relative '../cloud_resource_client'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + 'drivers/*.jar').each do |file|
+  require file unless file.start_with?('lcm-snowflake-driver')
+end
+
+module GoodData
+  module CloudResources
+    class SnowflakeClient < CloudResourceClient
+      class << self
+        def accept?(type)
+          type == 'snowflake'
+        end
+      end
+
+      def initialize(options = {})
+        raise("Data Source needs a client to Snowflake to be able to query the storage but 'snowflake_client' is empty.") unless options['snowflake_client']
+
+        if options['snowflake_client']['connection'].is_a?(Hash)
+          @database = options['snowflake_client']['connection']['database']
+          @schema = options['snowflake_client']['connection']['schema'] || 'public'
+          @warehouse = options['snowflake_client']['connection']['warehouse']
+          @url = options['snowflake_client']['connection']['url']
+          @authentication = options['snowflake_client']['connection']['authentication']
+        else
+          raise('Missing connection info for Snowflake client')
+
+        end
+
+        Java.net.snowflake.client.jdbc.SnowflakeDriver
+      end
+
+      def realize_query(query, _params)
+        GoodData.gd_logger.info("Realize SQL query: type=snowflake status=started")
+
+        connect
+        filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
+        measure = Benchmark.measure do
+          statement = @connection.create_statement
+
+          has_result = statement.execute(query)
+          if has_result
+            result = statement.get_result_set
+            metadata = result.get_meta_data
+            col_count = metadata.column_count
+            CSV.open(filename, 'wb') do |csv|
+              csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
+              csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
+            end
+          end
+        end
+        GoodData.gd_logger.info("Realize SQL query: type=snowflake status=finished duration=#{measure.real}")
+        filename
+      ensure
+        @connection&.close
+        @connection = nil
+      end
+
+      def connect
+        GoodData.logger.info "Setting up connection to Snowflake #{@url}"
+
+        prop = java.util.Properties.new
+        prop.setProperty('user', @authentication['basic']['userName'])
+        prop.setProperty('password', @authentication['basic']['password'])
+        prop.setProperty('schema', @schema)
+        prop.setProperty('warehouse', @warehouse)
+        prop.setProperty('db', @database)
+
+        @connection = java.sql.DriverManager.getConnection(@url, prop)
+      end
+    end
+  end
+end

data/lib/gooddata/helpers/data_helper.rb

@@ -44,7 +44,7 @@ module GoodData
           realize_link
         when 's3'
           realize_s3(params)
-        when 'redshift'
+        when 'redshift', 'snowflake', 'bigquery'
           raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
 
           require_relative '../cloud_resources/cloud_resources'

data/lib/gooddata/helpers/data_source_helpers.rb

@@ -0,0 +1,47 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2010-2020 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+# frozen_string_literal: false
+
+module GoodData
+  module Helpers
+    class << self
+      # Get a data source information from server by id
+      #
+      # @param [String] data_source_id The data source ID
+      # @param [Object] client The Rest Client object
+      # @return [Hash] Returns Data source
+      def get_data_source_by_id(data_source_id, client)
+        unless data_source_id.blank?
+          uri = "/gdc/dataload/dataSources/#{data_source_id}"
+          client.get(uri)
+        end
+      end
+
+      # Verify to see if the data source exists in the domain using its alias
+      #
+      # @param [String] ds_alias The data source's alias
+      # @param [Object] client The Rest Client object
+      # @return [String] Id of the data source or failed with the reason
+      def verify_data_source_alias(ds_alias, client)
+        domain = client.connection.server.url
+        fail "The data source alias is empty, check your data source configuration." unless ds_alias
+
+        uri = "/gdc/dataload/dataSources/internal/availableAlias?alias=#{ds_alias[:alias]}"
+        res = client.get(uri)
+        fail "Unable to get information about the Data Source '#{ds_alias[:alias]}' in the domain '#{domain}'" unless res
+        fail "Unable to find the #{ds_alias[:type]} Data Source '#{ds_alias[:alias]}' in the domain '#{domain}'" if res['availableAlias']['available']
+
+        ds_type = res['availableAlias']['existingDataSource']['type']
+        if ds_type && ds_type != ds_alias[:type]
+          fail "Wrong Data Source type - the '#{ds_type}' type is expected but the Data Source '#{ds_alias[:alias]}' in the domain '#{domain}' has the '#{ds_alias[:type]}' type"
+        else
+          res['availableAlias']['existingDataSource']['id']
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/helpers/global_helpers_params.rb

@@ -242,7 +242,7 @@ module GoodData
 
       def resolve_reference_params(data_params, params)
         reference_values = []
-        regexps = Regexp.union(/\\\\/, /\\\$/, /\$\{(\w+)\}/)
+        regexps = Regexp.union(/\\\\/, /\\\$/, /\$\{([^\n\{\}]+)\}/)
         resolve_reference = lambda do |v|
           if v.is_a? Hash
             Hash[
@@ -262,7 +262,7 @@ module GoodData
                 data_params.is_a?(Hash) ? '\\' : '\\\\' # rubocop: disable Metrics/BlockNesting
               elsif match =~ /\\\$/
                 '$'
-              elsif match =~ /\$\{(\w+)\}/
+              elsif match =~ /\$\{([^\n\{\}]+)\}/
                 val = params["#{$1}"]
                 if val
                   reference_values << val

data/lib/gooddata/lcm/actions/collect_clients.rb

@@ -67,11 +67,11 @@ module GoodData
         end
 
         def collect_clients(params, segment_names = nil)
-          client_id_column = params.client_id_column || 'client_id'
-          segment_id_column = params.segment_id_column || 'segment_id'
-          project_id_column = params.project_id_column || 'project_id'
-          project_title_column = params.project_title_column || 'project_title'
-          project_token_column = params.project_token_column || 'project_token'
+          client_id_column = params.client_id_column&.downcase || 'client_id'
+          segment_id_column = params.segment_id_column&.downcase || 'segment_id'
+          project_id_column = params.project_id_column&.downcase || 'project_id'
+          project_title_column = params.project_title_column&.downcase || 'project_title'
+          project_token_column = params.project_token_column&.downcase || 'project_token'
           client = params.gdc_gd_client
 
           clients = []
@@ -82,7 +82,7 @@ module GoodData
           end
           GoodData.logger.debug("Input data: #{input_data.read}")
           GoodData.logger.debug("Segment names: #{segment_names}")
-          CSV.foreach(input_data, :headers => true, :return_headers => false, encoding: 'utf-8') do |row|
+          CSV.foreach(input_data, :headers => true, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
             GoodData.logger.debug("Processing row: #{row}")
             segment_name = row[segment_id_column]
             GoodData.logger.debug("Segment name: #{segment_name}")

data/lib/gooddata/lcm/actions/collect_dynamic_schedule_params.rb

@@ -38,11 +38,11 @@ module GoodData
         def call(params)
           return [] unless params.dynamic_params
 
-          schedule_title_column = params.schedule_title_column || 'schedule_title'
-          client_id_column = params.client_id_column || 'client_id'
-          param_name_column = params.param_name_column || 'param_name'
-          param_value_column = params.param_value_column || 'param_value'
-          param_secure_column = params.param_secure_column || 'param_secure'
+          schedule_title_column = params.schedule_title_column&.downcase || 'schedule_title'
+          client_id_column = params.client_id_column&.downcase || 'client_id'
+          param_name_column = params.param_name_column&.downcase || 'param_name'
+          param_value_column = params.param_value_column&.downcase || 'param_value'
+          param_secure_column = params.param_secure_column&.downcase || 'param_secure'
 
           encryption_key = params.dynamic_params_encryption_key || ''
           exist_encryption_key = encryption_key.blank? ? false : true
@@ -59,7 +59,7 @@ module GoodData
           schedule_hidden_params = {}
           exist_param_secure = false
 
-          CSV.foreach(input_data, :headers => true, :return_headers => false, encoding: 'utf-8') do |row|
+          CSV.foreach(input_data, :headers => true, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
             is_param_secure = row[param_secure_column] == 'true'
             is_decrypt_secure_value = is_param_secure && exist_encryption_key ? true : false
             exist_param_secure = true if is_param_secure