gooddata 2.1.11 → 2.1.17

data/gdc_fossa_ruby_sdk.yaml

@@ -0,0 +1,5 @@
+---
+generate_attribution_file: true
+fossa_project: "gooddata-ruby-sdk"
+ignored_paths:
+  - 'ci/.*'

data/gooddata.gemspec

@@ -5,7 +5,7 @@ require 'gooddata/version'
 Gem::Specification.new do |s|
   s.name = 'gooddata'
   s.version = GoodData::VERSION
-  s.licenses = ['BSD']
+  s.licenses = ['BSD-3-Clause']
   s.platform = 'java' if RUBY_PLATFORM =~ /java/
 
   s.required_rubygems_version = Gem::Requirement.new('>= 0') if s.respond_to? :required_rubygems_version=
@@ -24,7 +24,7 @@ Gem::Specification.new do |s|
   s.email = 'lcm@gooddata.com'
   s.extra_rdoc_files = %w(LICENSE README.md)
 
-  s.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
+  s.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR).map { |f| f unless %w(NOTICES.txt LICENSE_FOR_RUBY_SDK_COMPONENT.txt).include?(f) }
   s.files.reject! { |fn| fn.start_with? 'spec/' }
   s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
 
@@ -55,14 +55,13 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'sqlite3' if RUBY_PLATFORM != 'java'
 
   if RUBY_VERSION >= '2.5'
-    s.add_dependency 'activesupport', '> 4.2.9', '< 6.1'
+    s.add_dependency 'activesupport', '>= 6.0.3.1', '< 6.2'
   else
-    s.add_dependency 'activesupport', '> 4.2.9', '< 6.0'
+    s.add_dependency 'activesupport', '>= 5.2.4.3', '< 6.0'
   end
 
   s.add_dependency 'aws-sdk-s3', '~> 1.16'
   s.add_dependency 'docile', '~> 1.1'
-  s.add_dependency 'erubis', '~> 2.7'
   s.add_dependency 'gli', '~> 2.15'
   s.add_dependency 'gooddata_datawarehouse', '~> 0.0.10' if RUBY_PLATFORM == 'java'
   s.add_dependency 'highline', '= 2.0.0.pre.develop.14'

data/k8s/charts/lcm-bricks/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 name: lcm-bricks
 description: LCM Bricks
-version: 2.0.1
+version: 2.0.6

data/k8s/charts/lcm-bricks/templates/prometheus/alertingRules.yaml

@@ -20,7 +20,7 @@ data:
         expr: container_pod:lcm_pod_container_status_restarts:increase10m >= 1
         labels:
           severity: warning
-          team: lcm # switch to msf in production
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "There is more than 0 restarts of {{`{{ $labels.pod }}`}} pod in the last 10 minutes"
@@ -28,8 +28,8 @@ data:
       - alert: "[LCM] Pod has too many restarts on cluster={{ .Values.clusterId }}"
         expr: container_pod:lcm_pod_container_status_restarts:increase10m >= 2
         labels:
-          severity: critical
-          team: lcm # switch to msf in production
+          severity: warning
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "There is more than 1 restart of {{`{{ $labels.pod }}`}} pod in the last 10 minutes"
@@ -40,7 +40,7 @@ data:
         expr: container_pod:lcm_pod_container_status_oomkilled:increase10m >= 1
         labels:
           severity: warning
-          team: lcm # switch to msf in production
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "{{`{{ $labels.pod }}`}} was OOMKilled in the last 30 minutes. Investigate and/or increase memoryRequest or memoryLimit."
@@ -48,8 +48,8 @@ data:
       - alert: "[LCM] OOMKill occured on cluster={{ .Values.clusterId }}"
         expr: container_pod:lcm_pod_container_status_oomkilled:increase10m >= 2
         labels:
-          severity: critical
-          team: lcm # switch to msf in production
+          severity: warning
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "{{`{{ $labels.pod }}`}} was OOMKilled in the last 10 minutes. Investigate and/or increase memoryRequest or memoryLimit."
@@ -58,8 +58,8 @@ data:
         expr: rate(container_cpu_cfs_throttled_seconds_total{namespace='{{ .Release.Namespace }}'}[1m]) > 1
         for: 5m
         labels:
-          severity: critical
-          team: lcm # switch to msf in production
+          severity: warning
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "{{`{{ $labels.pod_name }}`}} container is beeing throttled and probably hit CPU limit. Investigate root cause and increase limit and/or number of replicas if necessary."
@@ -68,8 +68,8 @@ data:
         expr: rate(jvm_gc_pause_seconds_sum{kubernetes_namespace='{{ .Release.Namespace }}'}[1m]) > 1
         for: 5m
         labels:
-          severity: critical
-          team: lcm # switch to msf in production
+          severity: warning
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "{{`{{ $labels.kubernetes_pod_name }}`}} container is spending too much time in pause garbage collector. Investigate root cause and increase heap size and/or number of replicas if necessary."
@@ -77,9 +77,29 @@ data:
       - alert: "[LCM] there is more than 100 jobs on cluster={{ .Values.clusterId }}"
         expr: count(kube_job_info{namespace="lcm"}) > 100
         labels:
-          severity: critical
-          team: lcm # switch to msf in production
+          severity: warning
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "There is more than 100 jobs in LCM namespace. They are likely not deleted."
           summary: "There is more than 100 jobs in LCM namespace."
+      - alert: "[LCM] Resource quotas hit CPU limit on cluster={{ .Values.clusterId }}"
+        expr: kube_resourcequota{namespace='{{ .Release.Namespace }}',resource="limits.cpu",type="hard"} - ignoring(type) kube_resourcequota{namespace='{{ .Release.Namespace }}',resource="limits.cpu",type="used"} == 0
+        labels:
+          severity: warning
+          team: lcm
+          cluster_id: {{ .Values.clusterId }}
+        annotations:
+          description: "We are hitting CPU limit in LCM namespace."
+          summary: "We are hitting CPU limit in LCM namespace."
+      - alert: "[LCM] POD is in undesirable state on cluster={{ .Values.clusterId }}"
+        expr: kube_pod_status_phase{namespace='{{ .Release.Namespace }}', phase!~"Running|Succeeded|Failed"} > 0
+        for: 5m
+        labels:
+          cluster_id: {{ .Values.clusterId }}
+          severity: critical
+          team: lcm
+        annotations:
+          description: "POD {{`{{ $labels.pod }}`}} is not in desirable state"
+          summary: "POD is not in desirable state"
+          runbook: "https://confluence.intgdc.com/display/plat/Generic+runbooks#Genericrunbooks-Podisinundesirablestate"

data/lib/gooddata.rb

@@ -37,3 +37,5 @@ require 'backports/2.1.0/array/to_h'
 
 # Helpers
 require 'gooddata/helpers/global_helpers'
+
+require 'active_support/core_ext/hash/compact' unless RUBY_VERSION >= '2.5'

data/lib/gooddata/cloud_resources/{cloud_resouce_factory.rb → cloud_resource_factory.rb}

@@ -11,7 +11,15 @@ module GoodData
   module CloudResources
     class CloudResourceFactory
       class << self
+        def load_cloud_resource(type)
+          base = "#{Pathname(__FILE__).dirname.expand_path}#{File::SEPARATOR}#{type}#{File::SEPARATOR}"
+          Dir.glob(base + '**/*.rb').each do |file|
+            require file
+          end
+        end
+
         def create(type, data = {}, opts = {})
+          load_cloud_resource(type)
           clients = CloudResourceClient.descendants.select { |c| c.respond_to?("accept?") && c.send("accept?", type) }
           raise "DataSource does not support type \"#{type}\"" if clients.empty?
 

data/lib/gooddata/cloud_resources/cloud_resources.rb

@@ -7,6 +7,6 @@
 require 'pathname'
 
 base = Pathname(__FILE__).dirname.expand_path
-Dir.glob(base + '**/*.rb').each do |file|
+Dir.glob(base + '*.rb').each do |file|
   require file
 end

data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb

@@ -0,0 +1,107 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2021 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'securerandom'
+require 'java'
+require 'pathname'
+require_relative '../cloud_resource_client'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + 'drivers/*.jar').each do |file|
+  require file unless file.start_with?('lcm-postgresql-driver')
+end
+
+module GoodData
+  module CloudResources
+    class PostgresClient < CloudResourceClient
+      JDBC_POSTGRES_PATTERN = %r{jdbc:postgresql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
+      POSTGRES_DEFAULT_PORT = 5432
+      JDBC_POSTGRES_PROTOCOL = 'jdbc:postgresql://'
+      SSL_JAVA_FACTORY = '&sslfactory=org.postgresql.ssl.DefaultJavaSSLFactory'
+      VERIFY_FULL = 'verify-full'
+      PREFER = 'prefer'
+      REQUIRE = 'require'
+      POSTGRES_SET_SCHEMA_COMMAND = "set search_path to"
+      POSTGRES_FETCH_SIZE = 1000
+
+      class << self
+        def accept?(type)
+          type == 'postgresql'
+        end
+      end
+
+      def initialize(options = {})
+        raise("Data Source needs a client to Postgres to be able to query the storage but 'postgresql_client' is empty.") unless options['postgresql_client']
+
+        if options['postgresql_client']['connection'].is_a?(Hash)
+          @database = options['postgresql_client']['connection']['database']
+          @schema = options['postgresql_client']['connection']['schema'] || 'public'
+          @authentication = options['postgresql_client']['connection']['authentication']
+          @ssl_mode = options['postgresql_client']['connection']['sslMode']
+          raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
+
+          @url = build_url(options['postgresql_client']['connection']['url'])
+        else
+          raise('Missing connection info for Postgres client')
+        end
+
+        Java.org.postgresql.Driver
+      end
+
+      def realize_query(query, _params)
+        GoodData.gd_logger.info("Realize SQL query: type=postgresql status=started")
+
+        connect
+        filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
+        measure = Benchmark.measure do
+          statement = @connection.create_statement
+          statement.set_fetch_size(POSTGRES_FETCH_SIZE)
+          has_result = statement.execute(query)
+          if has_result
+            result = statement.get_result_set
+            metadata = result.get_meta_data
+            col_count = metadata.column_count
+            CSV.open(filename, 'wb') do |csv|
+              csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
+              csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
+            end
+          end
+        end
+        GoodData.gd_logger.info("Realize SQL query: type=postgresql status=finished duration=#{measure.real}")
+        filename
+      ensure
+        @connection&.close
+        @connection = nil
+      end
+
+      def connect
+        GoodData.logger.info "Setting up connection to Postgresql #{@url}"
+
+        prop = java.util.Properties.new
+        prop.setProperty('user', @authentication['basic']['userName'])
+        prop.setProperty('password', @authentication['basic']['password'])
+        prop.setProperty('schema', @schema)
+
+        @connection = java.sql.DriverManager.getConnection(@url, prop)
+        statement = @connection.create_statement
+        statement.execute("#{POSTGRES_SET_SCHEMA_COMMAND} #{@schema}")
+        @connection.set_auto_commit(false)
+      end
+
+      def build_url(url)
+        matches = url.scan(JDBC_POSTGRES_PATTERN)
+        raise 'Cannot reach the url' unless matches
+
+        host = matches[0][0]
+        port = matches[0][2]&.to_i || POSTGRES_DEFAULT_PORT
+        raise "Custom port #{port} is not supported. Remove it or use the default port '5432'" if POSTGRES_DEFAULT_PORT != port
+
+        "#{JDBC_POSTGRES_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{@ssl_mode}#{VERIFY_FULL == @ssl_mode ? SSL_JAVA_FACTORY : ''}"
+      end
+    end
+  end
+end

data/lib/gooddata/commands/scaffold.rb

@@ -4,7 +4,7 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-require 'erubis'
+require 'erb'
 require 'fileutils'
 require 'pathname'
 
@@ -18,15 +18,14 @@ module GoodData
         # TODO: Add option for custom output dir
         def project(name)
           fail ArgumentError, 'No name specified' if name.nil?
-
           FileUtils.mkdir(name)
           FileUtils.cd(name) do
             FileUtils.mkdir('model')
             FileUtils.cd('model') do
               input = File.read(TEMPLATES_PATH + 'project/model/model.rb.erb')
-              eruby = Erubis::Eruby.new(input)
+              erb = ERB.new(input)
               File.open('model.rb', 'w') do |f|
-                f.write(eruby.result(:name => name))
+                f.write(erb.result_with_hash(:name => name))
               end
             end
 
@@ -36,9 +35,9 @@ module GoodData
             end
 
             input = File.read(TEMPLATES_PATH + 'project/Goodfile.erb')
-            eruby = Erubis::Eruby.new(input)
+            erb = ERB.new(input)
             File.open('Goodfile', 'w') do |f|
-              f.write(eruby.result)
+              f.write(erb.result)
             end
           end
         end
@@ -51,15 +50,15 @@ module GoodData
           FileUtils.mkdir(name)
           FileUtils.cd(name) do
             input = File.read(TEMPLATES_PATH + 'bricks/brick.rb.erb')
-            eruby = Erubis::Eruby.new(input)
+            erb = ERB.new(input)
             File.open('brick.rb', 'w') do |f|
-              f.write(eruby.result)
+              f.write(erb.result)
             end
 
             input = File.read(TEMPLATES_PATH + 'bricks/main.rb.erb')
-            eruby = Erubis::Eruby.new(input)
+            erb = ERB.new(input)
             File.open('main.rb', 'w') do |f|
-              f.write(eruby.result)
+              f.write(erb.result)
             end
           end
         end

data/lib/gooddata/core/nil_logger.rb

@@ -1,4 +1,5 @@
-# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# frozen_string_literal: true
+# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
@@ -11,6 +12,7 @@ module GoodData
       @level = nil
     end
 
+    # No body define need for dummy logger
     def debug(*_args)
     end
 

data/lib/gooddata/helpers/data_helper.rb

@@ -44,9 +44,8 @@ module GoodData
           realize_link
         when 's3'
           realize_s3(params)
-        when 'redshift', 'snowflake', 'bigquery'
+        when 'redshift', 'snowflake', 'bigquery', 'postgresql'
           raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
-
           require_relative '../cloud_resources/cloud_resources'
           realize_cloud_resource(source, params)
         else

data/lib/gooddata/helpers/global_helpers.rb

@@ -1,6 +1,6 @@
 # encoding: UTF-8
 #
-# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
@@ -34,6 +34,7 @@ module GoodData
     end
 
     set_const :GD_MAX_RETRY, (ENV['GD_MAX_RETRY'] && ENV['GD_MAX_RETRY'].to_i) || 12
+    AES_256_CBC_CIPHER = 'aes-256-cbc'
 
     class << self
       def error(msg)
@@ -222,7 +223,7 @@ module GoodData
       # encrypts data with the given key. returns a binary data with the
       # unhashed random iv in the first 16 bytes
       def encrypt(data, key)
-        cipher = OpenSSL::Cipher::Cipher.new('aes-256-cbc')
+        cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
         cipher.encrypt
         cipher.key = key = Digest::SHA256.digest(key)
         random_iv = cipher.random_iv
@@ -236,7 +237,7 @@ module GoodData
 
       # Simple encrypt data with given key
       def simple_encrypt(data, key)
-        cipher = OpenSSL::Cipher::Cipher.new('aes-256-cbc')
+        cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
         cipher.encrypt
         cipher.key = key
         encrypted = cipher.update(data)
@@ -253,7 +254,7 @@ module GoodData
 
         data = Base64.decode64(database64)
 
-        cipher = OpenSSL::Cipher::Cipher.new('aes-256-cbc')
+        cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
         cipher.decrypt
         cipher.key = cipher_key = Digest::SHA256.digest(key)
         random_iv = data[0..15] # extract iv from first 16 bytes
@@ -273,7 +274,7 @@ module GoodData
 
         data = Base64.decode64(database64)
 
-        cipher = OpenSSL::Cipher::Cipher.new('aes-256-cbc')
+        cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
         cipher.decrypt
         cipher.key = key
         decrypted = cipher.update(data)

data/lib/gooddata/helpers/global_helpers_params.rb

@@ -242,7 +242,7 @@ module GoodData
 
       def resolve_reference_params(data_params, params)
         reference_values = []
-        regexps = Regexp.union(/\\\\/, /\\\$/, /\$\{(\w+)\}/)
+        regexps = Regexp.union(/\\\\/, /\\\$/, /\$\{([^\n\{\}]+)\}/)
         resolve_reference = lambda do |v|
           if v.is_a? Hash
             Hash[
@@ -262,7 +262,7 @@ module GoodData
                 data_params.is_a?(Hash) ? '\\' : '\\\\' # rubocop: disable Metrics/BlockNesting
               elsif match =~ /\\\$/
                 '$'
-              elsif match =~ /\$\{(\w+)\}/
+              elsif match =~ /\$\{([^\n\{\}]+)\}/
                 val = params["#{$1}"]
                 if val
                   reference_values << val

data/lib/gooddata/lcm/actions/base_action.rb

@@ -8,8 +8,6 @@ require 'gooddata/extensions/integer'
 require 'gooddata/extensions/string'
 require 'gooddata/extensions/nil'
 
-require 'active_support/core_ext/hash/compact'
-
 require_relative '../dsl/dsl'
 require_relative '../helpers/helpers'
 require_relative '../types/types'