gooddata 2.1.8 → 2.1.13

data/ci/snowflake/pom.xml

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.gooddata.lcm</groupId>
+    <artifactId>lcm-snowflake-driver</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <dependencies>
+        <dependency>
+            <groupId>net.snowflake</groupId>
+            <artifactId>snowflake-jdbc</artifactId>
+            <version>3.6.22</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.2</version>
+        </dependency>
+    </dependencies>
+
+    <profiles>
+        <profile>
+            <id>binary-packaging</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-dependency-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>copy-dependencies</goal>
+                                </goals>
+                                <configuration>
+                                    <outputDirectory>${project.build.directory}</outputDirectory>
+                                    <!-- compile scope gives runtime and compile dependencies (skips test deps) -->
+                                    <includeScope>runtime</includeScope>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+
+    <repositories>
+        <repository>
+            <id>my-repo1</id>
+            <name>my custom repo</name>
+            <url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
+        </repository>
+    </repositories>
+</project>

data/dev-gooddata-sso.pub.encrypted

@@ -1,40 +1,40 @@
-BDaDY16tjuekrE/7oY03jMefPWuS8GDZjFUvFi0NTtShnvYfk2hQ1Dj2jJxX
-grIIadq2m3aGfbpYx8gtgmiCt4I1H4uHhYihz7/hlFcl9FkqdWWS5NIhEula
-5oANQcPQgUoPppdOq/g136Tg+yxhCqjcpRUpooNaFMlt4UHnK0J3VCifYo4o
-q0UGZ5VHtiJSAE27eSVi6CI04CtOR/2XBwZFytEyKBabQD89YEvtZgwXvRsO
-+A+HZc17HfYgKFhVT333/Xk7wck2cCTpdp0rTrS6Vvh/1a76v6WOqY4UH4T6
-NRjyeFjWWXlSAdEts9oqbce0hWohMzFxhruxnYgKYMGUwAsYEONeaJ+b2XP0
-gD0LMk+w3nuLldwIEy5x/ciIwhSr6xU1VQ5OR/3YLzDNZKjxEm4paOuQ9ocn
-IqyBZEEpASgupQJz77OOyF1lderX6/vC7MwNkjUuMmtRr0q3WwtxildcHKr0
-Q4dwCHkq+cLJ7RjtVQlVZUgnFvuZikGSRwnQB0RBSCRMA1ekmuQ3cukk8V7p
-LTmWCDW/cJzJcp678r7jOsE+zEIcnZ+LX3OA3eWyvtnZZMby6f/PGjqWbr0E
-xlIWj3JXgWVTIXLyaRxYn7JBAy3JqkNKhR/HiggSahlM1+EJff1svJqaIH/N
-nfqJP+fo4YW8xcVRd3KNI711T3EZQg8kl0jEZkJn4JPM2Vs7pRukCATuXF3J
-C3/4gtSQ9xYHLyPc2l1Ny0AKrfTztWQUqJn4mjxQxIu+Rw9xLw6Tv0887mL1
-dHWQ/bVw6vQ0fkARRP1xKAmpJwSZCQEisFDWVRy3ua87EDkvHOT+PHOnpGOB
-8bVflDd8fXJIpfNhQCXQEospQsEOmbSxijy9NxDk9P7QuhvMuXJkLnx4dRXI
-RyvhrNUB76FdHlhUQf2vuPyPChMwTabO6sqL6EZKpL4omqrXSogEjBDri1RS
-ua+ixPtXY4rC07v+UURt8jCRdcZBqD/H+cTAvcMdDm/339CxMViXZWb2Om6j
-14hWxQRPPlmZXqMJWGHCwr9tsylCZvgeaaCVcRRCRKxccARSBHNuh62eq6ZC
-9X5fSk8Yjbpw1bMKJXTB8moXAmuHgPO0KwHW6V2acJGWSr5yRiXVZ9f38y+g
-Fwvg/S4Fs8ESOh08Sh16ExIbHl98+DP6qyqRHpXKkMlGe2W3n+AxgPu1NBeb
-3TJVw6lF6/YEU1lbgYfqY4WFlT7orN0Jn0ZlDQpQPOF8bFaMrTs6ioP2X5g9
-k1T0pN2Zc4/oQl292DoqA8WapoqUJxv+5u1jLcQSsj592qKH0y/bJ0vasj5B
-TbPgUWxNnNdZLkAeLY5SKfE8/mI7AHPc2tCJzMPERzW/43Xyi4T22h1SVHtD
-ouK+H6BQHsOIvryDew7wRD6e17E7OYRPVTAXFOPFkOhsnKxLzG6gN+IJxvqM
-RsqLNc/po0IbySnlY1B5zsKn1QRs6A3Onu6xj4gOSUVDGs1a8L1xNWkqGiOh
-OQ9b6AaKJZG2qqImBHHOYEeZ1QB9clLJ3EawU6PiKxwwZOVZlU5qI1SasZIB
-WtRyeiSQQD1dKUVEkISI7+oVfS4OI07tlMRpzuJtrvYCV0ASImYyqD83lZFm
-3ZhSeUaXNhsYp9ac7Q+YRkLEKKbbWqtHG7LSkXVNpTbDVgzNIJiaj17bo6r0
-ZZLcW1Kqna9DxY/PLE5JbQS/CmRaPRt2VLKyKyqoSYQFdbdRzJyAIe0PT+6A
-2qEFRLkovlx7kdkqQCbEvnHN/C+cXdyKf/8EtYliGkLRoGfLc6xenQba14U9
-3aslQvItKtsRk0owEvHntag8uWZNmFVyEFDaVVG7peXyn5uigal+hXi5B8DJ
-AGOrZhycpHOJ+2HL9RlTrYps0KPKVSNgiauVpW3ZghugtfCF7sxX31+HxQAC
-4p/4kdy2R7R0cjqG8OjZ/xb3isSxhGSSjShh3RgCEJyRg55Zs+gR/XvEKk97
-kCiz8k8IbSmVdWLUb17Txy5hmGZJYakohdZuO8+h0vYl2SVIH8jkgsRF6Wad
-TtA+EQshJ87qqAugQjdR29Dk9NCL2VVFuebl647F0gnh538Kxm2i745W963f
-hblQuo+JPWUi/b4nMJXRBu8+/8/wnZfBl2gb2rvVFaSVF44ndJFPr06eFUFm
-eygWl/IlTNArWPHJJG4BRdIvzrvLCM8+3NcSQAWCsWp1ZaiTfw5NbKrP1Eze
-3bpKOdy5IKDdh5ACdXeX3l0CXGY53otgCpb9s87VrCDXFtA+wQU6kshNf1M6
-EuI/Fqx8XnPSLOdEH9TyAKVN0FEMfSVOqk9l8LQf1vqNh0jcjMcy18LKDTkz
-B+9dpPw=
+08UExe8tEw64I9RbLcz/HtM/Jynhv84Fos5f2z6sPS/RJxCsx1CULoG7SfCy
+udzARn2teIjlgW8H+F5g5OTmK5pyJXRnQBPUm+Dzf0lkWqY3ZLKNhlJLV/2d
+dFQV3TyOTF+wMk5ENhZG12txrgEi832tX8o2eLuNx5r0NRmW0zbzFDJKlwPC
+kcY/eDWIsb+KAQxJeR3GpMFQ/Wm5Wo9OaaR8QIJ5rHR+YHdCPy3Xfu0vndYN
+w/Mie1yuOnMeq9ieWwGQBhxcGMqJJSKt0jsl6+Ecs/8GJ8BULhTHrEUTiqBa
+3mrl2AiIIXPVuTtVVZOfJkNnct5pfOO7oEEhpafhJ9OsJROipvc3/gYLohRI
+xHytF0pOtABd2WjIchttpSqy9n6NCuzh2ULdfDziedJm270jqs1l1siYQV6q
+bs4ER0kDXJoUSPVN+kmbTca8K0tfjSYYA2LPbhxYG5oYriWL/uAp4BztnEkW
+Cscc6UJ3Z6CL99JfTp+kKwvE7AHDyWjJsUKNclRmXO33i4vkJf7wAy+E9/8w
+IUm2lWLpQybeIYwGJj6Ku72c9ZPBZyA9gfVPLDo5yR4YTZtUT6WQbn7FjbXl
+jTjg5DBUQuJYs75WxV62XRZAJjvYfk+nhv4CvMXdh2nntvATr2hRLZ2RNbTG
+lPOzX0oDBiSdWD4vKQZq2/dc8vrGdCT27s7konQRbUpxEh/5lBhc92LNQsgB
+V7agBWVfr+ASB6g1PQ480pXEI3BRmjmR/+5jIJ4VN9p4qyGqYFS5Q3wZV86t
++UNNOJhDHgr+H7hm4rLXpMYp2Fki7D/Xaijaq11t65ARCqcFpwtSj68NNeMv
+Y1Tq1ypcOjUGAszYc6K7Ngso4ZWZ8JAiR7lHZP5kHTvd/wG6f/EbavoEjpoW
+NHuYZ7JPZLCmRVRjPa1ivd7h0CfkwJpKn2bUHusgXNgmKRWoAUaQ3piGRNqa
+zyeNJJsZSSbfiC7U1lbj+uMBNPCPTtyAo0ad3IaVhZjzeK614MZN+kebDqLO
+4OmN2JYc5wWNz8PwKTLEH1Chwk9k0l2d2o3C6JSO0EKu9XrIfxXfAGSpTTwt
+h1krSicVjbYGzrYExm9Uha9xHjGdiJiv+PEPCEfKTbZI1HzqRbMbT9sRNnWK
+RmPpN3BGvYFs1g2lz7yNrCNZC8FamUCQPBL1h2xLohqKaWbNXbYq5Nild6NK
+ZFjp2UukfHm2+fgyDzMZNeEg40t6dLD5L7N+hx8FL0z+7hLVpksCDlom+U8P
+3EiClDFt6f6RLApUl3j+1lj4rc4jvUVqZ+ckmxDBKJEU2E9nM7ixx5oqybWf
+YuNI1AnZCpvDPpzWhJHlxlRu/YIzwJ8fyCPgPGbwZu+pGemd9TvWmBZevZvn
+GYZuxYuwPQhDORUA2FchF+BmLRu4hQtKJv8AEOCdikdu/F4WWM0dvfMRWUSo
+57nomhI3UUMm0AI9zxplLXUXIyt4cIL3owgb186C3VHLQRgedQ2ns5iRGRlS
+BEnWK4zoi4ERS/4JGi08sL4c4vydsIHYcsD0M7cM2qURfHSFESGbNGQnaIiE
++CTXvkqIIqwxmZ+Ff26AC0cUtoeymxxe5U54j5RVcv8ij9VvP4rrH0lkuJB/
+Ulz7hD38A9A3PwqqNRPnoXVFke/bBL9N0QXKNFOJpZ6ltZMXjZ7CEq2k0Bq5
++yM8PU7inoNEPjWaS9lPrfqTjTzrGFF62vgc6raRdde6LRTxfggA3pNxxOvK
+5WRedMkrwBSAFGbnsSb12qYYuZiaERLX5GtcHyw4AOmK5N49JLcSuvnX77YZ
+iNP04r4XV2oq1j8pAp6KZQU/j0q17HXWFBopGyGjE18cPpo4bR3NxChodFKe
+QMwgs8gX3xSdZd4UaYaUNKBZvmyxfb2gFTykfSetqikMpbTYxIzPz9ITz8E+
+CoJA3vwnEuBJTI9Owl1lX9FoqTU2wwNPALqhghf5cXseK/UB4evzGzC1gxOx
+nTpG+JkKyqRoiT5XmOidU+CZUA95IYhi5mGEMIqwmsS8HYR8qyULsvtN9RoW
+yI9aTA48UfTN/VazBpTomxrA44ChjLMR7WFwdWXaNGw00faCdhm0fLZo2Kox
+NTBZMhZQarV9RAUU5aozujhkuU+YZVD8CG40RoHFjeeZ8RhODVlntepUC1p1
+ALonjlg+GgzuFtodh6qe/Qx1zjNGLHaT1T/02hWl4ERYSMxkoGPLC7dPjLdc
+liQtPQFLFlEDX6mwmOe7Mcnz90ThyaO7K3MhnMjOFx4cTF84RDaD4Omj+U5Z
+N7k9HVGhYz5VSGvI/dO2j0QFrxwEPt1VVRPT0hlzKsrxSPI2jMtVFhLF2AH/
+PHRjAjc=

data/gdc_fossa_lcm.yaml

@@ -0,0 +1,2 @@
+---
+fossa_project: "gooddata-ruby-lcm"

data/gdc_fossa_ruby_sdk.yaml

@@ -0,0 +1,4 @@
+---
+fossa_project: "gooddata-ruby-sdk"
+ignored_paths:
+  - 'ci/.*'

data/gooddata.gemspec

@@ -49,12 +49,16 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'pronto', '~> 0.10' if RUBY_PLATFORM != 'java'
   s.add_development_dependency 'pronto-rubocop', '~> 0.9' if RUBY_PLATFORM != 'java'
   s.add_development_dependency 'pronto-reek', '~> 0.9' if RUBY_PLATFORM != 'java'
-  s.add_development_dependency 'vcr'
+  s.add_development_dependency 'vcr', '5.0.0'
   s.add_development_dependency 'hashdiff', '~> 0.4'
 
   s.add_development_dependency 'sqlite3' if RUBY_PLATFORM != 'java'
 
-  s.add_dependency 'activesupport', '> 4.2.9', '< 6.1'
+  if RUBY_VERSION >= '2.5'
+    s.add_dependency 'activesupport', '>= 6.0.3.1', '< 6.1'
+  else
+    s.add_dependency 'activesupport', '>= 5.2.4.3', '< 6.0'
+  end
 
   s.add_dependency 'aws-sdk-s3', '~> 1.16'
   s.add_dependency 'docile', '~> 1.1'

data/k8s/charts/lcm-bricks/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 name: lcm-bricks
 description: LCM Bricks
-version: 2.0.1
+version: 2.0.3

data/k8s/charts/lcm-bricks/templates/prometheus/alertingRules.yaml

@@ -20,7 +20,7 @@ data:
         expr: container_pod:lcm_pod_container_status_restarts:increase10m >= 1
         labels:
           severity: warning
-          team: lcm # switch to msf in production
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "There is more than 0 restarts of {{`{{ $labels.pod }}`}} pod in the last 10 minutes"
@@ -28,8 +28,8 @@ data:
       - alert: "[LCM] Pod has too many restarts on cluster={{ .Values.clusterId }}"
         expr: container_pod:lcm_pod_container_status_restarts:increase10m >= 2
         labels:
-          severity: critical
-          team: lcm # switch to msf in production
+          severity: warning
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "There is more than 1 restart of {{`{{ $labels.pod }}`}} pod in the last 10 minutes"
@@ -40,7 +40,7 @@ data:
         expr: container_pod:lcm_pod_container_status_oomkilled:increase10m >= 1
         labels:
           severity: warning
-          team: lcm # switch to msf in production
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "{{`{{ $labels.pod }}`}} was OOMKilled in the last 30 minutes. Investigate and/or increase memoryRequest or memoryLimit."
@@ -48,8 +48,8 @@ data:
       - alert: "[LCM] OOMKill occured on cluster={{ .Values.clusterId }}"
         expr: container_pod:lcm_pod_container_status_oomkilled:increase10m >= 2
         labels:
-          severity: critical
-          team: lcm # switch to msf in production
+          severity: warning
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "{{`{{ $labels.pod }}`}} was OOMKilled in the last 10 minutes. Investigate and/or increase memoryRequest or memoryLimit."
@@ -58,8 +58,8 @@ data:
         expr: rate(container_cpu_cfs_throttled_seconds_total{namespace='{{ .Release.Namespace }}'}[1m]) > 1
         for: 5m
         labels:
-          severity: critical
-          team: lcm # switch to msf in production
+          severity: warning
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "{{`{{ $labels.pod_name }}`}} container is beeing throttled and probably hit CPU limit. Investigate root cause and increase limit and/or number of replicas if necessary."
@@ -68,8 +68,8 @@ data:
         expr: rate(jvm_gc_pause_seconds_sum{kubernetes_namespace='{{ .Release.Namespace }}'}[1m]) > 1
         for: 5m
         labels:
-          severity: critical
-          team: lcm # switch to msf in production
+          severity: warning
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "{{`{{ $labels.kubernetes_pod_name }}`}} container is spending too much time in pause garbage collector. Investigate root cause and increase heap size and/or number of replicas if necessary."
@@ -77,9 +77,19 @@ data:
       - alert: "[LCM] there is more than 100 jobs on cluster={{ .Values.clusterId }}"
         expr: count(kube_job_info{namespace="lcm"}) > 100
         labels:
-          severity: critical
-          team: lcm # switch to msf in production
+          severity: warning
+          team: lcm
           cluster_id: {{ .Values.clusterId }}
         annotations:
           description: "There is more than 100 jobs in LCM namespace. They are likely not deleted."
           summary: "There is more than 100 jobs in LCM namespace."
+      - alert: "[LCM] Resource quotas hit CPU limit on cluster={{ .Values.clusterId }}"
+        expr: kube_resourcequota{namespace='{{ .Release.Namespace }}',resource="limits.cpu",type="hard"} - ignoring(type) kube_resourcequota{namespace='{{ .Release.Namespace }}',resource="limits.cpu",type="used"} == 0
+        labels:
+          severity: warning
+          team: lcm
+          cluster_id: {{ .Values.clusterId }}
+        annotations:
+          description: "We are hitting CPU limit in LCM namespace."
+          summary: "We are hitting CPU limit in LCM namespace."
+

data/lcm.rake

@@ -118,6 +118,7 @@ end
 namespace :docker do
   desc 'Build Docker image'
   task :build do
+    Rake::Task["maven:build_dependencies"].invoke
     system('docker build -f Dockerfile.jruby -t gooddata/appstore .')
   end
 
@@ -127,6 +128,19 @@ namespace :docker do
   end
 end
 
+namespace :maven do
+  task :build_dependencies do
+    system('mvn -f ci/snowflake/pom.xml clean install -P binary-packaging')
+    system('cp -rf ci/snowflake/target/*.jar lib/gooddata/cloud_resources/snowflake/drivers/')
+
+    system('mvn -f ci/bigquery/pom.xml clean install -P binary-packaging')
+    system('cp -rf ci/bigquery/target/*.jar lib/gooddata/cloud_resources/bigquery/drivers/')
+
+    system('mvn -f ci/redshift/pom.xml clean install -P binary-packaging')
+    system('cp -rf ci/redshift/target/*.jar lib/gooddata/cloud_resources/redshift/drivers/')
+  end
+end
+
 namespace :sdk do
   desc 'Updates gooddata-ruby to the version specified in the root Gemfile'
   task :update do

data/lib/gooddata/bricks/middleware/execution_result_middleware.rb

@@ -0,0 +1,68 @@
+# Copyright (c) 2019, GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_middleware'
+
+# rubocop:disable Style/ClassVars
+module GoodData
+  module Bricks
+    module ExecutionStatus
+      OK = 'OK'
+      ERROR = 'ERROR'
+      WARNING = 'WARNING'
+    end
+
+    class ExecutionResultMiddleware < Bricks::Middleware
+      @@result_log_path = nil
+
+      def call(params)
+        result_log_path(params)
+        @app.call(params)
+      end
+
+      # Update process execution result when the script is executed inside a GDC ETL process execution (ruby bricks).
+      # Ruby bricks should update execution result at the end of script with status=WARNING or OK and summary message if the script exits normally.
+      # If any fatal error, script should update execution result with status=ERROR and error message,
+      # then throw exception to notify GDC platform that the script terminated unexpectedly.
+      # @param [HashMap] params contains GDC_EXECUTION_RESULT_LOG_PATH or GDC_LOG_DIRECTORY, GDC_EXECUTION_ID
+      # @param [ExecutionStatus] status execution status
+      # @param [String] message execution message
+      def self.update_execution_result(status, message = "")
+        if status != ExecutionStatus::OK && status != ExecutionStatus::ERROR && status != ExecutionStatus::WARNING
+          GoodData.logger.warn("Unknown execution status #{status}, ignored it.")
+        end
+
+        result = {
+          executionResult: {
+            status: status,
+            message: message
+          }
+        }
+        update_result(result)
+      end
+
+      private
+
+      def result_log_path(params)
+        log_directory = params['GDC_LOG_DIRECTORY']
+        execution_id = params['GDC_EXECUTION_ID']
+        result_log_path = params['GDC_EXECUTION_RESULT_LOG_PATH'] || ENV['GDC_EXECUTION_RESULT_LOG_PATH']
+        result_log_path = "#{log_directory}/#{execution_id}_result.json" if result_log_path.nil? && !log_directory.nil?
+        @@result_log_path = result_log_path
+      end
+
+      def self.update_result(result)
+        if @@result_log_path.nil?
+          GoodData.gd_logger.warn("action=update_execution_result status=error Not found execution result logger file.") unless GoodData.gd_logger.nil?
+          return
+        end
+
+        File.open(@@result_log_path, 'w') { |file| file.write(JSON.pretty_generate(result)) }
+      rescue Exception => e # rubocop:disable RescueException
+        GoodData.gd_logger.error("action=update_execution_result status=error reason=#{e.message}") unless GoodData.gd_logger.nil?
+      end
+    end
+  end
+end
+# rubocop:enable Style/ClassVars

data/lib/gooddata/bricks/middleware/logger_middleware.rb

@@ -36,7 +36,8 @@ module GoodData
           log_directory = params['GDC_LOG_DIRECTORY']
           execution_id = params['GDC_EXECUTION_ID']
           FileUtils.mkpath log_directory
-          logger = Logger.new("#{log_directory}/#{execution_id}.log")
+          execution_log_path = params['GDC_EXECUTION_LOG_PATH'].nil? ? "#{log_directory}/#{execution_id}.log" : params['GDC_EXECUTION_LOG_PATH']
+          logger = Logger.new(execution_log_path)
           logger.level = params['GDC_LOG_LEVEL'] || 'info'
           values_to_mask = params['values_to_mask'] || []
           logger = MaskLoggerDecorator.new(logger, values_to_mask)

data/lib/gooddata/bricks/middleware/mask_logger_decorator.rb

@@ -20,7 +20,11 @@ module GoodData
         # @return [[String]] array of all String in values
         def extract_values(values)
           if values.is_a?(String)
-            [values]
+            if values.nil? || values.empty? || values.gsub(/[*|\s]/, '') == ''
+              []
+            else
+              [values]
+            end
           elsif values.is_a?(Hash) || values.is_a?(Array)
             (values.is_a?(Hash) ? values.values : values).reduce([]) do |strings, item|
               strings.concat extract_values(item)

data/lib/gooddata/bricks/pipeline.rb

@@ -32,6 +32,7 @@ module GoodData
         prepare([
           LoggerMiddleware,
           DecodeParamsMiddleware,
+          ExecutionResultMiddleware,
           BenchMiddleware,
           GoodDataMiddleware,
           AWSMiddleware,
@@ -44,6 +45,7 @@ module GoodData
         prepare([
           LoggerMiddleware,
           DecodeParamsMiddleware,
+          ExecutionResultMiddleware,
           BenchMiddleware,
           GoodDataMiddleware,
           AWSMiddleware,
@@ -56,6 +58,7 @@ module GoodData
       def self.release_brick_pipeline
         prepare([
         LoggerMiddleware,
+        ExecutionResultMiddleware,
         DecodeParamsMiddleware,
         BenchMiddleware,
         GoodDataMiddleware,
@@ -68,6 +71,7 @@ module GoodData
       def self.provisioning_brick_pipeline
         prepare([
         LoggerMiddleware,
+        ExecutionResultMiddleware,
         DecodeParamsMiddleware,
         BenchMiddleware,
         GoodDataMiddleware,
@@ -80,6 +84,7 @@ module GoodData
       def self.rollout_brick_pipeline
         prepare([
         LoggerMiddleware,
+        ExecutionResultMiddleware,
         DecodeParamsMiddleware,
         BenchMiddleware,
         GoodDataMiddleware,
@@ -93,6 +98,7 @@ module GoodData
         prepare(
           [
             LoggerMiddleware,
+            ExecutionResultMiddleware,
             DecodeParamsMiddleware,
             BenchMiddleware,
             HelloWorldBrick
@@ -104,6 +110,7 @@ module GoodData
         prepare(
           [
             LoggerMiddleware,
+            ExecutionResultMiddleware,
             DecodeParamsMiddleware,
             BenchMiddleware,
             HelpBrick

data/lib/gooddata/cloud_resources/bigquery/bigquery_client.rb

@@ -0,0 +1,86 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'securerandom'
+require 'java'
+require 'pathname'
+require_relative '../cloud_resource_client'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + 'drivers/*.jar').each do |file|
+  require file unless file.start_with?('lcm-bigquery-driver')
+end
+
+java_import 'com.google.auth.oauth2.ServiceAccountCredentials'
+java_import 'com.google.cloud.bigquery.BigQuery'
+java_import 'com.google.cloud.bigquery.BigQueryOptions'
+java_import 'com.google.cloud.bigquery.FieldList'
+java_import 'com.google.cloud.bigquery.FieldValue'
+java_import 'com.google.cloud.bigquery.FieldValueList'
+java_import 'com.google.cloud.bigquery.QueryJobConfiguration'
+java_import 'com.google.cloud.bigquery.TableResult'
+java_import 'org.apache.commons.text.StringEscapeUtils'
+
+module GoodData
+  module CloudResources
+    class BigQueryClient < CloudResourceClient
+      class << self
+        def accept?(type)
+          type == 'bigquery'
+        end
+      end
+
+      def initialize(options = {})
+        raise("Data Source needs a client to BigQuery to be able to query the storage but 'bigquery_client' is empty.") unless options['bigquery_client']
+
+        if options['bigquery_client']['connection'].is_a?(Hash)
+          @project = options['bigquery_client']['connection']['project']
+          @schema = options['bigquery_client']['connection']['schema'] || 'public'
+          @authentication = options['bigquery_client']['connection']['authentication']
+        else
+          raise('Missing connection info for BigQuery client')
+
+        end
+      end
+
+      def realize_query(query, _params)
+        GoodData.gd_logger.info("Realize SQL query: type=bigquery status=started")
+
+        client = create_client
+        filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
+        measure = Benchmark.measure do
+          query_config = QueryJobConfiguration.newBuilder(query).setDefaultDataset(@schema).build
+          table_result = client.query(query_config)
+
+          if table_result.getTotalRows.positive?
+            result = table_result.iterateAll
+            field_list = table_result.getSchema.getFields
+            col_count = field_list.size
+            CSV.open(filename, 'wb') do |csv|
+              csv << Array(1..col_count).map { |i| field_list.get(i - 1).getName } # build the header
+              result.each do |row|
+                csv << Array(1..col_count).map { |i| row.get(i - 1).getValue&.to_s }
+              end
+            end
+          end
+        end
+        GoodData.gd_logger.info("Realize SQL query: type=bigquery status=finished duration=#{measure.real}")
+        filename
+      end
+
+      private
+
+      def create_client
+        GoodData.logger.info "Setting up connection to BigQuery"
+        client_email = @authentication['serviceAccount']['clientEmail']
+        private_key = @authentication['serviceAccount']['privateKey']
+        credentials = ServiceAccountCredentials.fromPkcs8(nil, client_email, StringEscapeUtils.unescapeJson(private_key), nil, nil)
+        BigQueryOptions.newBuilder.setProjectId(@project).setCredentials(credentials).build.getService
+      end
+    end
+  end
+end