gooddata 2.1.7-java → 2.1.12-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 63f952c3c47f2ec63afa90a1b47d9d859506c9463f5b123b9b445e8c76bb23b3
-  data.tar.gz: a4b96e6471def8248af6ddc6bcf1dffcb3dbb7b686821ec010ae7ba2684d6041
+  metadata.gz: 73f3c8fde12dff87986355749de47fbdbbc334416d032f58c18d993fd5f28967
+  data.tar.gz: 73e045492e7cccc765c4ad60c08242c53efaf3a92f9c7adf56af9259a22c14e0
 SHA512:
-  metadata.gz: 806a086561c697cbefa90efc4f3207723939ec99b983b9fd2f0ab72480bc794b62acf8af053af9aa76a62f779a03d881778e6d52590d98a715e3efbf26abe119
-  data.tar.gz: e0ee983afa9c57a18242ba10ed0ea5495a2d5411d89b3d62bb87ffc7a12a9d363cf55638d5ff3f9062b9d7761498360237436b0598c92e9b57e8ffe55a4db40d
+  metadata.gz: 1ab74bd538fecff1536d028993fc62cc287435eeb19163fc3db0b842effd3feca3b746113358c7bd3236c9a9ac15f30ac37361b944cadfad6cfa7161f8219f81
+  data.tar.gz: 18c48a23901266649fc9f4cf1db10d5f289ea4726f0cc5f63ceb919007f44f2dbbd6b9a37589bc87994c4104136237f49f0540ab7a4648506b5d5521f80c150b

data/.gdc-ii-config.yaml

@@ -20,3 +20,6 @@ integratedTests:
 configFilesForUpdate:
   - '.gdc-ii-config.yaml'
   - '.gdc-ii-config-chart.yaml'
+
+customConstants:
+  pipeline.deploy.testEnvironments: ['stg3', 'stg2', 'stg']

data/.rubocop.yml

@@ -2,6 +2,7 @@ AllCops:
   DisplayCopNames: true
   Exclude:
       - 'spec/helpers/appstore_project_helper.rb'
+  TargetRubyVersion: 2.3
 
 AbcSize:
   Enabled: false
@@ -98,3 +99,9 @@ UselessAccessModifier:
 
 Style/PerlBackrefs:
   Enabled: false
+
+Style/Encoding:
+  Enabled: false
+
+Layout/EmptyLineAfterMagicComment:
+  Enabled: false

data/.travis.yml

@@ -15,11 +15,9 @@ stages:
 git:
   depth: false # this is needed for pronto
 
-matrix:
+jobs:
   allow_failures:
     - rvm: jruby-9.1.14
-
-jobs:
   include:
   # BEFORE MERGE
   - name: pronto code review
@@ -259,7 +257,7 @@ jobs:
     stage: after-merge
     script: bundle exec rake test:unit
     os: osx
-    osx_image: xcode9.3
+    osx_image: xcode9.4
     rvm: 2.4.3
 
   - name: unit tests 2.5

data/CHANGELOG.md

@@ -1,4 +1,43 @@
 # GoodData Ruby SDK Changelog
+## 2.1.12
+ - FEATURE: MSF-17621 Apply patched version for activesupport to fix vulnerable issue
+ - CONFIG: SETI-4379 Add gdc-fossa configuration for gooddata-ruby
+ - CONFIG: MSF-17345 Set umask 0002 for lcm brick
+ - BUGFIX: TMA-1015 check case sensitivity for group name
+ - BUGFIX: MSF-17219 Fixed incorrect client used in domain.users
+ - BUGFIX: TMA-1022 Add more logs for case roll out without LDM change in master
+ - FEATURE: TMA-1640 Add alert for CPU limit hit
+
+## 2.1.11
+ - FEATURE: TMA-1647 Set VCR version to 5.0.0 due to incompatible license of the latest version
+
+## 2.1.10
+ - BUGFIX: TMA-1653 fix performance issue in functions project.users and domain.users
+ - BUGFIX: TMA-1643 Don't convert null value to empty string
+ - BUGFIX: TMA-1620 Users Brick, sometimes update users are conflicted
+ - BUGFIX: TMA-1642 parse csv from input source with case insensitive
+ - BUGFIX: TMA-1528: remove CollectSegments and CollectDataProduct for add and remove in users brick
+ - FEATURE: TMA-1629 Add parameter "set_master_project" to support reset latest master project
+ - FEATURE: TMA-1630 Support Snowflake, BigQuery as input source
+## 2.1.9
+ - FEATURE: TMA-1076 support new version api 2
+ - BUGFIX: TMA-1637 handle input_source of dynamic params
+ - BUGFIX: TMA-1636 Build csv file with force_quotes
+ - FEATURE: TMA-1614 Support redshift input source
+ - FEATURE: TMA-1259 Start using dataproduct in NFS release table
+ - FEATURE: MSF-16455 support yellow WARNING status into RubySDK
+ - CONFIG: TMA-1625 update version lock for test docker images
+ - BUGFIX: TMA-1602 User filter brick failed - K8s bricks don't show error properly
+ - BUGFIX: TMA-1593 Increase java heap space during execute bricks
+ - BUGFIX: TMA-1558 K8s bricks don't show error when config invalid
+ - BUGFIX: TMA-1596 The error propagated from a LCM brick is shown multiple times in the log
+ - BUGFIX: TMA-1582 show synchronize ldm mode is running
+ - FEATURE: TMA-1588 support schedule param include deprecated
+ - FEATURE: TMA-1597 Logging lcm execution result
+
+## 2.1.8
+ - FEATURE: TMA-1604 Upgrade Restforce version to 3.x
+
 ## 2.1.7
  - BUGFIX: TMA-1577 Bump brick version to 3.7.9
  - FEATURE: MSF-16279 Allow gooddata-ruby to be installed with Rails 6 apps

data/Dockerfile

@@ -1,4 +1,4 @@
-FROM harbor.intgdc.com/tools/gdc-java-8-jre:d0f5e08
+FROM harbor.intgdc.com/tools/gdc-java-8-jre:2fce51b
 
 ARG RVM_VERSION=stable
 ARG JRUBY_VERSION=9.2.5.0
@@ -6,10 +6,10 @@ ARG JRUBY_VERSION=9.2.5.0
 LABEL image_name="GDC LCM Bricks"
 LABEL maintainer="LCM <lcm@gooddata.com>"
 LABEL git_repository_url="https://github.com/gooddata/gooddata-ruby/"
-LABEL parent_image="harbor.intgdc.com/tools/gdc-java-8-jre:d0f5e08"
+LABEL parent_image="harbor.intgdc.com/tools/gdc-java-8-jre:2fce51b"
 
 # which is required by RVM
-RUN yum install -y curl which patch make git \
+RUN yum install -y curl which patch make git maven \
     && yum clean all \
     && rm -rf /var/cache/yum
 
@@ -43,12 +43,25 @@ RUN groupadd -g 48 apache \
 USER apache
 
 ADD ./bin ./bin
-ADD ./lib ./lib
+ADD --chown=apache:apache ./ci ./ci
+ADD --chown=apache:apache ./lib ./lib
 ADD ./SDK_VERSION .
 ADD ./VERSION .
 ADD ./Gemfile .
 ADD ./gooddata.gemspec .
 
+#build redshift dependencies
+RUN mvn -f ci/redshift/pom.xml clean install -P binary-packaging
+RUN cp -rf ci/redshift/target/*.jar ./lib/gooddata/cloud_resources/redshift/drivers/
+
+#build snowflake dependencies
+RUN mvn -f ci/snowflake/pom.xml clean install -P binary-packaging
+RUN cp -rf ci/snowflake/target/*.jar ./lib/gooddata/cloud_resources/snowflake/drivers/
+
+#build bigquery dependencies
+RUN mvn -f ci/bigquery/pom.xml clean install -P binary-packaging
+RUN cp -rf ci/bigquery/target/*.jar ./lib/gooddata/cloud_resources/bigquery/drivers/
+
 RUN bundle install
 
 ARG GIT_COMMIT=unspecified

data/Dockerfile.jruby

@@ -7,11 +7,11 @@ RUN apk add --no-cache curl make gcc git g++ python linux-headers binutils-gold
 # Switch to directory with sources
 WORKDIR /src
 ENV HOME=/src
+ENV BUNDLE_PATH=$HOME/bundle
 
-RUN gem update --system \
-    && gem install bundler
-
-ENV BUNDLE_PATH=/bundle
+RUN gem update --system 3.0.6 \
+    && gem install --install-dir $BUNDLE_PATH bundler -v 1.17.3 \
+    && gem install --install-dir $BUNDLE_PATH rake -v 11.3.0
 
 ADD . .
 

data/Dockerfile.ruby

@@ -8,11 +8,12 @@ RUN ln -s /usr/bin/make /usr/bin/gmake
 
 # Switch to directory with sources
 WORKDIR /src
+ENV HOME=/src
+ENV BUNDLE_PATH=$HOME/bundle
 
-RUN gem update --system \
-    && gem install bundler
-
-ENV BUNDLE_PATH=/bundle
+RUN gem update --system 3.0.6 \
+    && gem install --install-dir $BUNDLE_PATH bundler -v 1.17.3 \
+    && gem install --install-dir $BUNDLE_PATH rake -v 11.3.0
 
 ADD . .
 

data/SDK_VERSION

@@ -1 +1 @@
-2.1.7
+2.1.12

data/VERSION

@@ -1 +1 @@
-3.7.9
+3.7.20

data/bin/provision.sh

@@ -1,3 +1,5 @@
 #!/usr/bin/env bash
 
+export JRUBY_OPTS=-J-Xmx2560m
+
 /bin/bash -l -c "bundle exec ./bin/run_brick.rb provisioning_brick"

data/bin/release.sh

@@ -1,3 +1,5 @@
 #!/usr/bin/env bash
 
+export JRUBY_OPTS=-J-Xmx2560m
+
 /bin/bash -l -c "bundle exec ./bin/run_brick.rb release_brick"

data/bin/rollout.sh

@@ -1,3 +1,5 @@
 #!/usr/bin/env bash
 
+export JRUBY_OPTS=-J-Xmx2560m
+
 /bin/bash -l -c "bundle exec ./bin/run_brick.rb rollout_brick"

data/bin/run_brick.rb

@@ -8,34 +8,58 @@ DEFAULT_BRICK = 'hello_world_brick'
 BRICK_PARAM_PREFIX = 'BRICK_PARAM_'
 HIDDEN_BRICK_PARAMS_PREFIX = 'HIDDEN_BRICK_PARAM_'
 
+# MSF-17345 Set umask so files are group-writable
+File.umask(0002)
+
 brick_type = !ARGV.empty? ? ARGV[0] : DEFAULT_BRICK
 
 def get_brick_params(prefix)
   ENV.select { |k,| k.to_s.match(/^#{prefix}.*/) }.map { |k, v| [k.slice(prefix.length..-1), v] }.to_h
 end
 
+def handle_error(params, log, brick_type, error, error_message)
+  execution_log = GoodData.logger
+  execution_log.error "Execution failed. Error: #{error}" unless execution_log.nil?
+  GoodData::Bricks::ExecutionResultMiddleware.update_execution_result(GoodData::Bricks::ExecutionStatus::ERROR, error_message)
+  log.error "action=#{brick_type}_execution status=failed commit_hash=#{params['GOODDATA_RUBY_COMMIT']} execution_id=#{params['GDC_EXECUTION_ID']} exception=#{error}"
+  raise
+end
+
 syslog_node = ENV['NODE_NAME']
 log = RemoteSyslogLogger.new(syslog_node, 514, :program => "ruby_#{brick_type}", :facility => 'local2')
 
 log.info "action=#{brick_type}_execution status=init"
 
 begin
+  commit_hash = ENV['GOODDATA_RUBY_COMMIT'] || ''
+  execution_id = ENV['GDC_EXECUTION_ID']
+  log_directory = ENV['GDC_LOG_DIRECTORY'] || '/tmp/'
+  execution_log_path = ENV['GDC_EXECUTION_LOG_PATH']
+  execution_result_log_path = ENV['GDC_EXECUTION_RESULT_LOG_PATH']
+  execution_result_log_params = {
+    'GOODDATA_RUBY_COMMIT' => commit_hash,
+    'GDC_EXECUTION_ID' => execution_id,
+    'GDC_LOG_DIRECTORY' => log_directory,
+    'GDC_EXECUTION_LOG_PATH' => execution_log_path,
+    'GDC_EXECUTION_RESULT_LOG_PATH' => execution_result_log_path
+  }
+
   brick_pipeline = GoodData::Bricks::Pipeline.send("#{brick_type}_pipeline")
   normal_params = get_brick_params(BRICK_PARAM_PREFIX)
   hidden_params = get_brick_params(HIDDEN_BRICK_PARAMS_PREFIX)
   params = normal_params.merge(hidden_params)
 
   params['values_to_mask'] = hidden_params.values
-  commit_hash = ENV['GOODDATA_RUBY_COMMIT'] || ''
-  execution_id = ENV['GDC_EXECUTION_ID']
   params['gooddata_ruby_commit'] = commit_hash
-  params['GDC_LOG_DIRECTORY'] = ENV['GDC_LOG_DIRECTORY'] || '/tmp/'
+  params['GDC_LOG_DIRECTORY'] = log_directory
   params['GDC_EXECUTION_ID'] = execution_id
+  params['GDC_EXECUTION_LOG_PATH'] = execution_log_path
+  params['GDC_EXECUTION_RESULT_LOG_PATH'] = execution_result_log_path
+
   log.info "action=#{brick_type}_execution status=start commit_hash=#{commit_hash} execution_id=#{execution_id}"
   brick_pipeline.call(params)
+rescue GoodData::LcmExecutionError => lcm_error
+  handle_error(execution_result_log_params, log, brick_type, lcm_error, lcm_error.summary_error)
 rescue Exception => e # rubocop:disable RescueException
-	execution_log = GoodData.logger
-	execution_log.error "Execution failed. Message: #{e.message}. Error: #{e}" unless execution_log.nil?
-  log.info "action=#{brick_type}_execution status=failed commit_hash=#{commit_hash} execution_id=#{execution_id} exception=#{e}"
-  raise
+  handle_error(execution_result_log_params, log, brick_type, e, e.to_s)
 end

data/bin/test_projects_cleanup.rb

@@ -29,6 +29,7 @@ def delete_project_by_title(title, projects, days = 14, force = false)
     p.title.match(title) && p.created < dead_line
   end
   filtered_projects.each do |project|
+    begin
     if force
       puts "Deleting: #{project.pid} - #{project.title} - #{project.created}"
       project_add = project.add
@@ -37,6 +38,9 @@ def delete_project_by_title(title, projects, days = 14, force = false)
     else
       puts "Would delete: #{project.pid} - #{project.title} - #{project.created}"
     end
+    rescue StandardError => ex
+      puts "Failed to delete project #{project.pid}, reason: #{ex}"
+    end
   end
   puts "#{filtered_projects.length} projects matching \"#{title}\" #{'would be ' unless force}deleted."
 end
@@ -86,7 +90,11 @@ def clean_up!(client, force, days)
   delete_project_by_title(/LCM spec Client With Conflicting LDM Changes/, projects, days, force)
   delete_project_by_title(/LCM spec master project/, projects, days, force)
   delete_project_by_title(/users brick load test/, projects, days, force)
-  delete_project_by_title(/#transfer_processes and #transfer_schedules test/, projects, days, force)
+  delete_project_by_title(/transfer_processes and #transfer_schedules test/, projects, days, force)
+  delete_project_by_title(/DailyUse Project for gooddata-ruby integration tests/, projects, days, force)
+  delete_project_by_title(/^New project$/, projects, days, force)
+  delete_project_by_title(/RubyGem Dev Week test/, projects, days, force)
+  delete_project_by_title(/My project from blueprint/, projects, days, force)
   delete_ads_by_title(/Development ADS/, client, days, force)
   delete_ads_by_title(/Production ADS/, client, days, force)
   delete_ads_by_title(/TEST ADS/, client, days, force)
@@ -108,7 +116,7 @@ dev_client = init_client(username, password, "https://#{config[:dev_server]}")
 prod_client = init_client(username, password, "https://#{config[:prod_server]}")
 
 force = options[:force]
-days = options[:days] || 14
+days = options[:days] || 3
 clean_up!(dev_client, force, days)
 clean_up!(prod_client, force, days)
 

data/bin/user_filters.sh

@@ -1,3 +1,5 @@
 #!/usr/bin/env bash
 
+export JRUBY_OPTS=-J-Xmx2560m
+
 /bin/bash -l -c "bundle exec ./bin/run_brick.rb user_filters_brick"

data/ci.rake

@@ -1,7 +1,7 @@
 namespace :docker do
   desc 'Build Docker image'
   task :build do
-    system('docker-compose build')
+    system('docker-compose build --no-cache')
   end
 
   desc 'Bundles gems using cache'

data/ci/bigquery/pom.xml

@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.gooddata.lcm</groupId>
+    <artifactId>lcm-bigquery-driver</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.google.auth</groupId>
+            <artifactId>google-auth-library-oauth2-http</artifactId>
+            <version>0.16.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-text</artifactId>
+            <version>1.7</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-bigquery</artifactId>
+            <version>1.102.0</version>
+        </dependency>
+    </dependencies>
+
+    <profiles>
+        <profile>
+            <id>binary-packaging</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-dependency-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>copy-dependencies</goal>
+                                </goals>
+                                <configuration>
+                                    <outputDirectory>${project.build.directory}</outputDirectory>
+                                    <!-- compile scope gives runtime and compile dependencies (skips test deps) -->
+                                    <includeScope>runtime</includeScope>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+</project>

data/ci/redshift/pom.xml

@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.gooddata.lcm</groupId>
+    <artifactId>lcm-redshift-driver</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.amazon.redshift</groupId>
+            <artifactId>redshift-jdbc42-no-awssdk</artifactId>
+            <version>1.2.36.1060</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-core</artifactId>
+            <version>1.11.633</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-redshift</artifactId>
+            <version>1.11.633</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-sts</artifactId>
+            <version>1.11.633</version>
+        </dependency>
+        <dependency>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+            <version>1.2.17</version>
+            <scope>compile</scope>
+        </dependency>
+    </dependencies>
+
+    <profiles>
+        <profile>
+            <id>binary-packaging</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-dependency-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>copy-dependencies</goal>
+                                </goals>
+                                <configuration>
+                                    <outputDirectory>${project.build.directory}</outputDirectory>
+                                    <!-- compile scope gives runtime and compile dependencies (skips test deps) -->
+                                    <includeScope>runtime</includeScope>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+
+    <repositories>
+        <repository>
+            <id>my-repo1</id>
+            <name>my custom repo</name>
+            <url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
+        </repository>
+    </repositories>
+</project>