gooddata 2.1.8-java → 2.1.13-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fec58c94885527f530a78738e98c1af58921a28203b51d82cd355cec902c2b1c
-  data.tar.gz: 9759123a281f2be6dffaca14fa992e075514a3d6c2915f97b10275b5c8371cf5
+  metadata.gz: efc9086e1ddd4dea37bcc2836f3511352e04244d53195ced638d9cf5fc5f2edc
+  data.tar.gz: b71fb4b7b838aa0722ac8d1e193680848996691040373b82c821548d022a3adf
 SHA512:
-  metadata.gz: e4d92dd24e6e5cfec17a0dd83b3969a80a5b616ba2aa3e9bb7e08dd44898538eb26b2cbe104da61fa65d0cddcb8829a5902e1895bdb1539be10b56c6b90f4371
-  data.tar.gz: e88afe5e138338229f1aaba220dee25ae4aca2ad1d4330a451377742fea30c1b2a12daeabc05e07eebacc62ca828ce144e74fe6a2357c62d170230fe70a228bc
+  metadata.gz: 5a2bcab685647330aa37a7201f629c8b4d73105506e0330b116bd7a0b5cbe5bfbd9f0f5838c40eb64319b20d503073ec68248603c7dfbf504f28e8058d9e1d1a
+  data.tar.gz: 7d193cdafe6a9ba629b66960415d402cd2a371deeff3f7fcae7311bfcf5c9e07f4f9d4571a7b7612aa430fff8cf15542730892b0907ff974b5a054d14b73577c

data/.rubocop.yml

@@ -2,6 +2,7 @@ AllCops:
   DisplayCopNames: true
   Exclude:
       - 'spec/helpers/appstore_project_helper.rb'
+  TargetRubyVersion: 2.3
 
 AbcSize:
   Enabled: false
@@ -98,3 +99,9 @@ UselessAccessModifier:
 
 Style/PerlBackrefs:
   Enabled: false
+
+Style/Encoding:
+  Enabled: false
+
+Layout/EmptyLineAfterMagicComment:
+  Enabled: false

data/.travis.yml

@@ -15,11 +15,9 @@ stages:
 git:
   depth: false # this is needed for pronto
 
-matrix:
+jobs:
   allow_failures:
     - rvm: jruby-9.1.14
-
-jobs:
   include:
   # BEFORE MERGE
   - name: pronto code review
@@ -259,7 +257,7 @@ jobs:
     stage: after-merge
     script: bundle exec rake test:unit
     os: osx
-    osx_image: xcode9.3
+    osx_image: xcode9.4
     rvm: 2.4.3
 
   - name: unit tests 2.5

data/CHANGELOG.md

@@ -1,4 +1,47 @@
 # GoodData Ruby SDK Changelog
+## 2.1.13
+ - FEATURE: TMA-1676 Support LCM release across domain
+ - FEATURE: TMA-1672 Support sync process with generic datasource
+ - FEATURE: MSF-17743 upgrade custom v2 for rollout brick
+ - BUGFIX: MSF-17975 Introduce gdcshare to lcm bricks
+ - BUGFIX: TMA-1673 Update params processing to accept dot and space
+
+## 2.1.12
+ - FEATURE: MSF-17621 Apply patched version for activesupport to fix vulnerable issue
+ - CONFIG: SETI-4379 Add gdc-fossa configuration for gooddata-ruby
+ - CONFIG: MSF-17345 Set umask 0002 for lcm brick
+ - BUGFIX: TMA-1015 check case sensitivity for group name
+ - BUGFIX: MSF-17219 Fixed incorrect client used in domain.users
+ - BUGFIX: TMA-1022 Add more logs for case roll out without LDM change in master
+ - FEATURE: TMA-1640 Add alert for CPU limit hit
+
+## 2.1.11
+ - FEATURE: TMA-1647 Set VCR version to 5.0.0 due to incompatible license of the latest version
+
+## 2.1.10
+ - BUGFIX: TMA-1653 fix performance issue in functions project.users and domain.users
+ - BUGFIX: TMA-1643 Don't convert null value to empty string
+ - BUGFIX: TMA-1620 Users Brick, sometimes update users are conflicted
+ - BUGFIX: TMA-1642 parse csv from input source with case insensitive
+ - BUGFIX: TMA-1528: remove CollectSegments and CollectDataProduct for add and remove in users brick
+ - FEATURE: TMA-1629 Add parameter "set_master_project" to support reset latest master project
+ - FEATURE: TMA-1630 Support Snowflake, BigQuery as input source
+## 2.1.9
+ - FEATURE: TMA-1076 support new version api 2
+ - BUGFIX: TMA-1637 handle input_source of dynamic params
+ - BUGFIX: TMA-1636 Build csv file with force_quotes
+ - FEATURE: TMA-1614 Support redshift input source
+ - FEATURE: TMA-1259 Start using dataproduct in NFS release table
+ - FEATURE: MSF-16455 support yellow WARNING status into RubySDK
+ - CONFIG: TMA-1625 update version lock for test docker images
+ - BUGFIX: TMA-1602 User filter brick failed - K8s bricks don't show error properly
+ - BUGFIX: TMA-1593 Increase java heap space during execute bricks
+ - BUGFIX: TMA-1558 K8s bricks don't show error when config invalid
+ - BUGFIX: TMA-1596 The error propagated from a LCM brick is shown multiple times in the log
+ - BUGFIX: TMA-1582 show synchronize ldm mode is running
+ - FEATURE: TMA-1588 support schedule param include deprecated
+ - FEATURE: TMA-1597 Logging lcm execution result
+
 ## 2.1.8
  - FEATURE: TMA-1604 Upgrade Restforce version to 3.x
 

data/Dockerfile

@@ -1,4 +1,4 @@
-FROM harbor.intgdc.com/tools/gdc-java-8-jre:d0f5e08
+FROM harbor.intgdc.com/tools/gdc-java-8-jre:2fce51b
 
 ARG RVM_VERSION=stable
 ARG JRUBY_VERSION=9.2.5.0
@@ -6,10 +6,10 @@ ARG JRUBY_VERSION=9.2.5.0
 LABEL image_name="GDC LCM Bricks"
 LABEL maintainer="LCM <lcm@gooddata.com>"
 LABEL git_repository_url="https://github.com/gooddata/gooddata-ruby/"
-LABEL parent_image="harbor.intgdc.com/tools/gdc-java-8-jre:d0f5e08"
+LABEL parent_image="harbor.intgdc.com/tools/gdc-java-8-jre:2fce51b"
 
 # which is required by RVM
-RUN yum install -y curl which patch make git \
+RUN yum install -y curl which patch make git maven \
     && yum clean all \
     && rm -rf /var/cache/yum
 
@@ -37,18 +37,33 @@ RUN rvm install jruby-${JRUBY_VERSION} && gem update --system \
 WORKDIR /src
 
 RUN groupadd -g 48 apache \
+    && groupadd -g 65065 gdcshare \
     && useradd -u 48 -m --no-log-init -r -g apache -G rvm apache \
+    && usermod -a -G gdcshare apache \
     && chown apache: /src
 
 USER apache
 
 ADD ./bin ./bin
-ADD ./lib ./lib
+ADD --chown=apache:apache ./ci ./ci
+ADD --chown=apache:apache ./lib ./lib
 ADD ./SDK_VERSION .
 ADD ./VERSION .
 ADD ./Gemfile .
 ADD ./gooddata.gemspec .
 
+#build redshift dependencies
+RUN mvn -f ci/redshift/pom.xml clean install -P binary-packaging
+RUN cp -rf ci/redshift/target/*.jar ./lib/gooddata/cloud_resources/redshift/drivers/
+
+#build snowflake dependencies
+RUN mvn -f ci/snowflake/pom.xml clean install -P binary-packaging
+RUN cp -rf ci/snowflake/target/*.jar ./lib/gooddata/cloud_resources/snowflake/drivers/
+
+#build bigquery dependencies
+RUN mvn -f ci/bigquery/pom.xml clean install -P binary-packaging
+RUN cp -rf ci/bigquery/target/*.jar ./lib/gooddata/cloud_resources/bigquery/drivers/
+
 RUN bundle install
 
 ARG GIT_COMMIT=unspecified

data/Dockerfile.jruby

@@ -7,11 +7,11 @@ RUN apk add --no-cache curl make gcc git g++ python linux-headers binutils-gold
 # Switch to directory with sources
 WORKDIR /src
 ENV HOME=/src
+ENV BUNDLE_PATH=$HOME/bundle
 
-RUN gem update --system \
-    && gem install bundler
-
-ENV BUNDLE_PATH=/bundle
+RUN gem update --system 3.0.6 \
+    && gem install --install-dir $BUNDLE_PATH bundler -v 1.17.3 \
+    && gem install --install-dir $BUNDLE_PATH rake -v 11.3.0
 
 ADD . .
 

data/Dockerfile.ruby

@@ -8,11 +8,12 @@ RUN ln -s /usr/bin/make /usr/bin/gmake
 
 # Switch to directory with sources
 WORKDIR /src
+ENV HOME=/src
+ENV BUNDLE_PATH=$HOME/bundle
 
-RUN gem update --system \
-    && gem install bundler
-
-ENV BUNDLE_PATH=/bundle
+RUN gem update --system 3.0.6 \
+    && gem install --install-dir $BUNDLE_PATH bundler -v 1.17.3 \
+    && gem install --install-dir $BUNDLE_PATH rake -v 11.3.0
 
 ADD . .
 

data/README.md

@@ -9,6 +9,8 @@ The best documentation for the GoodData API can be found using these resources:
  * http://developer.gooddata.com/api
  * https://secure.gooddata.com/gdc
  * http://rubydoc.info/gems/gooddata/frames
+ 
+Feel free to check out the [GoodData community website](http://community.gooddata.com/) if you have any questions about the GoodData Analytics platform, our API, or this library.
 
 ## Status
 

data/SDK_VERSION

@@ -1 +1 @@
-2.1.8
+2.1.13

data/VERSION

@@ -1 +1 @@
-3.7.9
+3.7.22

data/bin/provision.sh

@@ -1,3 +1,5 @@
 #!/usr/bin/env bash
 
+export JRUBY_OPTS=-J-Xmx2560m
+
 /bin/bash -l -c "bundle exec ./bin/run_brick.rb provisioning_brick"

data/bin/release.sh

@@ -1,3 +1,5 @@
 #!/usr/bin/env bash
 
+export JRUBY_OPTS=-J-Xmx2560m
+
 /bin/bash -l -c "bundle exec ./bin/run_brick.rb release_brick"

data/bin/rollout.sh

@@ -1,3 +1,5 @@
 #!/usr/bin/env bash
 
+export JRUBY_OPTS=-J-Xmx2560m
+
 /bin/bash -l -c "bundle exec ./bin/run_brick.rb rollout_brick"

data/bin/run_brick.rb

@@ -8,34 +8,58 @@ DEFAULT_BRICK = 'hello_world_brick'
 BRICK_PARAM_PREFIX = 'BRICK_PARAM_'
 HIDDEN_BRICK_PARAMS_PREFIX = 'HIDDEN_BRICK_PARAM_'
 
+# MSF-17345 Set umask so files are group-writable
+File.umask(0002)
+
 brick_type = !ARGV.empty? ? ARGV[0] : DEFAULT_BRICK
 
 def get_brick_params(prefix)
   ENV.select { |k,| k.to_s.match(/^#{prefix}.*/) }.map { |k, v| [k.slice(prefix.length..-1), v] }.to_h
 end
 
+def handle_error(params, log, brick_type, error, error_message)
+  execution_log = GoodData.logger
+  execution_log.error "Execution failed. Error: #{error}" unless execution_log.nil?
+  GoodData::Bricks::ExecutionResultMiddleware.update_execution_result(GoodData::Bricks::ExecutionStatus::ERROR, error_message)
+  log.error "action=#{brick_type}_execution status=failed commit_hash=#{params['GOODDATA_RUBY_COMMIT']} execution_id=#{params['GDC_EXECUTION_ID']} exception=#{error}"
+  raise
+end
+
 syslog_node = ENV['NODE_NAME']
 log = RemoteSyslogLogger.new(syslog_node, 514, :program => "ruby_#{brick_type}", :facility => 'local2')
 
 log.info "action=#{brick_type}_execution status=init"
 
 begin
+  commit_hash = ENV['GOODDATA_RUBY_COMMIT'] || ''
+  execution_id = ENV['GDC_EXECUTION_ID']
+  log_directory = ENV['GDC_LOG_DIRECTORY'] || '/tmp/'
+  execution_log_path = ENV['GDC_EXECUTION_LOG_PATH']
+  execution_result_log_path = ENV['GDC_EXECUTION_RESULT_LOG_PATH']
+  execution_result_log_params = {
+    'GOODDATA_RUBY_COMMIT' => commit_hash,
+    'GDC_EXECUTION_ID' => execution_id,
+    'GDC_LOG_DIRECTORY' => log_directory,
+    'GDC_EXECUTION_LOG_PATH' => execution_log_path,
+    'GDC_EXECUTION_RESULT_LOG_PATH' => execution_result_log_path
+  }
+
   brick_pipeline = GoodData::Bricks::Pipeline.send("#{brick_type}_pipeline")
   normal_params = get_brick_params(BRICK_PARAM_PREFIX)
   hidden_params = get_brick_params(HIDDEN_BRICK_PARAMS_PREFIX)
   params = normal_params.merge(hidden_params)
 
   params['values_to_mask'] = hidden_params.values
-  commit_hash = ENV['GOODDATA_RUBY_COMMIT'] || ''
-  execution_id = ENV['GDC_EXECUTION_ID']
   params['gooddata_ruby_commit'] = commit_hash
-  params['GDC_LOG_DIRECTORY'] = ENV['GDC_LOG_DIRECTORY'] || '/tmp/'
+  params['GDC_LOG_DIRECTORY'] = log_directory
   params['GDC_EXECUTION_ID'] = execution_id
+  params['GDC_EXECUTION_LOG_PATH'] = execution_log_path
+  params['GDC_EXECUTION_RESULT_LOG_PATH'] = execution_result_log_path
+
   log.info "action=#{brick_type}_execution status=start commit_hash=#{commit_hash} execution_id=#{execution_id}"
   brick_pipeline.call(params)
+rescue GoodData::LcmExecutionError => lcm_error
+  handle_error(execution_result_log_params, log, brick_type, lcm_error, lcm_error.summary_error)
 rescue Exception => e # rubocop:disable RescueException
-	execution_log = GoodData.logger
-	execution_log.error "Execution failed. Message: #{e.message}. Error: #{e}" unless execution_log.nil?
-  log.info "action=#{brick_type}_execution status=failed commit_hash=#{commit_hash} execution_id=#{execution_id} exception=#{e}"
-  raise
+  handle_error(execution_result_log_params, log, brick_type, e, e.to_s)
 end

data/bin/test_projects_cleanup.rb

@@ -29,6 +29,7 @@ def delete_project_by_title(title, projects, days = 14, force = false)
     p.title.match(title) && p.created < dead_line
   end
   filtered_projects.each do |project|
+    begin
     if force
       puts "Deleting: #{project.pid} - #{project.title} - #{project.created}"
       project_add = project.add
@@ -37,6 +38,9 @@ def delete_project_by_title(title, projects, days = 14, force = false)
     else
       puts "Would delete: #{project.pid} - #{project.title} - #{project.created}"
     end
+    rescue StandardError => ex
+      puts "Failed to delete project #{project.pid}, reason: #{ex}"
+    end
   end
   puts "#{filtered_projects.length} projects matching \"#{title}\" #{'would be ' unless force}deleted."
 end
@@ -86,7 +90,11 @@ def clean_up!(client, force, days)
   delete_project_by_title(/LCM spec Client With Conflicting LDM Changes/, projects, days, force)
   delete_project_by_title(/LCM spec master project/, projects, days, force)
   delete_project_by_title(/users brick load test/, projects, days, force)
-  delete_project_by_title(/#transfer_processes and #transfer_schedules test/, projects, days, force)
+  delete_project_by_title(/transfer_processes and #transfer_schedules test/, projects, days, force)
+  delete_project_by_title(/DailyUse Project for gooddata-ruby integration tests/, projects, days, force)
+  delete_project_by_title(/^New project$/, projects, days, force)
+  delete_project_by_title(/RubyGem Dev Week test/, projects, days, force)
+  delete_project_by_title(/My project from blueprint/, projects, days, force)
   delete_ads_by_title(/Development ADS/, client, days, force)
   delete_ads_by_title(/Production ADS/, client, days, force)
   delete_ads_by_title(/TEST ADS/, client, days, force)
@@ -108,7 +116,7 @@ dev_client = init_client(username, password, "https://#{config[:dev_server]}")
 prod_client = init_client(username, password, "https://#{config[:prod_server]}")
 
 force = options[:force]
-days = options[:days] || 14
+days = options[:days] || 3
 clean_up!(dev_client, force, days)
 clean_up!(prod_client, force, days)
 

data/bin/user_filters.sh

@@ -1,3 +1,5 @@
 #!/usr/bin/env bash
 
+export JRUBY_OPTS=-J-Xmx2560m
+
 /bin/bash -l -c "bundle exec ./bin/run_brick.rb user_filters_brick"

data/ci.rake

@@ -1,7 +1,7 @@
 namespace :docker do
   desc 'Build Docker image'
   task :build do
-    system('docker-compose build')
+    system('docker-compose build --no-cache')
   end
 
   desc 'Bundles gems using cache'

data/ci/bigquery/pom.xml

@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.gooddata.lcm</groupId>
+    <artifactId>lcm-bigquery-driver</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.google.auth</groupId>
+            <artifactId>google-auth-library-oauth2-http</artifactId>
+            <version>0.16.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-text</artifactId>
+            <version>1.7</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-bigquery</artifactId>
+            <version>1.102.0</version>
+        </dependency>
+    </dependencies>
+
+    <profiles>
+        <profile>
+            <id>binary-packaging</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-dependency-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>copy-dependencies</goal>
+                                </goals>
+                                <configuration>
+                                    <outputDirectory>${project.build.directory}</outputDirectory>
+                                    <!-- compile scope gives runtime and compile dependencies (skips test deps) -->
+                                    <includeScope>runtime</includeScope>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+</project>

data/ci/redshift/pom.xml

@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.gooddata.lcm</groupId>
+    <artifactId>lcm-redshift-driver</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.amazon.redshift</groupId>
+            <artifactId>redshift-jdbc42-no-awssdk</artifactId>
+            <version>1.2.36.1060</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-core</artifactId>
+            <version>1.11.633</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-redshift</artifactId>
+            <version>1.11.633</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-sts</artifactId>
+            <version>1.11.633</version>
+        </dependency>
+        <dependency>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+            <version>1.2.17</version>
+            <scope>compile</scope>
+        </dependency>
+    </dependencies>
+
+    <profiles>
+        <profile>
+            <id>binary-packaging</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-dependency-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>copy-dependencies</goal>
+                                </goals>
+                                <configuration>
+                                    <outputDirectory>${project.build.directory}</outputDirectory>
+                                    <!-- compile scope gives runtime and compile dependencies (skips test deps) -->
+                                    <includeScope>runtime</includeScope>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+
+    <repositories>
+        <repository>
+            <id>my-repo1</id>
+            <name>my custom repo</name>
+            <url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
+        </repository>
+    </repositories>
+</project>