gooddata 2.3.2-java → 2.3.3-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gdc-ii-config.yaml +1 -1
- data/.github/workflows/build.yml +2 -2
- data/.github/workflows/pre-merge.yml +1 -1
- data/CHANGELOG.md +3 -0
- data/Dockerfile +2 -2
- data/Dockerfile.jruby +1 -1
- data/DockerfileOldImage +88 -0
- data/SDK_VERSION +1 -1
- data/VERSION +1 -1
- data/bin/provision.sh +1 -1
- data/bin/release.sh +1 -1
- data/bin/rollout.sh +1 -1
- data/bin/user_filters.sh +1 -1
- data/bin/users.sh +1 -1
- data/ci/snowflake/pom.xml +11 -1
- data/gooddata.gemspec +21 -9
- data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +54 -2
- data/lib/gooddata/lcm/actions/import_object_collections.rb +17 -0
- data/lib/gooddata/lcm/actions/synchronize_clients.rb +9 -1
- data/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb +3 -3
- data/lib/gooddata/models/model.rb +1 -1
- data/lib/gooddata/models/project.rb +2 -1
- data/lib/gooddata/models/segment.rb +3 -2
- data/lib/gooddata/models/user_filters/user_filter_builder.rb +1 -1
- data/lib/gooddata/rest/client.rb +6 -1
- data/lib/gooddata/rest/connection.rb +13 -4
- metadata +92 -91
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 72ae4a4589edafb5e1b47e9fc62742f94f2b345507eca1db4d9ebad2e1ac6169
|
4
|
+
data.tar.gz: 27ced931546ce5d069e34457c9eb421b7728bffab2478ff90cc69545b9d792c7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6a815c2f8312242265fd0bfd91c3f79ffd941f10befa23c9ea342bcf9c4e7ffd891126191bcbadc07ff6b319add8aa58efdd34244d7b79bb612e938aa90701ee
|
7
|
+
data.tar.gz: c624292d7984a608c05e0d1944bb42bb92aeb082930500e465e919e0a0c81bde7d185fe09b44fc8616c7e3d27a6d46e0113d328dbec96e77fdf1395941657eaf
|
data/.gdc-ii-config.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
microservices:
|
2
2
|
lcm-bricks:
|
3
3
|
docker:
|
4
|
-
dockerfile: './
|
4
|
+
dockerfile: './DockerfileOldImage'
|
5
5
|
# specifies whether promote image to M3 image tag when release LCM. Default always promote image. Don't remote
|
6
6
|
# image when promoteImage configure false
|
7
7
|
promoteImage: false
|
data/.github/workflows/build.yml
CHANGED
@@ -9,7 +9,7 @@ jobs:
|
|
9
9
|
- uses: actions/checkout@v4
|
10
10
|
- uses: ruby/setup-ruby@v1
|
11
11
|
with:
|
12
|
-
ruby-version: jruby-9.4.1
|
12
|
+
ruby-version: jruby-9.4.12.1
|
13
13
|
bundler-cache: true
|
14
14
|
- uses: actions/setup-java@v4
|
15
15
|
with:
|
@@ -41,7 +41,7 @@ jobs:
|
|
41
41
|
needs: [jruby-gem-release, rubygems-release]
|
42
42
|
strategy:
|
43
43
|
matrix:
|
44
|
-
ruby-version: [3.2.1, jruby-9.4.1]
|
44
|
+
ruby-version: [3.2.1, jruby-9.4.12.1]
|
45
45
|
steps:
|
46
46
|
- uses: actions/checkout@v4
|
47
47
|
- uses: ruby/setup-ruby@v1
|
@@ -9,7 +9,7 @@ jobs:
|
|
9
9
|
runs-on: ubuntu-22.04
|
10
10
|
strategy:
|
11
11
|
matrix:
|
12
|
-
ruby-version: [2.5, 2.6, 2.7.6, 3.2.1, jruby-9.2.21, jruby-9.3.14, jruby-9.4.1]
|
12
|
+
ruby-version: [2.5, 2.6, 2.7.6, 3.2.1, jruby-9.2.21, jruby-9.3.14, jruby-9.4.12.1]
|
13
13
|
steps:
|
14
14
|
- uses: actions/checkout@v4
|
15
15
|
- uses: ruby/setup-ruby@v1
|
data/CHANGELOG.md
CHANGED
data/Dockerfile
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
FROM 020413372491.dkr.ecr.us-east-1.amazonaws.com/tools/gdc-java-11-jre-centos9:202501070635.05b6a77
|
2
2
|
|
3
3
|
ARG RVM_VERSION=stable
|
4
|
-
ARG JRUBY_VERSION=9.4.1
|
4
|
+
ARG JRUBY_VERSION=9.4.12.1
|
5
5
|
|
6
6
|
LABEL image_name="GDC LCM Bricks"
|
7
7
|
LABEL maintainer="LCM <lcm@gooddata.com>"
|
@@ -9,7 +9,7 @@ LABEL git_repository_url="https://github.com/gooddata/gooddata-ruby/"
|
|
9
9
|
LABEL parent_image="020413372491.dkr.ecr.us-east-1.amazonaws.com/tools/gdc-java-11-jre-centos9:202501070635.05b6a77"
|
10
10
|
|
11
11
|
# which is required by RVM
|
12
|
-
RUN yum install -y which patch make unzip gnupg git maven procps gzip \
|
12
|
+
RUN yum install -y gcc gcc-c++ which patch make unzip gnupg git maven procps gzip \
|
13
13
|
&& yum clean all \
|
14
14
|
&& rm -rf /var/cache/yum
|
15
15
|
|
data/Dockerfile.jruby
CHANGED
data/DockerfileOldImage
ADDED
@@ -0,0 +1,88 @@
|
|
1
|
+
FROM 020413372491.dkr.ecr.us-east-1.amazonaws.com/tools/gdc-java-11-jre-centos9:202501070635.05b6a77
|
2
|
+
|
3
|
+
ARG RVM_VERSION=stable
|
4
|
+
ARG JRUBY_VERSION=9.2.20.0
|
5
|
+
|
6
|
+
LABEL image_name="GDC LCM Bricks"
|
7
|
+
LABEL maintainer="LCM <lcm@gooddata.com>"
|
8
|
+
LABEL git_repository_url="https://github.com/gooddata/gooddata-ruby/"
|
9
|
+
LABEL parent_image="020413372491.dkr.ecr.us-east-1.amazonaws.com/tools/gdc-java-11-jre-centos9:202501070635.05b6a77"
|
10
|
+
|
11
|
+
# which is required by RVM
|
12
|
+
RUN yum install -y gcc gcc-c++ diffutils curl-minimal which patch make git maven procps \
|
13
|
+
&& yum clean all \
|
14
|
+
&& rm -rf /var/cache/yum
|
15
|
+
|
16
|
+
# Install + verify RVM with gpg (https://rvm.io/rvm/security)
|
17
|
+
RUN curl -sSL https://rvm.io/mpapis.asc | gpg --import - \
|
18
|
+
&& curl -sSL https://rvm.io/pkuczynski.asc | gpg2 --import - \
|
19
|
+
&& curl -sSL https://get.rvm.io | bash -s ${RVM_VERSION} \
|
20
|
+
&& curl -sSL https://raw.githubusercontent.com/rvm/rvm/${RVM_VERSION}/binscripts/rvm-installer -o rvm-installer \
|
21
|
+
&& curl -sSL https://raw.githubusercontent.com/rvm/rvm/${RVM_VERSION}/binscripts/rvm-installer.asc -o rvm-installer.asc \
|
22
|
+
&& gpg2 --verify rvm-installer.asc rvm-installer \
|
23
|
+
&& bash rvm-installer \
|
24
|
+
&& rm rvm-installer rvm-installer.asc \
|
25
|
+
&& echo "bundler" >> /usr/local/rvm/gemsets/global.gems \
|
26
|
+
&& echo "rvm_silence_path_mismatch_check_flag=1" >> /etc/rvmrc \
|
27
|
+
&& echo "install: --no-document" > /etc/gemrc
|
28
|
+
|
29
|
+
# Switch to a bash login shell to allow simple 'rvm' in RUN commands
|
30
|
+
SHELL ["/bin/bash", "-l", "-c"]
|
31
|
+
|
32
|
+
RUN rvm install jruby-${JRUBY_VERSION} \
|
33
|
+
&& gem update --system 3.3.26 \
|
34
|
+
&& gem install bundler -v 2.3.27 \
|
35
|
+
&& gem install rake -v 11.3.0
|
36
|
+
|
37
|
+
WORKDIR /src
|
38
|
+
|
39
|
+
RUN groupadd -g 48 apache \
|
40
|
+
&& groupadd -g 65065 gdcshare \
|
41
|
+
&& useradd -u 48 -m --no-log-init -r -g apache -G rvm apache \
|
42
|
+
&& usermod -a -G gdcshare apache \
|
43
|
+
&& chown apache: /src
|
44
|
+
|
45
|
+
USER apache
|
46
|
+
|
47
|
+
ADD ./bin ./bin
|
48
|
+
ADD --chown=apache:apache ./ci ./ci
|
49
|
+
ADD --chown=apache:apache ./lib ./lib
|
50
|
+
ADD ./SDK_VERSION .
|
51
|
+
ADD ./VERSION .
|
52
|
+
ADD ./Gemfile .
|
53
|
+
ADD ./gooddata.gemspec .
|
54
|
+
|
55
|
+
#build redshift dependencies
|
56
|
+
RUN mvn -f ci/redshift/pom.xml clean install -P binary-packaging
|
57
|
+
RUN cp -rf ci/redshift/target/*.jar ./lib/gooddata/cloud_resources/redshift/drivers/
|
58
|
+
|
59
|
+
#build snowflake dependencies
|
60
|
+
RUN mvn -f ci/snowflake/pom.xml clean install -P binary-packaging
|
61
|
+
RUN cp -rf ci/snowflake/target/*.jar ./lib/gooddata/cloud_resources/snowflake/drivers/
|
62
|
+
|
63
|
+
#build bigquery dependencies
|
64
|
+
RUN mvn -f ci/bigquery/pom.xml clean install -P binary-packaging
|
65
|
+
RUN cp -rf ci/bigquery/target/*.jar ./lib/gooddata/cloud_resources/bigquery/drivers/
|
66
|
+
|
67
|
+
#build postgresql dependencies
|
68
|
+
RUN mvn -f ci/postgresql/pom.xml clean install -P binary-packaging
|
69
|
+
RUN cp -rf ci/postgresql/target/*.jar ./lib/gooddata/cloud_resources/postgresql/drivers/
|
70
|
+
|
71
|
+
#build mssql dependencies
|
72
|
+
RUN mvn -f ci/mssql/pom.xml clean install -P binary-packaging
|
73
|
+
RUN cp -rf ci/mssql/target/*.jar ./lib/gooddata/cloud_resources/mssql/drivers/
|
74
|
+
|
75
|
+
#build mysql dependencies
|
76
|
+
RUN mvn -f ci/mysql/pom.xml clean install -P binary-packaging
|
77
|
+
RUN cp -rf ci/mysql/target/*.jar ./lib/gooddata/cloud_resources/mysql/drivers/
|
78
|
+
|
79
|
+
RUN bundle install
|
80
|
+
|
81
|
+
ARG GIT_COMMIT=unspecified
|
82
|
+
ARG BRICKS_VERSION=unspecified
|
83
|
+
LABEL git_commit=$GIT_COMMIT
|
84
|
+
LABEL bricks_version=$BRICKS_VERSION
|
85
|
+
|
86
|
+
ENV GOODDATA_RUBY_COMMIT=$GIT_COMMIT
|
87
|
+
|
88
|
+
CMD [ "./bin/help.sh" ]
|
data/SDK_VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.3.
|
1
|
+
2.3.3
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
3.7.
|
1
|
+
3.7.95
|
data/bin/provision.sh
CHANGED
data/bin/release.sh
CHANGED
data/bin/rollout.sh
CHANGED
data/bin/user_filters.sh
CHANGED
data/bin/users.sh
CHANGED
data/ci/snowflake/pom.xml
CHANGED
@@ -12,13 +12,23 @@
|
|
12
12
|
<dependency>
|
13
13
|
<groupId>net.snowflake</groupId>
|
14
14
|
<artifactId>snowflake-jdbc</artifactId>
|
15
|
-
<version>3.
|
15
|
+
<version>3.24.2</version>
|
16
16
|
</dependency>
|
17
17
|
<dependency>
|
18
18
|
<groupId>org.slf4j</groupId>
|
19
19
|
<artifactId>slf4j-api</artifactId>
|
20
20
|
<version>1.7.2</version>
|
21
21
|
</dependency>
|
22
|
+
<dependency>
|
23
|
+
<groupId>org.bouncycastle</groupId>
|
24
|
+
<artifactId>bcprov-jdk18on</artifactId>
|
25
|
+
<version>1.80</version>
|
26
|
+
</dependency>
|
27
|
+
<dependency>
|
28
|
+
<groupId>org.bouncycastle</groupId>
|
29
|
+
<artifactId>bcpkix-jdk18on</artifactId>
|
30
|
+
<version>1.80</version>
|
31
|
+
</dependency>
|
22
32
|
</dependencies>
|
23
33
|
|
24
34
|
<profiles>
|
data/gooddata.gemspec
CHANGED
@@ -27,16 +27,33 @@ Gem::Specification.new do |s|
|
|
27
27
|
s.require_paths = ['lib']
|
28
28
|
# s.add_development_dependency 'bundler', "~> 1.14"
|
29
29
|
s.add_development_dependency 'license_finder', '~> 2.0'
|
30
|
-
s.add_development_dependency 'rake', '~> 13.0'
|
31
30
|
s.add_development_dependency 'redcarpet', '~> 3.1' if RUBY_PLATFORM != 'java'
|
32
|
-
s.add_development_dependency 'rspec', '~> 3.12.0'
|
33
|
-
s.add_development_dependency 'rspec-expectations', '~> 3.12'
|
34
|
-
s.add_development_dependency 'rspec_junit_formatter', '~> 0.6.0'
|
35
31
|
if RUBY_VERSION >= '2.6'
|
32
|
+
s.add_development_dependency 'rake', '~> 13.0'
|
36
33
|
s.add_development_dependency 'rubocop', '>= 1.28'
|
37
34
|
s.add_development_dependency 'rubocop-ast', '>= 1.24.1', '<= 1.42.0'
|
35
|
+
s.add_development_dependency 'rspec', '~> 3.12.0'
|
36
|
+
s.add_development_dependency 'rspec-expectations', '~> 3.12'
|
37
|
+
s.add_development_dependency 'rspec_junit_formatter', '~> 0.6.0'
|
38
|
+
|
39
|
+
s.add_dependency 'azure-storage-blob', '~> 2.0'
|
40
|
+
s.add_dependency 'nokogiri', '~> 1', '>= 1.10.8'
|
41
|
+
s.add_dependency 'json_pure', '~> 2.6'
|
42
|
+
s.add_dependency 'restforce', '>= 2.4'
|
43
|
+
s.add_dependency 'rubyzip'
|
38
44
|
else
|
45
|
+
s.add_development_dependency 'rake', '~> 11.1'
|
39
46
|
s.add_development_dependency 'rubocop', '~> 0.81'
|
47
|
+
s.add_development_dependency 'rspec', '~> 3.5.0'
|
48
|
+
s.add_development_dependency 'rspec-expectations', '~> 3.5'
|
49
|
+
s.add_development_dependency 'rspec_junit_formatter', '~> 0.3.0'
|
50
|
+
|
51
|
+
s.add_dependency 'azure-storage-blob', '~> 1.1.0'
|
52
|
+
s.add_dependency 'nokogiri', '~> 1.10.0'
|
53
|
+
s.add_dependency 'json_pure', '~> 1.8'
|
54
|
+
s.add_dependency 'restforce', '>= 2.4', '< 4.0'
|
55
|
+
s.add_dependency 'rubyzip', '~> 1.2', '>= 1.2.1'
|
56
|
+
s.add_dependency 'unf', '~> 0.1.4'
|
40
57
|
end
|
41
58
|
s.add_development_dependency 'simplecov', '~> 0.12'
|
42
59
|
s.add_development_dependency 'webmock', '~> 2.3.1'
|
@@ -75,21 +92,16 @@ Gem::Specification.new do |s|
|
|
75
92
|
else
|
76
93
|
s.add_dependency 'docile', '> 1.1', '< 1.4.0'
|
77
94
|
end
|
78
|
-
s.add_dependency 'azure-storage-blob', '~> 2.0'
|
79
|
-
s.add_dependency 'nokogiri', '~> 1', '>= 1.10.8'
|
80
95
|
s.add_dependency 'gli', '~> 2.15'
|
81
96
|
s.add_dependency 'gooddata_datawarehouse', '~> 0.0.12' if RUBY_PLATFORM == 'java'
|
82
97
|
s.add_dependency 'highline', '= 2.0.0.pre.develop.14'
|
83
|
-
s.add_dependency 'json_pure', '~> 2.6'
|
84
98
|
s.add_dependency 'multi_json', '~> 1.12'
|
85
99
|
s.add_dependency 'parseconfig', '~> 1.0'
|
86
100
|
s.add_dependency 'path_expander', '< 1.1.2'
|
87
101
|
s.add_dependency 'pmap', '~> 1.1'
|
88
102
|
s.add_dependency 'sequel', '< 5.72.0'
|
89
103
|
s.add_dependency 'remote_syslog_logger', '~> 1.0.3'
|
90
|
-
s.add_dependency 'restforce', '>= 2.4'
|
91
104
|
s.add_dependency 'rest-client', '~> 2.0'
|
92
|
-
s.add_dependency 'rubyzip'
|
93
105
|
s.add_dependency 'terminal-table', '~> 1.7'
|
94
106
|
s.add_dependency 'thread_safe'
|
95
107
|
s.add_dependency 'backports'
|
@@ -15,6 +15,14 @@ Dir.glob(base + 'drivers/*.jar').each do |file|
|
|
15
15
|
require file unless file.start_with?('lcm-snowflake-driver')
|
16
16
|
end
|
17
17
|
|
18
|
+
java_import 'java.io.StringReader'
|
19
|
+
java_import 'org.bouncycastle.openssl.PEMParser'
|
20
|
+
java_import 'org.bouncycastle.jce.provider.BouncyCastleProvider'
|
21
|
+
java_import 'org.bouncycastle.asn1.pkcs.PrivateKeyInfo'
|
22
|
+
java_import 'org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo'
|
23
|
+
java_import 'org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder'
|
24
|
+
java_import 'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter'
|
25
|
+
|
18
26
|
module GoodData
|
19
27
|
module CloudResources
|
20
28
|
class SnowflakeClient < CloudResourceClient
|
@@ -75,11 +83,19 @@ module GoodData
|
|
75
83
|
GoodData.logger.info "Setting up connection to Snowflake #{@url}"
|
76
84
|
|
77
85
|
prop = java.util.Properties.new
|
78
|
-
prop.setProperty('user', @authentication['basic']['userName'])
|
79
|
-
prop.setProperty('password', @authentication['basic']['password'])
|
80
86
|
prop.setProperty('schema', @schema)
|
81
87
|
prop.setProperty('warehouse', @warehouse)
|
82
88
|
prop.setProperty('db', @database)
|
89
|
+
|
90
|
+
if @authentication['keyPair']
|
91
|
+
prop.setProperty('user', @authentication['keyPair']['userName'])
|
92
|
+
private_key_str = build_private_key(@authentication['keyPair']['privateKey'], @authentication['keyPair']['passPhrase'])
|
93
|
+
prop.setProperty('private_key_base64', private_key_str)
|
94
|
+
else
|
95
|
+
prop.setProperty('user', @authentication['basic']['userName'])
|
96
|
+
prop.setProperty('password', @authentication['basic']['password'])
|
97
|
+
end
|
98
|
+
|
83
99
|
# Add JDBC_QUERY_RESULT_FORMAT parameter to fix unsafe memory issue of Snowflake JDBC driver
|
84
100
|
prop.setProperty('JDBC_QUERY_RESULT_FORMAT', 'JSON')
|
85
101
|
|
@@ -99,6 +115,42 @@ module GoodData
|
|
99
115
|
|
100
116
|
url
|
101
117
|
end
|
118
|
+
|
119
|
+
private
|
120
|
+
|
121
|
+
def build_private_key(private_key_string, pass_phrase)
|
122
|
+
java.security.Security.removeProvider("BC")
|
123
|
+
java.security.Security.addProvider(BouncyCastleProvider.new)
|
124
|
+
|
125
|
+
begin
|
126
|
+
pem_parser = PEMParser.new(StringReader.new(private_key_string))
|
127
|
+
pem_object = pem_parser.readObject
|
128
|
+
|
129
|
+
if pem_object.is_a?(PKCS8EncryptedPrivateKeyInfo)
|
130
|
+
builder = JceOpenSSLPKCS8DecryptorProviderBuilder.new
|
131
|
+
decryptor = builder.build(pass_phrase.to_java.to_char_array)
|
132
|
+
private_key_info = pem_object.decryptPrivateKeyInfo(decryptor)
|
133
|
+
elsif pem_object.is_a?(PrivateKeyInfo)
|
134
|
+
private_key_info = pem_object
|
135
|
+
end
|
136
|
+
|
137
|
+
ensure
|
138
|
+
pem_parser&.close
|
139
|
+
end
|
140
|
+
|
141
|
+
converter = JcaPEMKeyConverter.new
|
142
|
+
private_key = converter.getPrivateKey(private_key_info)
|
143
|
+
pem_str = convert_private_key(private_key)
|
144
|
+
java.util.Base64.getEncoder.encodeToString(pem_str.encode('UTF-8').bytes)
|
145
|
+
end
|
146
|
+
|
147
|
+
def convert_private_key(private_key)
|
148
|
+
pem = "-----BEGIN PRIVATE KEY-----\n"
|
149
|
+
encoder = java.util.Base64.getMimeEncoder(64, "\n".to_java_bytes)
|
150
|
+
base64 = encoder.encodeToString(private_key.getEncoded)
|
151
|
+
"#{pem}#{base64}\n-----END PRIVATE KEY-----"
|
152
|
+
end
|
153
|
+
|
102
154
|
end
|
103
155
|
end
|
104
156
|
end
|
@@ -29,6 +29,7 @@ module GoodData
|
|
29
29
|
def call(params)
|
30
30
|
results = []
|
31
31
|
|
32
|
+
GoodData.logger.info 'Starting ImportObjectCollections action'
|
32
33
|
client = params.gdc_gd_client
|
33
34
|
development_client = params.development_client
|
34
35
|
number_of_threads = Integer(params.number_of_threads || '8')
|
@@ -45,7 +46,9 @@ module GoodData
|
|
45
46
|
to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
|
46
47
|
|
47
48
|
if transfer_uris.any?
|
49
|
+
logging_data(from, pid, transfer_uris, true)
|
48
50
|
from_project.partial_md_export(transfer_uris, project: to_project)
|
51
|
+
logging_data(from, pid, transfer_uris, false)
|
49
52
|
end
|
50
53
|
|
51
54
|
results << {
|
@@ -58,6 +61,20 @@ module GoodData
|
|
58
61
|
|
59
62
|
results
|
60
63
|
end
|
64
|
+
|
65
|
+
private
|
66
|
+
|
67
|
+
def logging_data(from_project, to_project, transfer_uris, start_action)
|
68
|
+
if start_action
|
69
|
+
# Logging to execution log
|
70
|
+
GoodData.logger.info "Starting import objects, from_project: #{from_project}, to_project: #{to_project}"
|
71
|
+
# Logging to Splunk log
|
72
|
+
GoodData.gd_logger.info "Starting import objects, action=objects_import, from_project=#{from_project}, to_project=#{to_project}, transfer_uris=#{transfer_uris}"
|
73
|
+
else
|
74
|
+
GoodData.logger.info "Success import objects, from_project: #{from_project}, to_project: #{to_project}"
|
75
|
+
GoodData.gd_logger.info "Success import objects, action=objects_import, from_project=#{from_project}, to_project=#{to_project}"
|
76
|
+
end
|
77
|
+
end
|
61
78
|
end
|
62
79
|
end
|
63
80
|
end
|
@@ -50,6 +50,9 @@ module GoodData
|
|
50
50
|
|
51
51
|
description 'Sync failed list'
|
52
52
|
param :sync_failed_list, instance_of(Type::HashType), required: false
|
53
|
+
|
54
|
+
description 'Synchronize clients time limit'
|
55
|
+
param :sync_clients_timeout, instance_of(Type::StringType), required: false
|
53
56
|
end
|
54
57
|
|
55
58
|
RESULT_HEADER = [
|
@@ -61,6 +64,7 @@ module GoodData
|
|
61
64
|
|
62
65
|
class << self
|
63
66
|
def call(params)
|
67
|
+
GoodData.logger.info 'Starting SynchronizeClients action'
|
64
68
|
client = params.gdc_gd_client
|
65
69
|
|
66
70
|
domain_name = params.organization || params.domain
|
@@ -69,6 +73,8 @@ module GoodData
|
|
69
73
|
data_product = params.data_product
|
70
74
|
domain_segments = domain.segments(:all, data_product)
|
71
75
|
keep_only_previous_masters_count = Integer(params.keep_only_previous_masters_count || "-1")
|
76
|
+
sync_clients_options = {}
|
77
|
+
sync_clients_options = sync_clients_options.merge(:time_limit => Integer(params.sync_clients_timeout)) if params.sync_clients_timeout
|
72
78
|
|
73
79
|
segments = params.segments.map do |seg|
|
74
80
|
domain_segments.find do |s|
|
@@ -91,7 +97,9 @@ module GoodData
|
|
91
97
|
segment.master_project = master
|
92
98
|
segment.save
|
93
99
|
|
94
|
-
|
100
|
+
GoodData.logger.info "Starting synchronize clients for segment: '#{segment.segment_id}' with master workspace: '#{current_master[:master_project_id]}'"
|
101
|
+
res = segment.synchronize_clients(sync_clients_options)
|
102
|
+
GoodData.logger.info "Finish synchronize clients for segment: '#{segment.segment_id}'"
|
95
103
|
|
96
104
|
sync_result = res.json['synchronizationResult']
|
97
105
|
failed_count = sync_result['failedClients']['count']
|
@@ -179,10 +179,10 @@ module GoodData
|
|
179
179
|
hidden_params_for_this_client_schedule_name = hidden_params_for_this_client[schedule_name]
|
180
180
|
|
181
181
|
schedule.update_params(schedule_additional_params) if schedule_additional_params
|
182
|
-
schedule.update_params(
|
183
|
-
schedule.update_params(
|
182
|
+
schedule.update_params(params_for_all_schedules_in_all_projects) if params_for_all_schedules_in_all_projects
|
183
|
+
schedule.update_params(params_for_all_projects_schedule_name) if params_for_all_projects_schedule_name
|
184
184
|
schedule.update_params(params_for_all_schedules_in_this_client) if params_for_all_schedules_in_this_client
|
185
|
-
schedule.update_params(
|
185
|
+
schedule.update_params(params_for_this_client_schedule_name) if params_for_this_client_schedule_name
|
186
186
|
|
187
187
|
schedule.update_hidden_params(schedule_additional_hidden_params) if schedule_additional_hidden_params
|
188
188
|
schedule.update_hidden_params(hidden_params_for_all_schedules_in_all_projects) if hidden_params_for_all_schedules_in_all_projects
|
@@ -166,7 +166,7 @@ module GoodData
|
|
166
166
|
|
167
167
|
dir = Dir.mktmpdir
|
168
168
|
begin
|
169
|
-
Zip::File.open("#{dir}/upload.zip",
|
169
|
+
Zip::File.open("#{dir}/upload.zip", create: true) do |zip|
|
170
170
|
# TODO: make sure schema columns match CSV column names
|
171
171
|
zip.get_output_stream('upload_info.json') { |f| f.puts JSON.pretty_generate(manifest) }
|
172
172
|
|
@@ -1437,7 +1437,8 @@ module GoodData
|
|
1437
1437
|
export_uri = "/gdc/md/#{pid}/maintenance/partialmdexport"
|
1438
1438
|
GoodData.gd_logger.info("Project export action=objects_export, project_id=#{pid}, uri=#{export_uri}, export_status=start, export_objs=#{export_payload}") if GoodData.gd_logger
|
1439
1439
|
|
1440
|
-
|
1440
|
+
# Export api will take time to finish So increasing timeout during calling the api
|
1441
|
+
result = client.post(export_uri, export_payload, :timeout => 10)
|
1441
1442
|
polling_url = result['partialMDArtifact']['status']['uri']
|
1442
1443
|
token = result['partialMDArtifact']['token']
|
1443
1444
|
GoodData.gd_logger.info("Project export action=objects_export, project_id=#{pid}, uri=#{polling_url}, export_status=polling") if GoodData.gd_logger
|
@@ -193,13 +193,14 @@ module GoodData
|
|
193
193
|
|
194
194
|
# Runs async process that walks through segments and provisions projects if necessary.
|
195
195
|
#
|
196
|
+
# @param options [Hash] Options
|
196
197
|
# @return [Array] Returns array of results
|
197
|
-
def synchronize_clients
|
198
|
+
def synchronize_clients(options = {})
|
198
199
|
sync_uri = SYNCHRONIZE_URI % [domain.obj_id, data_product.data_product_id, id]
|
199
200
|
res = client.post sync_uri, nil
|
200
201
|
|
201
202
|
# wait until the instance is created
|
202
|
-
res = client.poll_on_response(res['asyncTask']['links']['poll'], :sleep_interval => 1) do |r|
|
203
|
+
res = client.poll_on_response(res['asyncTask']['links']['poll'], options.merge(:sleep_interval => 1)) do |r|
|
203
204
|
r['synchronizationResult'].nil?
|
204
205
|
end
|
205
206
|
|
data/lib/gooddata/rest/client.rb
CHANGED
@@ -25,7 +25,7 @@ module GoodData
|
|
25
25
|
# Constants
|
26
26
|
#################################
|
27
27
|
DEFAULT_CONNECTION_IMPLEMENTATION = GoodData::Rest::Connection
|
28
|
-
DEFAULT_SLEEP_INTERVAL = 10
|
28
|
+
DEFAULT_SLEEP_INTERVAL = 10 # 10 seconds
|
29
29
|
DEFAULT_POLL_TIME_LIMIT = 5 * 60 * 60 # 5 hours
|
30
30
|
|
31
31
|
#################################
|
@@ -353,6 +353,11 @@ module GoodData
|
|
353
353
|
end
|
354
354
|
sleep retry_time
|
355
355
|
retry_time *= GoodData::Rest::Connection::RETRY_TIME_COEFFICIENT
|
356
|
+
# Polling response will wait result from calling APIs. So don't need wait a long time for each polling
|
357
|
+
if retry_time > DEFAULT_SLEEP_INTERVAL
|
358
|
+
retry_time = DEFAULT_SLEEP_INTERVAL
|
359
|
+
end
|
360
|
+
|
356
361
|
GoodData::Rest::Client.retryable(:tries => Helpers::GD_MAX_RETRY, :refresh_token => proc { connection.refresh_token }) do
|
357
362
|
response = get(link, process: process)
|
358
363
|
end
|
@@ -58,6 +58,8 @@ module GoodData
|
|
58
58
|
RETRYABLE_ERRORS = [
|
59
59
|
Net::HTTPBadResponse,
|
60
60
|
RestClient::InternalServerError,
|
61
|
+
RestClient::Exceptions::OpenTimeout,
|
62
|
+
RestClient::Exceptions::ReadTimeout,
|
61
63
|
RestClient::RequestTimeout,
|
62
64
|
RestClient::MethodNotAllowed,
|
63
65
|
SystemCallError,
|
@@ -110,30 +112,37 @@ module GoodData
|
|
110
112
|
raise e unless options[:refresh_token]
|
111
113
|
raise e if options[:dont_reauth]
|
112
114
|
|
115
|
+
logging_retry_error(e, retry_time)
|
113
116
|
options[:refresh_token].call # (dont_reauth: true)
|
114
117
|
if (retries -= 1) > 0
|
115
118
|
retry
|
116
119
|
else
|
117
|
-
|
120
|
+
fail e
|
118
121
|
end
|
119
122
|
rescue RestClient::TooManyRequests, RestClient::ServiceUnavailable, *retry_exception => e
|
123
|
+
logging_retry_error(e, retry_time)
|
120
124
|
sleep retry_time
|
125
|
+
if OpenSSL::SSL.const_defined?(:SSLErrorWaitReadable) && e.is_a?(OpenSSL::SSL::SSLErrorWaitReadable)
|
126
|
+
GoodData.gd_logger.warn "Refresh token status=start retry_time=#{retry_time}. Error: #{e.message}"
|
127
|
+
options[:refresh_token].call # (dont_reauth: true)
|
128
|
+
GoodData.gd_logger.warn "Refresh token status=success retry_time=#{retry_time}. Error: #{e.message}"
|
129
|
+
end
|
130
|
+
|
121
131
|
# Total 10 retry requests with 1.5 coefficent should take ~ 2 mins to finish
|
122
132
|
if (retries -= 1) > 0
|
123
133
|
retry_time *= RETRY_TIME_COEFFICIENT
|
124
134
|
retry
|
125
135
|
else
|
126
|
-
|
136
|
+
fail e
|
127
137
|
end
|
128
138
|
end
|
129
139
|
yield
|
130
140
|
end
|
131
141
|
|
132
|
-
def
|
142
|
+
def logging_retry_error(e, retry_time)
|
133
143
|
error_message = "#{e.message}, retrying in #{retry_time} seconds"
|
134
144
|
GoodData.logger.warn error_message
|
135
145
|
GoodData.gd_logger.warn error_message
|
136
|
-
fail e
|
137
146
|
end
|
138
147
|
end
|
139
148
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gooddata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.3.
|
4
|
+
version: 2.3.3
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Pavel Kolesnikov
|
@@ -13,7 +13,7 @@ authors:
|
|
13
13
|
- Jakub Mahnert
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
|
-
date: 2025-
|
16
|
+
date: 2025-08-06 00:00:00.000000000 Z
|
17
17
|
dependencies:
|
18
18
|
- !ruby/object:Gem::Dependency
|
19
19
|
name: license_finder
|
@@ -43,6 +43,40 @@ dependencies:
|
|
43
43
|
- - "~>"
|
44
44
|
- !ruby/object:Gem::Version
|
45
45
|
version: '13.0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: rubocop
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
requirements:
|
50
|
+
- - ">="
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: '1.28'
|
53
|
+
type: :development
|
54
|
+
prerelease: false
|
55
|
+
version_requirements: !ruby/object:Gem::Requirement
|
56
|
+
requirements:
|
57
|
+
- - ">="
|
58
|
+
- !ruby/object:Gem::Version
|
59
|
+
version: '1.28'
|
60
|
+
- !ruby/object:Gem::Dependency
|
61
|
+
name: rubocop-ast
|
62
|
+
requirement: !ruby/object:Gem::Requirement
|
63
|
+
requirements:
|
64
|
+
- - ">="
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: 1.24.1
|
67
|
+
- - "<="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: 1.42.0
|
70
|
+
type: :development
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - ">="
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: 1.24.1
|
77
|
+
- - "<="
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: 1.42.0
|
46
80
|
- !ruby/object:Gem::Dependency
|
47
81
|
name: rspec
|
48
82
|
requirement: !ruby/object:Gem::Requirement
|
@@ -86,39 +120,81 @@ dependencies:
|
|
86
120
|
- !ruby/object:Gem::Version
|
87
121
|
version: 0.6.0
|
88
122
|
- !ruby/object:Gem::Dependency
|
89
|
-
name:
|
123
|
+
name: azure-storage-blob
|
90
124
|
requirement: !ruby/object:Gem::Requirement
|
91
125
|
requirements:
|
126
|
+
- - "~>"
|
127
|
+
- !ruby/object:Gem::Version
|
128
|
+
version: '2.0'
|
129
|
+
type: :runtime
|
130
|
+
prerelease: false
|
131
|
+
version_requirements: !ruby/object:Gem::Requirement
|
132
|
+
requirements:
|
133
|
+
- - "~>"
|
134
|
+
- !ruby/object:Gem::Version
|
135
|
+
version: '2.0'
|
136
|
+
- !ruby/object:Gem::Dependency
|
137
|
+
name: nokogiri
|
138
|
+
requirement: !ruby/object:Gem::Requirement
|
139
|
+
requirements:
|
140
|
+
- - "~>"
|
141
|
+
- !ruby/object:Gem::Version
|
142
|
+
version: '1'
|
92
143
|
- - ">="
|
93
144
|
- !ruby/object:Gem::Version
|
94
|
-
version:
|
95
|
-
type: :
|
145
|
+
version: 1.10.8
|
146
|
+
type: :runtime
|
96
147
|
prerelease: false
|
97
148
|
version_requirements: !ruby/object:Gem::Requirement
|
98
149
|
requirements:
|
150
|
+
- - "~>"
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '1'
|
99
153
|
- - ">="
|
100
154
|
- !ruby/object:Gem::Version
|
101
|
-
version:
|
155
|
+
version: 1.10.8
|
102
156
|
- !ruby/object:Gem::Dependency
|
103
|
-
name:
|
157
|
+
name: json_pure
|
104
158
|
requirement: !ruby/object:Gem::Requirement
|
105
159
|
requirements:
|
106
|
-
- - "
|
160
|
+
- - "~>"
|
107
161
|
- !ruby/object:Gem::Version
|
108
|
-
version:
|
109
|
-
|
162
|
+
version: '2.6'
|
163
|
+
type: :runtime
|
164
|
+
prerelease: false
|
165
|
+
version_requirements: !ruby/object:Gem::Requirement
|
166
|
+
requirements:
|
167
|
+
- - "~>"
|
110
168
|
- !ruby/object:Gem::Version
|
111
|
-
version:
|
112
|
-
|
169
|
+
version: '2.6'
|
170
|
+
- !ruby/object:Gem::Dependency
|
171
|
+
name: restforce
|
172
|
+
requirement: !ruby/object:Gem::Requirement
|
173
|
+
requirements:
|
174
|
+
- - ">="
|
175
|
+
- !ruby/object:Gem::Version
|
176
|
+
version: '2.4'
|
177
|
+
type: :runtime
|
113
178
|
prerelease: false
|
114
179
|
version_requirements: !ruby/object:Gem::Requirement
|
115
180
|
requirements:
|
116
181
|
- - ">="
|
117
182
|
- !ruby/object:Gem::Version
|
118
|
-
version:
|
119
|
-
|
183
|
+
version: '2.4'
|
184
|
+
- !ruby/object:Gem::Dependency
|
185
|
+
name: rubyzip
|
186
|
+
requirement: !ruby/object:Gem::Requirement
|
187
|
+
requirements:
|
188
|
+
- - ">="
|
120
189
|
- !ruby/object:Gem::Version
|
121
|
-
version:
|
190
|
+
version: '0'
|
191
|
+
type: :runtime
|
192
|
+
prerelease: false
|
193
|
+
version_requirements: !ruby/object:Gem::Requirement
|
194
|
+
requirements:
|
195
|
+
- - ">="
|
196
|
+
- !ruby/object:Gem::Version
|
197
|
+
version: '0'
|
122
198
|
- !ruby/object:Gem::Dependency
|
123
199
|
name: simplecov
|
124
200
|
requirement: !ruby/object:Gem::Requirement
|
@@ -301,40 +377,6 @@ dependencies:
|
|
301
377
|
- - "~>"
|
302
378
|
- !ruby/object:Gem::Version
|
303
379
|
version: '1.1'
|
304
|
-
- !ruby/object:Gem::Dependency
|
305
|
-
name: azure-storage-blob
|
306
|
-
requirement: !ruby/object:Gem::Requirement
|
307
|
-
requirements:
|
308
|
-
- - "~>"
|
309
|
-
- !ruby/object:Gem::Version
|
310
|
-
version: '2.0'
|
311
|
-
type: :runtime
|
312
|
-
prerelease: false
|
313
|
-
version_requirements: !ruby/object:Gem::Requirement
|
314
|
-
requirements:
|
315
|
-
- - "~>"
|
316
|
-
- !ruby/object:Gem::Version
|
317
|
-
version: '2.0'
|
318
|
-
- !ruby/object:Gem::Dependency
|
319
|
-
name: nokogiri
|
320
|
-
requirement: !ruby/object:Gem::Requirement
|
321
|
-
requirements:
|
322
|
-
- - "~>"
|
323
|
-
- !ruby/object:Gem::Version
|
324
|
-
version: '1'
|
325
|
-
- - ">="
|
326
|
-
- !ruby/object:Gem::Version
|
327
|
-
version: 1.10.8
|
328
|
-
type: :runtime
|
329
|
-
prerelease: false
|
330
|
-
version_requirements: !ruby/object:Gem::Requirement
|
331
|
-
requirements:
|
332
|
-
- - "~>"
|
333
|
-
- !ruby/object:Gem::Version
|
334
|
-
version: '1'
|
335
|
-
- - ">="
|
336
|
-
- !ruby/object:Gem::Version
|
337
|
-
version: 1.10.8
|
338
380
|
- !ruby/object:Gem::Dependency
|
339
381
|
name: gli
|
340
382
|
requirement: !ruby/object:Gem::Requirement
|
@@ -377,20 +419,6 @@ dependencies:
|
|
377
419
|
- - '='
|
378
420
|
- !ruby/object:Gem::Version
|
379
421
|
version: 2.0.0.pre.develop.14
|
380
|
-
- !ruby/object:Gem::Dependency
|
381
|
-
name: json_pure
|
382
|
-
requirement: !ruby/object:Gem::Requirement
|
383
|
-
requirements:
|
384
|
-
- - "~>"
|
385
|
-
- !ruby/object:Gem::Version
|
386
|
-
version: '2.6'
|
387
|
-
type: :runtime
|
388
|
-
prerelease: false
|
389
|
-
version_requirements: !ruby/object:Gem::Requirement
|
390
|
-
requirements:
|
391
|
-
- - "~>"
|
392
|
-
- !ruby/object:Gem::Version
|
393
|
-
version: '2.6'
|
394
422
|
- !ruby/object:Gem::Dependency
|
395
423
|
name: multi_json
|
396
424
|
requirement: !ruby/object:Gem::Requirement
|
@@ -475,20 +503,6 @@ dependencies:
|
|
475
503
|
- - "~>"
|
476
504
|
- !ruby/object:Gem::Version
|
477
505
|
version: 1.0.3
|
478
|
-
- !ruby/object:Gem::Dependency
|
479
|
-
name: restforce
|
480
|
-
requirement: !ruby/object:Gem::Requirement
|
481
|
-
requirements:
|
482
|
-
- - ">="
|
483
|
-
- !ruby/object:Gem::Version
|
484
|
-
version: '2.4'
|
485
|
-
type: :runtime
|
486
|
-
prerelease: false
|
487
|
-
version_requirements: !ruby/object:Gem::Requirement
|
488
|
-
requirements:
|
489
|
-
- - ">="
|
490
|
-
- !ruby/object:Gem::Version
|
491
|
-
version: '2.4'
|
492
506
|
- !ruby/object:Gem::Dependency
|
493
507
|
name: rest-client
|
494
508
|
requirement: !ruby/object:Gem::Requirement
|
@@ -503,20 +517,6 @@ dependencies:
|
|
503
517
|
- - "~>"
|
504
518
|
- !ruby/object:Gem::Version
|
505
519
|
version: '2.0'
|
506
|
-
- !ruby/object:Gem::Dependency
|
507
|
-
name: rubyzip
|
508
|
-
requirement: !ruby/object:Gem::Requirement
|
509
|
-
requirements:
|
510
|
-
- - ">="
|
511
|
-
- !ruby/object:Gem::Version
|
512
|
-
version: '0'
|
513
|
-
type: :runtime
|
514
|
-
prerelease: false
|
515
|
-
version_requirements: !ruby/object:Gem::Requirement
|
516
|
-
requirements:
|
517
|
-
- - ">="
|
518
|
-
- !ruby/object:Gem::Version
|
519
|
-
version: '0'
|
520
520
|
- !ruby/object:Gem::Dependency
|
521
521
|
name: terminal-table
|
522
522
|
requirement: !ruby/object:Gem::Requirement
|
@@ -617,6 +617,7 @@ files:
|
|
617
617
|
- Dockerfile
|
618
618
|
- Dockerfile.jruby
|
619
619
|
- Dockerfile.ruby
|
620
|
+
- DockerfileOldImage
|
620
621
|
- Gemfile
|
621
622
|
- Guardfile
|
622
623
|
- Jenkinsfile
|