gooddata 2.1.9-java → 2.1.10-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -0
- data/.travis.yml +1 -3
- data/CHANGELOG.md +24 -0
- data/Dockerfile +12 -4
- data/SDK_VERSION +1 -1
- data/VERSION +1 -1
- data/ci/bigquery/pom.xml +54 -0
- data/ci/redshift/pom.xml +73 -0
- data/ci/snowflake/pom.xml +57 -0
- data/lcm.rake +10 -6
- data/lib/gooddata/cloud_resources/bigquery/bigquery_client.rb +86 -0
- data/lib/gooddata/cloud_resources/bigquery/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/redshift/redshift_client.rb +3 -2
- data/lib/gooddata/cloud_resources/snowflake/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +84 -0
- data/lib/gooddata/helpers/data_helper.rb +1 -1
- data/lib/gooddata/lcm/actions/collect_clients.rb +6 -6
- data/lib/gooddata/lcm/actions/collect_dynamic_schedule_params.rb +6 -6
- data/lib/gooddata/lcm/actions/collect_users_brick_users.rb +7 -6
- data/lib/gooddata/lcm/actions/set_master_project.rb +76 -0
- data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +2 -2
- data/lib/gooddata/lcm/actions/synchronize_users.rb +31 -30
- data/lib/gooddata/lcm/lcm2.rb +21 -1
- data/lib/gooddata/models/domain.rb +16 -14
- data/lib/gooddata/models/project.rb +19 -17
- metadata +10 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 00e8b6ecab4ffb8da4f9904c24a4d6fe9e5d4a5144da33a41539a82412494756
|
|
4
|
+
data.tar.gz: f4f0aec0b4d7d38f5a63ee742e5e4eb37104aa91f7bfcbc7e95060eaa98e83b5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 11a8f24cb89b5e2832d58ce34b5e209a90f820c8b50a0b68355d8938dff2bab9bece0c251e5688ddfaa7ef4838ebc74e7ebc975966a67065fde3c0da72851bbe
|
|
7
|
+
data.tar.gz: 2173449f161e26e975f7a46aadc35e4d4b546facaaf67b9ff3a6e43286ecabe287f9cc6962b03e2d49066835a1e5d6b08e15d40c55a79f793d7c7e327658c576
|
data/.rubocop.yml
CHANGED
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
|
@@ -1,4 +1,28 @@
|
|
|
1
1
|
# GoodData Ruby SDK Changelog
|
|
2
|
+
## 2.1.10
|
|
3
|
+
- BUGFIX: TMA-1653 fix performance issue in functions project.users and domain.users
|
|
4
|
+
- BUGFIX: TMA-1643 Don't convert null value to empty string
|
|
5
|
+
- BUGFIX: TMA-1620 Users Brick, sometimes update users are conflicted
|
|
6
|
+
- BUGFIX: TMA-1642 parse csv from input source with case insensitive
|
|
7
|
+
- BUGFIX: TMA-1528: remove CollectSegments and CollectDataProduct for add and remove in users brick
|
|
8
|
+
- FEATURE: TMA-1629 Add parameter "set_master_project" to support reset latest master project
|
|
9
|
+
- FEATURE: TMA-1630 Support Snowflake, BigQuery as input source
|
|
10
|
+
## 2.1.9
|
|
11
|
+
- FEATURE: TMA-1076 support new version api 2
|
|
12
|
+
- BUGFIX: TMA-1637 handle input_source of dynamic params
|
|
13
|
+
- BUGFIX: TMA-1636 Build csv file with force_quotes
|
|
14
|
+
- FEATURE: TMA-1614 Support redshift input source
|
|
15
|
+
- FEATURE: TMA-1259 Start using dataproduct in NFS release table
|
|
16
|
+
- FEATURE: MSF-16455 support yellow WARNING status into RubySDK
|
|
17
|
+
- CONFIG: TMA-1625 update version lock for test docker images
|
|
18
|
+
- BUGFIX: TMA-1602 User filter brick failed - K8s bricks don't show error properly
|
|
19
|
+
- BUGFIX: TMA-1593 Increase java heap space during execute bricks
|
|
20
|
+
- BUGFIX: TMA-1558 K8s bricks don't show error when config invalid
|
|
21
|
+
- BUGFIX: TMA-1596 The error propagated from a LCM brick is shown multiple times in the log
|
|
22
|
+
- BUGFIX: TMA-1582 show synchronize ldm mode is running
|
|
23
|
+
- FEATURE: TMA-1588 support schedule param include deprecated
|
|
24
|
+
- FEATURE: TMA-1597 Logging lcm execution result
|
|
25
|
+
|
|
2
26
|
## 2.1.8
|
|
3
27
|
- FEATURE: TMA-1604 Upgrade Restforce version to 3.x
|
|
4
28
|
|
data/Dockerfile
CHANGED
|
@@ -43,16 +43,24 @@ RUN groupadd -g 48 apache \
|
|
|
43
43
|
USER apache
|
|
44
44
|
|
|
45
45
|
ADD ./bin ./bin
|
|
46
|
+
ADD --chown=apache:apache ./ci ./ci
|
|
46
47
|
ADD --chown=apache:apache ./lib ./lib
|
|
47
48
|
ADD ./SDK_VERSION .
|
|
48
49
|
ADD ./VERSION .
|
|
49
50
|
ADD ./Gemfile .
|
|
50
51
|
ADD ./gooddata.gemspec .
|
|
51
52
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
RUN
|
|
55
|
-
|
|
53
|
+
#build redshift dependencies
|
|
54
|
+
RUN mvn -f ci/redshift/pom.xml clean install -P binary-packaging
|
|
55
|
+
RUN cp -rf ci/redshift/target/*.jar ./lib/gooddata/cloud_resources/redshift/drivers/
|
|
56
|
+
|
|
57
|
+
#build snowflake dependencies
|
|
58
|
+
RUN mvn -f ci/snowflake/pom.xml clean install -P binary-packaging
|
|
59
|
+
RUN cp -rf ci/snowflake/target/*.jar ./lib/gooddata/cloud_resources/snowflake/drivers/
|
|
60
|
+
|
|
61
|
+
#build bigquery dependencies
|
|
62
|
+
RUN mvn -f ci/bigquery/pom.xml clean install -P binary-packaging
|
|
63
|
+
RUN cp -rf ci/bigquery/target/*.jar ./lib/gooddata/cloud_resources/bigquery/drivers/
|
|
56
64
|
|
|
57
65
|
RUN bundle install
|
|
58
66
|
|
data/SDK_VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
2.1.
|
|
1
|
+
2.1.10
|
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
3.7.
|
|
1
|
+
3.7.17
|
data/ci/bigquery/pom.xml
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
|
3
|
+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
4
|
+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
5
|
+
<modelVersion>4.0.0</modelVersion>
|
|
6
|
+
|
|
7
|
+
<groupId>com.gooddata.lcm</groupId>
|
|
8
|
+
<artifactId>lcm-bigquery-driver</artifactId>
|
|
9
|
+
<version>1.0-SNAPSHOT</version>
|
|
10
|
+
|
|
11
|
+
<dependencies>
|
|
12
|
+
<dependency>
|
|
13
|
+
<groupId>com.google.auth</groupId>
|
|
14
|
+
<artifactId>google-auth-library-oauth2-http</artifactId>
|
|
15
|
+
<version>0.16.2</version>
|
|
16
|
+
</dependency>
|
|
17
|
+
<dependency>
|
|
18
|
+
<groupId>org.apache.commons</groupId>
|
|
19
|
+
<artifactId>commons-text</artifactId>
|
|
20
|
+
<version>1.7</version>
|
|
21
|
+
</dependency>
|
|
22
|
+
<dependency>
|
|
23
|
+
<groupId>com.google.cloud</groupId>
|
|
24
|
+
<artifactId>google-cloud-bigquery</artifactId>
|
|
25
|
+
<version>1.102.0</version>
|
|
26
|
+
</dependency>
|
|
27
|
+
</dependencies>
|
|
28
|
+
|
|
29
|
+
<profiles>
|
|
30
|
+
<profile>
|
|
31
|
+
<id>binary-packaging</id>
|
|
32
|
+
<build>
|
|
33
|
+
<plugins>
|
|
34
|
+
<plugin>
|
|
35
|
+
<artifactId>maven-dependency-plugin</artifactId>
|
|
36
|
+
<executions>
|
|
37
|
+
<execution>
|
|
38
|
+
<phase>package</phase>
|
|
39
|
+
<goals>
|
|
40
|
+
<goal>copy-dependencies</goal>
|
|
41
|
+
</goals>
|
|
42
|
+
<configuration>
|
|
43
|
+
<outputDirectory>${project.build.directory}</outputDirectory>
|
|
44
|
+
<!-- compile scope gives runtime and compile dependencies (skips test deps) -->
|
|
45
|
+
<includeScope>runtime</includeScope>
|
|
46
|
+
</configuration>
|
|
47
|
+
</execution>
|
|
48
|
+
</executions>
|
|
49
|
+
</plugin>
|
|
50
|
+
</plugins>
|
|
51
|
+
</build>
|
|
52
|
+
</profile>
|
|
53
|
+
</profiles>
|
|
54
|
+
</project>
|
data/ci/redshift/pom.xml
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
|
3
|
+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
4
|
+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
5
|
+
<modelVersion>4.0.0</modelVersion>
|
|
6
|
+
|
|
7
|
+
<groupId>com.gooddata.lcm</groupId>
|
|
8
|
+
<artifactId>lcm-redshift-driver</artifactId>
|
|
9
|
+
<version>1.0-SNAPSHOT</version>
|
|
10
|
+
|
|
11
|
+
<dependencies>
|
|
12
|
+
<dependency>
|
|
13
|
+
<groupId>com.amazon.redshift</groupId>
|
|
14
|
+
<artifactId>redshift-jdbc42-no-awssdk</artifactId>
|
|
15
|
+
<version>1.2.36.1060</version>
|
|
16
|
+
</dependency>
|
|
17
|
+
<dependency>
|
|
18
|
+
<groupId>com.amazonaws</groupId>
|
|
19
|
+
<artifactId>aws-java-sdk-core</artifactId>
|
|
20
|
+
<version>1.11.633</version>
|
|
21
|
+
</dependency>
|
|
22
|
+
<dependency>
|
|
23
|
+
<groupId>com.amazonaws</groupId>
|
|
24
|
+
<artifactId>aws-java-sdk-redshift</artifactId>
|
|
25
|
+
<version>1.11.633</version>
|
|
26
|
+
</dependency>
|
|
27
|
+
<dependency>
|
|
28
|
+
<groupId>com.amazonaws</groupId>
|
|
29
|
+
<artifactId>aws-java-sdk-sts</artifactId>
|
|
30
|
+
<version>1.11.633</version>
|
|
31
|
+
</dependency>
|
|
32
|
+
<dependency>
|
|
33
|
+
<groupId>log4j</groupId>
|
|
34
|
+
<artifactId>log4j</artifactId>
|
|
35
|
+
<version>1.2.17</version>
|
|
36
|
+
<scope>compile</scope>
|
|
37
|
+
</dependency>
|
|
38
|
+
</dependencies>
|
|
39
|
+
|
|
40
|
+
<profiles>
|
|
41
|
+
<profile>
|
|
42
|
+
<id>binary-packaging</id>
|
|
43
|
+
<build>
|
|
44
|
+
<plugins>
|
|
45
|
+
<plugin>
|
|
46
|
+
<artifactId>maven-dependency-plugin</artifactId>
|
|
47
|
+
<executions>
|
|
48
|
+
<execution>
|
|
49
|
+
<phase>package</phase>
|
|
50
|
+
<goals>
|
|
51
|
+
<goal>copy-dependencies</goal>
|
|
52
|
+
</goals>
|
|
53
|
+
<configuration>
|
|
54
|
+
<outputDirectory>${project.build.directory}</outputDirectory>
|
|
55
|
+
<!-- compile scope gives runtime and compile dependencies (skips test deps) -->
|
|
56
|
+
<includeScope>runtime</includeScope>
|
|
57
|
+
</configuration>
|
|
58
|
+
</execution>
|
|
59
|
+
</executions>
|
|
60
|
+
</plugin>
|
|
61
|
+
</plugins>
|
|
62
|
+
</build>
|
|
63
|
+
</profile>
|
|
64
|
+
</profiles>
|
|
65
|
+
|
|
66
|
+
<repositories>
|
|
67
|
+
<repository>
|
|
68
|
+
<id>my-repo1</id>
|
|
69
|
+
<name>my custom repo</name>
|
|
70
|
+
<url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
|
|
71
|
+
</repository>
|
|
72
|
+
</repositories>
|
|
73
|
+
</project>
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
|
3
|
+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
4
|
+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
5
|
+
<modelVersion>4.0.0</modelVersion>
|
|
6
|
+
|
|
7
|
+
<groupId>com.gooddata.lcm</groupId>
|
|
8
|
+
<artifactId>lcm-snowflake-driver</artifactId>
|
|
9
|
+
<version>1.0-SNAPSHOT</version>
|
|
10
|
+
|
|
11
|
+
<dependencies>
|
|
12
|
+
<dependency>
|
|
13
|
+
<groupId>net.snowflake</groupId>
|
|
14
|
+
<artifactId>snowflake-jdbc</artifactId>
|
|
15
|
+
<version>3.6.22</version>
|
|
16
|
+
</dependency>
|
|
17
|
+
<dependency>
|
|
18
|
+
<groupId>org.slf4j</groupId>
|
|
19
|
+
<artifactId>slf4j-api</artifactId>
|
|
20
|
+
<version>1.7.2</version>
|
|
21
|
+
</dependency>
|
|
22
|
+
</dependencies>
|
|
23
|
+
|
|
24
|
+
<profiles>
|
|
25
|
+
<profile>
|
|
26
|
+
<id>binary-packaging</id>
|
|
27
|
+
<build>
|
|
28
|
+
<plugins>
|
|
29
|
+
<plugin>
|
|
30
|
+
<artifactId>maven-dependency-plugin</artifactId>
|
|
31
|
+
<executions>
|
|
32
|
+
<execution>
|
|
33
|
+
<phase>package</phase>
|
|
34
|
+
<goals>
|
|
35
|
+
<goal>copy-dependencies</goal>
|
|
36
|
+
</goals>
|
|
37
|
+
<configuration>
|
|
38
|
+
<outputDirectory>${project.build.directory}</outputDirectory>
|
|
39
|
+
<!-- compile scope gives runtime and compile dependencies (skips test deps) -->
|
|
40
|
+
<includeScope>runtime</includeScope>
|
|
41
|
+
</configuration>
|
|
42
|
+
</execution>
|
|
43
|
+
</executions>
|
|
44
|
+
</plugin>
|
|
45
|
+
</plugins>
|
|
46
|
+
</build>
|
|
47
|
+
</profile>
|
|
48
|
+
</profiles>
|
|
49
|
+
|
|
50
|
+
<repositories>
|
|
51
|
+
<repository>
|
|
52
|
+
<id>my-repo1</id>
|
|
53
|
+
<name>my custom repo</name>
|
|
54
|
+
<url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
|
|
55
|
+
</repository>
|
|
56
|
+
</repositories>
|
|
57
|
+
</project>
|
data/lcm.rake
CHANGED
|
@@ -118,7 +118,7 @@ end
|
|
|
118
118
|
namespace :docker do
|
|
119
119
|
desc 'Build Docker image'
|
|
120
120
|
task :build do
|
|
121
|
-
Rake::Task["maven:
|
|
121
|
+
Rake::Task["maven:build_dependencies"].invoke
|
|
122
122
|
system('docker build -f Dockerfile.jruby -t gooddata/appstore .')
|
|
123
123
|
end
|
|
124
124
|
|
|
@@ -129,11 +129,15 @@ namespace :docker do
|
|
|
129
129
|
end
|
|
130
130
|
|
|
131
131
|
namespace :maven do
|
|
132
|
-
task :
|
|
133
|
-
system(
|
|
134
|
-
system('
|
|
135
|
-
|
|
136
|
-
system('
|
|
132
|
+
task :build_dependencies do
|
|
133
|
+
system('mvn -f ci/snowflake/pom.xml clean install -P binary-packaging')
|
|
134
|
+
system('cp -rf ci/snowflake/target/*.jar lib/gooddata/cloud_resources/snowflake/drivers/')
|
|
135
|
+
|
|
136
|
+
system('mvn -f ci/bigquery/pom.xml clean install -P binary-packaging')
|
|
137
|
+
system('cp -rf ci/bigquery/target/*.jar lib/gooddata/cloud_resources/bigquery/drivers/')
|
|
138
|
+
|
|
139
|
+
system('mvn -f ci/redshift/pom.xml clean install -P binary-packaging')
|
|
140
|
+
system('cp -rf ci/redshift/target/*.jar lib/gooddata/cloud_resources/redshift/drivers/')
|
|
137
141
|
end
|
|
138
142
|
end
|
|
139
143
|
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# encoding: UTF-8
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
#
|
|
4
|
+
# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
|
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
|
7
|
+
|
|
8
|
+
require 'securerandom'
|
|
9
|
+
require 'java'
|
|
10
|
+
require 'pathname'
|
|
11
|
+
require_relative '../cloud_resource_client'
|
|
12
|
+
|
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
|
15
|
+
require file unless file.start_with?('lcm-bigquery-driver')
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
java_import 'com.google.auth.oauth2.ServiceAccountCredentials'
|
|
19
|
+
java_import 'com.google.cloud.bigquery.BigQuery'
|
|
20
|
+
java_import 'com.google.cloud.bigquery.BigQueryOptions'
|
|
21
|
+
java_import 'com.google.cloud.bigquery.FieldList'
|
|
22
|
+
java_import 'com.google.cloud.bigquery.FieldValue'
|
|
23
|
+
java_import 'com.google.cloud.bigquery.FieldValueList'
|
|
24
|
+
java_import 'com.google.cloud.bigquery.QueryJobConfiguration'
|
|
25
|
+
java_import 'com.google.cloud.bigquery.TableResult'
|
|
26
|
+
java_import 'org.apache.commons.text.StringEscapeUtils'
|
|
27
|
+
|
|
28
|
+
module GoodData
|
|
29
|
+
module CloudResources
|
|
30
|
+
class BigQueryClient < CloudResourceClient
|
|
31
|
+
class << self
|
|
32
|
+
def accept?(type)
|
|
33
|
+
type == 'bigquery'
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def initialize(options = {})
|
|
38
|
+
raise("Data Source needs a client to BigQuery to be able to query the storage but 'bigquery_client' is empty.") unless options['bigquery_client']
|
|
39
|
+
|
|
40
|
+
if options['bigquery_client']['connection'].is_a?(Hash)
|
|
41
|
+
@project = options['bigquery_client']['connection']['project']
|
|
42
|
+
@schema = options['bigquery_client']['connection']['schema'] || 'public'
|
|
43
|
+
@authentication = options['bigquery_client']['connection']['authentication']
|
|
44
|
+
else
|
|
45
|
+
raise('Missing connection info for BigQuery client')
|
|
46
|
+
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def realize_query(query, _params)
|
|
51
|
+
GoodData.gd_logger.info("Realize SQL query: type=bigquery status=started")
|
|
52
|
+
|
|
53
|
+
client = create_client
|
|
54
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
|
55
|
+
measure = Benchmark.measure do
|
|
56
|
+
query_config = QueryJobConfiguration.newBuilder(query).setDefaultDataset(@schema).build
|
|
57
|
+
table_result = client.query(query_config)
|
|
58
|
+
|
|
59
|
+
if table_result.getTotalRows.positive?
|
|
60
|
+
result = table_result.iterateAll
|
|
61
|
+
field_list = table_result.getSchema.getFields
|
|
62
|
+
col_count = field_list.size
|
|
63
|
+
CSV.open(filename, 'wb') do |csv|
|
|
64
|
+
csv << Array(1..col_count).map { |i| field_list.get(i - 1).getName } # build the header
|
|
65
|
+
result.each do |row|
|
|
66
|
+
csv << Array(1..col_count).map { |i| row.get(i - 1).getValue&.to_s }
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
GoodData.gd_logger.info("Realize SQL query: type=bigquery status=finished duration=#{measure.real}")
|
|
72
|
+
filename
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
private
|
|
76
|
+
|
|
77
|
+
def create_client
|
|
78
|
+
GoodData.logger.info "Setting up connection to BigQuery"
|
|
79
|
+
client_email = @authentication['serviceAccount']['clientEmail']
|
|
80
|
+
private_key = @authentication['serviceAccount']['privateKey']
|
|
81
|
+
credentials = ServiceAccountCredentials.fromPkcs8(nil, client_email, StringEscapeUtils.unescapeJson(private_key), nil, nil)
|
|
82
|
+
BigQueryOptions.newBuilder.setProjectId(@project).setCredentials(credentials).build.getService
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
File without changes
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
# encoding: UTF-8
|
|
2
|
+
# frozen_string_literal: true
|
|
2
3
|
#
|
|
3
4
|
# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
|
|
4
5
|
# This source code is licensed under the BSD-style license found in the
|
|
@@ -57,9 +58,9 @@ module GoodData
|
|
|
57
58
|
result = statement.get_result_set
|
|
58
59
|
metadata = result.get_meta_data
|
|
59
60
|
col_count = metadata.column_count
|
|
60
|
-
CSV.open(filename, 'wb'
|
|
61
|
+
CSV.open(filename, 'wb') do |csv|
|
|
61
62
|
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
|
62
|
-
csv << Array(1..col_count).map { |i| result.get_string(i) } while result.next
|
|
63
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
|
63
64
|
end
|
|
64
65
|
end
|
|
65
66
|
end
|
|
File without changes
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# encoding: UTF-8
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
#
|
|
4
|
+
# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
|
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
|
7
|
+
|
|
8
|
+
require 'securerandom'
|
|
9
|
+
require 'java'
|
|
10
|
+
require 'pathname'
|
|
11
|
+
require_relative '../cloud_resource_client'
|
|
12
|
+
|
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
|
15
|
+
require file unless file.start_with?('lcm-snowflake-driver')
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
module GoodData
|
|
19
|
+
module CloudResources
|
|
20
|
+
class SnowflakeClient < CloudResourceClient
|
|
21
|
+
class << self
|
|
22
|
+
def accept?(type)
|
|
23
|
+
type == 'snowflake'
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def initialize(options = {})
|
|
28
|
+
raise("Data Source needs a client to Snowflake to be able to query the storage but 'snowflake_client' is empty.") unless options['snowflake_client']
|
|
29
|
+
|
|
30
|
+
if options['snowflake_client']['connection'].is_a?(Hash)
|
|
31
|
+
@database = options['snowflake_client']['connection']['database']
|
|
32
|
+
@schema = options['snowflake_client']['connection']['schema'] || 'public'
|
|
33
|
+
@warehouse = options['snowflake_client']['connection']['warehouse']
|
|
34
|
+
@url = options['snowflake_client']['connection']['url']
|
|
35
|
+
@authentication = options['snowflake_client']['connection']['authentication']
|
|
36
|
+
else
|
|
37
|
+
raise('Missing connection info for Snowflake client')
|
|
38
|
+
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
Java.net.snowflake.client.jdbc.SnowflakeDriver
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def realize_query(query, _params)
|
|
45
|
+
GoodData.gd_logger.info("Realize SQL query: type=snowflake status=started")
|
|
46
|
+
|
|
47
|
+
connect
|
|
48
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
|
49
|
+
measure = Benchmark.measure do
|
|
50
|
+
statement = @connection.create_statement
|
|
51
|
+
|
|
52
|
+
has_result = statement.execute(query)
|
|
53
|
+
if has_result
|
|
54
|
+
result = statement.get_result_set
|
|
55
|
+
metadata = result.get_meta_data
|
|
56
|
+
col_count = metadata.column_count
|
|
57
|
+
CSV.open(filename, 'wb') do |csv|
|
|
58
|
+
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
|
59
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
GoodData.gd_logger.info("Realize SQL query: type=snowflake status=finished duration=#{measure.real}")
|
|
64
|
+
filename
|
|
65
|
+
ensure
|
|
66
|
+
@connection&.close
|
|
67
|
+
@connection = nil
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def connect
|
|
71
|
+
GoodData.logger.info "Setting up connection to Snowflake #{@url}"
|
|
72
|
+
|
|
73
|
+
prop = java.util.Properties.new
|
|
74
|
+
prop.setProperty('user', @authentication['basic']['userName'])
|
|
75
|
+
prop.setProperty('password', @authentication['basic']['password'])
|
|
76
|
+
prop.setProperty('schema', @schema)
|
|
77
|
+
prop.setProperty('warehouse', @warehouse)
|
|
78
|
+
prop.setProperty('db', @database)
|
|
79
|
+
|
|
80
|
+
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
@@ -44,7 +44,7 @@ module GoodData
|
|
|
44
44
|
realize_link
|
|
45
45
|
when 's3'
|
|
46
46
|
realize_s3(params)
|
|
47
|
-
when 'redshift'
|
|
47
|
+
when 'redshift', 'snowflake', 'bigquery'
|
|
48
48
|
raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
|
|
49
49
|
|
|
50
50
|
require_relative '../cloud_resources/cloud_resources'
|
|
@@ -67,11 +67,11 @@ module GoodData
|
|
|
67
67
|
end
|
|
68
68
|
|
|
69
69
|
def collect_clients(params, segment_names = nil)
|
|
70
|
-
client_id_column = params.client_id_column || 'client_id'
|
|
71
|
-
segment_id_column = params.segment_id_column || 'segment_id'
|
|
72
|
-
project_id_column = params.project_id_column || 'project_id'
|
|
73
|
-
project_title_column = params.project_title_column || 'project_title'
|
|
74
|
-
project_token_column = params.project_token_column || 'project_token'
|
|
70
|
+
client_id_column = params.client_id_column&.downcase || 'client_id'
|
|
71
|
+
segment_id_column = params.segment_id_column&.downcase || 'segment_id'
|
|
72
|
+
project_id_column = params.project_id_column&.downcase || 'project_id'
|
|
73
|
+
project_title_column = params.project_title_column&.downcase || 'project_title'
|
|
74
|
+
project_token_column = params.project_token_column&.downcase || 'project_token'
|
|
75
75
|
client = params.gdc_gd_client
|
|
76
76
|
|
|
77
77
|
clients = []
|
|
@@ -82,7 +82,7 @@ module GoodData
|
|
|
82
82
|
end
|
|
83
83
|
GoodData.logger.debug("Input data: #{input_data.read}")
|
|
84
84
|
GoodData.logger.debug("Segment names: #{segment_names}")
|
|
85
|
-
CSV.foreach(input_data, :headers => true, :return_headers => false,
|
|
85
|
+
CSV.foreach(input_data, :headers => true, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
|
|
86
86
|
GoodData.logger.debug("Processing row: #{row}")
|
|
87
87
|
segment_name = row[segment_id_column]
|
|
88
88
|
GoodData.logger.debug("Segment name: #{segment_name}")
|
|
@@ -38,11 +38,11 @@ module GoodData
|
|
|
38
38
|
def call(params)
|
|
39
39
|
return [] unless params.dynamic_params
|
|
40
40
|
|
|
41
|
-
schedule_title_column = params.schedule_title_column || 'schedule_title'
|
|
42
|
-
client_id_column = params.client_id_column || 'client_id'
|
|
43
|
-
param_name_column = params.param_name_column || 'param_name'
|
|
44
|
-
param_value_column = params.param_value_column || 'param_value'
|
|
45
|
-
param_secure_column = params.param_secure_column || 'param_secure'
|
|
41
|
+
schedule_title_column = params.schedule_title_column&.downcase || 'schedule_title'
|
|
42
|
+
client_id_column = params.client_id_column&.downcase || 'client_id'
|
|
43
|
+
param_name_column = params.param_name_column&.downcase || 'param_name'
|
|
44
|
+
param_value_column = params.param_value_column&.downcase || 'param_value'
|
|
45
|
+
param_secure_column = params.param_secure_column&.downcase || 'param_secure'
|
|
46
46
|
|
|
47
47
|
encryption_key = params.dynamic_params_encryption_key || ''
|
|
48
48
|
exist_encryption_key = encryption_key.blank? ? false : true
|
|
@@ -59,7 +59,7 @@ module GoodData
|
|
|
59
59
|
schedule_hidden_params = {}
|
|
60
60
|
exist_param_secure = false
|
|
61
61
|
|
|
62
|
-
CSV.foreach(input_data, :headers => true, :return_headers => false,
|
|
62
|
+
CSV.foreach(input_data, :headers => true, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
|
|
63
63
|
is_param_secure = row[param_secure_column] == 'true'
|
|
64
64
|
is_decrypt_secure_value = is_param_secure && exist_encryption_key ? true : false
|
|
65
65
|
exist_param_secure = true if is_param_secure
|
|
@@ -35,7 +35,7 @@ module GoodData
|
|
|
35
35
|
class << self
|
|
36
36
|
def call(params)
|
|
37
37
|
users_brick_users = []
|
|
38
|
-
login_column = params.users_brick_config.login_column || 'login'
|
|
38
|
+
login_column = params.users_brick_config.login_column&.downcase || 'login'
|
|
39
39
|
users_brick_data_source = GoodData::Helpers::DataSource.new(params.users_brick_config.input_source)
|
|
40
40
|
|
|
41
41
|
users_brick_data_source_file = without_check(PARAMS, params) do
|
|
@@ -45,14 +45,15 @@ module GoodData
|
|
|
45
45
|
)
|
|
46
46
|
end
|
|
47
47
|
CSV.foreach(users_brick_data_source_file,
|
|
48
|
-
headers
|
|
49
|
-
return_headers
|
|
50
|
-
|
|
51
|
-
|
|
48
|
+
:headers => true,
|
|
49
|
+
:return_headers => false,
|
|
50
|
+
:header_converters => :downcase,
|
|
51
|
+
:encoding => 'utf-8') do |row|
|
|
52
|
+
pid = row[params.multiple_projects_column&.downcase]
|
|
52
53
|
fail "The set multiple_projects_column '#{params.multiple_projects_column}' of the users input is empty" if !pid && MULTIPLE_COLUMN_MODES.include?(params.sync_mode)
|
|
53
54
|
|
|
54
55
|
users_brick_users << {
|
|
55
|
-
login: row[login_column].downcase,
|
|
56
|
+
login: row[login_column].nil? ? nil : row[login_column].strip.downcase,
|
|
56
57
|
pid: pid
|
|
57
58
|
}
|
|
58
59
|
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# encoding: UTF-8
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
#
|
|
4
|
+
# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
|
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
|
7
|
+
|
|
8
|
+
require_relative 'base_action'
|
|
9
|
+
|
|
10
|
+
module GoodData
|
|
11
|
+
module LCM2
|
|
12
|
+
class SetMasterProject < BaseAction
|
|
13
|
+
DESCRIPTION = 'Set master project'
|
|
14
|
+
|
|
15
|
+
PARAMS = define_params(self) do
|
|
16
|
+
description 'Organization Name'
|
|
17
|
+
param :organization, instance_of(Type::StringType), required: false
|
|
18
|
+
|
|
19
|
+
description 'Domain'
|
|
20
|
+
param :domain, instance_of(Type::StringType), required: false
|
|
21
|
+
|
|
22
|
+
description 'ADS Client'
|
|
23
|
+
param :ads_client, instance_of(Type::AdsClientType), required: false
|
|
24
|
+
|
|
25
|
+
description 'Table Name'
|
|
26
|
+
param :release_table_name, instance_of(Type::StringType), required: false
|
|
27
|
+
|
|
28
|
+
description 'Segments to manage'
|
|
29
|
+
param :segments, array_of(instance_of(Type::SegmentType)), required: true
|
|
30
|
+
|
|
31
|
+
description 'DataProduct to manage'
|
|
32
|
+
param :data_product, instance_of(Type::GDDataProductType), required: false
|
|
33
|
+
|
|
34
|
+
description 'Released master project should be used in next rollout'
|
|
35
|
+
param :set_master_project, instance_of(Type::StringType), required: false
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
class << self
|
|
39
|
+
def call(params)
|
|
40
|
+
results = []
|
|
41
|
+
domain_name = params.organization || params.domain
|
|
42
|
+
data_product = params.data_product
|
|
43
|
+
params.segments.each do |segment_in|
|
|
44
|
+
version = get_latest_version(params, domain_name, data_product.data_product_id, segment_in.segment_id) + 1
|
|
45
|
+
segment_in[:data_product_id] = data_product.data_product_id
|
|
46
|
+
segment_in[:master_pid] = params.set_master_project
|
|
47
|
+
segment_in[:version] = version
|
|
48
|
+
segment_in[:timestamp] = Time.now.utc.iso8601
|
|
49
|
+
|
|
50
|
+
results << {
|
|
51
|
+
data_product_id: data_product.data_product_id,
|
|
52
|
+
segment_id: segment_in.segment_id,
|
|
53
|
+
version: version
|
|
54
|
+
}
|
|
55
|
+
end
|
|
56
|
+
results
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def get_latest_version(params, domain_name, data_product_id, segment_id)
|
|
60
|
+
if params.ads_client
|
|
61
|
+
current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
|
|
62
|
+
params.release_table_name,
|
|
63
|
+
params.ads_client,
|
|
64
|
+
segment_id
|
|
65
|
+
)
|
|
66
|
+
else
|
|
67
|
+
current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product_id, segment_id)
|
|
68
|
+
end
|
|
69
|
+
return 0 unless current_master
|
|
70
|
+
|
|
71
|
+
current_master[:version].to_i
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -230,8 +230,8 @@ module GoodData
|
|
|
230
230
|
begin
|
|
231
231
|
GoodData.logger.info('Start reading data')
|
|
232
232
|
row_count = 0
|
|
233
|
-
CSV.foreach(tmp, headers
|
|
234
|
-
filters << row.to_hash.merge(pid: row[multiple_projects_column])
|
|
233
|
+
CSV.foreach(tmp, :headers => csv_with_headers, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
|
|
234
|
+
filters << row.to_hash.merge(pid: row[multiple_projects_column.downcase])
|
|
235
235
|
row_count += 1
|
|
236
236
|
GoodData.logger.info("Read #{row_count} rows") if (row_count % 50_000).zero?
|
|
237
237
|
end
|
|
@@ -347,38 +347,33 @@ module GoodData
|
|
|
347
347
|
end
|
|
348
348
|
|
|
349
349
|
def load_data(params, data_source)
|
|
350
|
-
first_name_column = params.first_name_column || 'first_name'
|
|
351
|
-
last_name_column = params.last_name_column || 'last_name'
|
|
352
|
-
login_column = params.login_column || 'login'
|
|
353
|
-
password_column = params.password_column || 'password'
|
|
354
|
-
email_column = params.email_column || 'email'
|
|
355
|
-
role_column = params.role_column || 'role'
|
|
356
|
-
sso_provider_column = params.sso_provider_column || 'sso_provider'
|
|
357
|
-
authentication_modes_column = params.authentication_modes_column || 'authentication_modes'
|
|
358
|
-
user_groups_column = params.user_groups_column || 'user_groups'
|
|
359
|
-
language_column = params.language_column || 'language'
|
|
360
|
-
company_column = params.company_column || 'company'
|
|
361
|
-
position_column = params.position_column || 'position'
|
|
362
|
-
country_column = params.country_column || 'country'
|
|
363
|
-
phone_column = params.phone_column || 'phone'
|
|
364
|
-
ip_whitelist_column = params.ip_whitelist_column || 'ip_whitelist'
|
|
350
|
+
first_name_column = params.first_name_column&.downcase || 'first_name'
|
|
351
|
+
last_name_column = params.last_name_column&.downcase || 'last_name'
|
|
352
|
+
login_column = params.login_column&.downcase || 'login'
|
|
353
|
+
password_column = params.password_column&.downcase || 'password'
|
|
354
|
+
email_column = params.email_column&.downcase || 'email'
|
|
355
|
+
role_column = params.role_column&.downcase || 'role'
|
|
356
|
+
sso_provider_column = params.sso_provider_column&.downcase || 'sso_provider'
|
|
357
|
+
authentication_modes_column = params.authentication_modes_column&.downcase || 'authentication_modes'
|
|
358
|
+
user_groups_column = params.user_groups_column&.downcase || 'user_groups'
|
|
359
|
+
language_column = params.language_column&.downcase || 'language'
|
|
360
|
+
company_column = params.company_column&.downcase || 'company'
|
|
361
|
+
position_column = params.position_column&.downcase || 'position'
|
|
362
|
+
country_column = params.country_column&.downcase || 'country'
|
|
363
|
+
phone_column = params.phone_column&.downcase || 'phone'
|
|
364
|
+
ip_whitelist_column = params.ip_whitelist_column&.downcase || 'ip_whitelist'
|
|
365
365
|
|
|
366
366
|
sso_provider = params.sso_provider
|
|
367
367
|
authentication_modes = params.authentication_modes || []
|
|
368
368
|
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
else
|
|
373
|
-
tmp = without_check(PARAMS, params) do
|
|
374
|
-
File.open(data_source.realize(params), 'r:UTF-8')
|
|
375
|
-
end
|
|
369
|
+
tmp = without_check(PARAMS, params) do
|
|
370
|
+
File.open(data_source.realize(params), 'r:UTF-8')
|
|
371
|
+
end
|
|
376
372
|
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
end
|
|
373
|
+
begin
|
|
374
|
+
data = read_csv_file(tmp)
|
|
375
|
+
rescue Exception => e # rubocop:disable RescueException
|
|
376
|
+
fail "There was an error during loading users from csv file. Message: #{e.message}. Error: #{e}"
|
|
382
377
|
end
|
|
383
378
|
|
|
384
379
|
data.map do |row|
|
|
@@ -398,12 +393,18 @@ module GoodData
|
|
|
398
393
|
ip_whitelist = row[ip_whitelist_column] || row[ip_whitelist_column.to_sym]
|
|
399
394
|
ip_whitelist = ip_whitelist.split(',').map(&:strip) if ip_whitelist
|
|
400
395
|
|
|
396
|
+
user_login = row[login_column] || row[login_column.to_sym]
|
|
397
|
+
user_login = user_login.strip unless user_login.nil?
|
|
398
|
+
|
|
399
|
+
user_email = row[email_column] || row[login_column] || row[email_column.to_sym] || row[login_column.to_sym]
|
|
400
|
+
user_email = user_email.strip unless user_email.nil?
|
|
401
|
+
|
|
401
402
|
{
|
|
402
403
|
:first_name => row[first_name_column] || row[first_name_column.to_sym],
|
|
403
404
|
:last_name => row[last_name_column] || row[last_name_column.to_sym],
|
|
404
|
-
:login =>
|
|
405
|
+
:login => user_login,
|
|
405
406
|
:password => row[password_column] || row[password_column.to_sym],
|
|
406
|
-
:email =>
|
|
407
|
+
:email => user_email,
|
|
407
408
|
:role => row[role_column] || row[role_column.to_sym],
|
|
408
409
|
:sso_provider => sso_provider || row[sso_provider_column] || row[sso_provider_column.to_sym],
|
|
409
410
|
:authentication_modes => modes,
|
|
@@ -424,7 +425,7 @@ module GoodData
|
|
|
424
425
|
res = []
|
|
425
426
|
row_count = 0
|
|
426
427
|
|
|
427
|
-
CSV.foreach(path, :headers => true) do |row|
|
|
428
|
+
CSV.foreach(path, :headers => true, :header_converters => :downcase, :encoding => 'utf-8') do |row|
|
|
428
429
|
if block_given?
|
|
429
430
|
data = yield row
|
|
430
431
|
else
|
data/lib/gooddata/lcm/lcm2.rb
CHANGED
|
@@ -106,6 +106,14 @@ module GoodData
|
|
|
106
106
|
UpdateReleaseTable
|
|
107
107
|
],
|
|
108
108
|
|
|
109
|
+
release_set_master_project: [
|
|
110
|
+
EnsureReleaseTable,
|
|
111
|
+
CollectDataProduct,
|
|
112
|
+
SegmentsFilter,
|
|
113
|
+
SetMasterProject,
|
|
114
|
+
UpdateReleaseTable
|
|
115
|
+
],
|
|
116
|
+
|
|
109
117
|
provision: [
|
|
110
118
|
EnsureReleaseTable,
|
|
111
119
|
CollectDataProduct,
|
|
@@ -271,8 +279,14 @@ module GoodData
|
|
|
271
279
|
|
|
272
280
|
GoodData.gd_logger.brick = mode
|
|
273
281
|
|
|
282
|
+
final_mode = if params.set_master_project && mode == 'release'
|
|
283
|
+
'release_set_master_project'
|
|
284
|
+
else
|
|
285
|
+
mode
|
|
286
|
+
end
|
|
287
|
+
|
|
274
288
|
# Get actions for mode specified
|
|
275
|
-
actions = get_mode_actions(
|
|
289
|
+
actions = get_mode_actions(final_mode)
|
|
276
290
|
|
|
277
291
|
if params.actions
|
|
278
292
|
actions = params.actions.map do |action|
|
|
@@ -305,6 +319,12 @@ module GoodData
|
|
|
305
319
|
skip_actions.include?(action.name.split('::').last)
|
|
306
320
|
end
|
|
307
321
|
|
|
322
|
+
sync_mode = params.fetch(:sync_mode, nil)
|
|
323
|
+
if mode == 'users' && %w[add_to_organization remove_from_organization].include?(sync_mode)
|
|
324
|
+
actions = actions.reject do |action|
|
|
325
|
+
%w[CollectDataProduct CollectSegments].include?(action.name.split('::').last)
|
|
326
|
+
end
|
|
327
|
+
end
|
|
308
328
|
check_unused_params(actions, params)
|
|
309
329
|
print_action_names(mode, actions)
|
|
310
330
|
|
|
@@ -222,22 +222,24 @@ module GoodData
|
|
|
222
222
|
domain = client.domain(domain)
|
|
223
223
|
if id == :all
|
|
224
224
|
GoodData.logger.warn("Retrieving all users from domain #{domain.name}")
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
end
|
|
232
|
-
|
|
233
|
-
tmp['accountSettings']['items'].each do |user_data|
|
|
234
|
-
user = client.create(GoodData::Profile, user_data)
|
|
235
|
-
y << user if user
|
|
236
|
-
end
|
|
237
|
-
break if tmp['accountSettings']['items'].count < page_limit
|
|
238
|
-
offset += page_limit
|
|
225
|
+
all_users = []
|
|
226
|
+
page_limit = opts[:page_limit] || 1000
|
|
227
|
+
offset = opts[:offset] || 0
|
|
228
|
+
loop do
|
|
229
|
+
begin
|
|
230
|
+
tmp = client(opts).get("#{domain.uri}/users", params: { offset: offset, limit: page_limit })
|
|
239
231
|
end
|
|
232
|
+
|
|
233
|
+
tmp['accountSettings']['items'].each do |user_data|
|
|
234
|
+
user = client.create(GoodData::Profile, user_data)
|
|
235
|
+
all_users << user if user
|
|
236
|
+
end
|
|
237
|
+
break if tmp['accountSettings']['items'].count < page_limit
|
|
238
|
+
|
|
239
|
+
offset += page_limit
|
|
240
240
|
end
|
|
241
|
+
|
|
242
|
+
all_users
|
|
241
243
|
else
|
|
242
244
|
find_user_by_login(domain, id)
|
|
243
245
|
end
|
|
@@ -1546,26 +1546,28 @@ module GoodData
|
|
|
1546
1546
|
# @return [Array<GoodData::User>] List of users
|
|
1547
1547
|
def users(opts = {})
|
|
1548
1548
|
client = client(opts)
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
end
|
|
1549
|
+
all_users = []
|
|
1550
|
+
offset = opts[:offset] || 0
|
|
1551
|
+
limit = opts[:limit] || 1_000
|
|
1552
|
+
loop do
|
|
1553
|
+
tmp = client.get("/gdc/projects/#{pid}/users", params: { offset: offset, limit: limit })
|
|
1554
|
+
tmp['users'].each do |user_data|
|
|
1555
|
+
user = client.create(GoodData::Membership, user_data, project: self)
|
|
1556
|
+
|
|
1557
|
+
if opts[:all]
|
|
1558
|
+
all_users << user
|
|
1559
|
+
elsif opts[:disabled]
|
|
1560
|
+
all_users << user if user&.disabled?
|
|
1561
|
+
else
|
|
1562
|
+
all_users << user if user&.enabled?
|
|
1564
1563
|
end
|
|
1565
|
-
break if tmp['users'].count < limit
|
|
1566
|
-
offset += limit
|
|
1567
1564
|
end
|
|
1565
|
+
break if tmp['users'].count < limit
|
|
1566
|
+
|
|
1567
|
+
offset += limit
|
|
1568
1568
|
end
|
|
1569
|
+
|
|
1570
|
+
all_users
|
|
1569
1571
|
end
|
|
1570
1572
|
|
|
1571
1573
|
alias_method :members, :users
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: gooddata
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.1.
|
|
4
|
+
version: 2.1.10
|
|
5
5
|
platform: java
|
|
6
6
|
authors:
|
|
7
7
|
- Pavel Kolesnikov
|
|
@@ -14,7 +14,7 @@ authors:
|
|
|
14
14
|
autorequire:
|
|
15
15
|
bindir: bin
|
|
16
16
|
cert_chain: []
|
|
17
|
-
date: 2020-
|
|
17
|
+
date: 2020-05-08 00:00:00.000000000 Z
|
|
18
18
|
dependencies:
|
|
19
19
|
- !ruby/object:Gem::Dependency
|
|
20
20
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -560,6 +560,9 @@ files:
|
|
|
560
560
|
- bin/user_filters.sh
|
|
561
561
|
- bin/users.sh
|
|
562
562
|
- ci.rake
|
|
563
|
+
- ci/bigquery/pom.xml
|
|
564
|
+
- ci/redshift/pom.xml
|
|
565
|
+
- ci/snowflake/pom.xml
|
|
563
566
|
- data/2008.crt
|
|
564
567
|
- data/new_ca.cer
|
|
565
568
|
- data/new_prodgdc_ca.crt
|
|
@@ -616,11 +619,15 @@ files:
|
|
|
616
619
|
- lib/gooddata/cli/shared.rb
|
|
617
620
|
- lib/gooddata/cli/terminal.rb
|
|
618
621
|
- lib/gooddata/client.rb
|
|
622
|
+
- lib/gooddata/cloud_resources/bigquery/bigquery_client.rb
|
|
623
|
+
- lib/gooddata/cloud_resources/bigquery/drivers/.gitkeepme
|
|
619
624
|
- lib/gooddata/cloud_resources/cloud_resouce_factory.rb
|
|
620
625
|
- lib/gooddata/cloud_resources/cloud_resource_client.rb
|
|
621
626
|
- lib/gooddata/cloud_resources/cloud_resources.rb
|
|
622
627
|
- lib/gooddata/cloud_resources/redshift/drivers/log4j.properties
|
|
623
628
|
- lib/gooddata/cloud_resources/redshift/redshift_client.rb
|
|
629
|
+
- lib/gooddata/cloud_resources/snowflake/drivers/.gitkeepme
|
|
630
|
+
- lib/gooddata/cloud_resources/snowflake/snowflake_client.rb
|
|
624
631
|
- lib/gooddata/commands/api.rb
|
|
625
632
|
- lib/gooddata/commands/auth.rb
|
|
626
633
|
- lib/gooddata/commands/base.rb
|
|
@@ -710,6 +717,7 @@ files:
|
|
|
710
717
|
- lib/gooddata/lcm/actions/purge_clients.rb
|
|
711
718
|
- lib/gooddata/lcm/actions/rename_existing_client_projects.rb
|
|
712
719
|
- lib/gooddata/lcm/actions/segments_filter.rb
|
|
720
|
+
- lib/gooddata/lcm/actions/set_master_project.rb
|
|
713
721
|
- lib/gooddata/lcm/actions/synchronize_attribute_drillpaths.rb
|
|
714
722
|
- lib/gooddata/lcm/actions/synchronize_cas.rb
|
|
715
723
|
- lib/gooddata/lcm/actions/synchronize_clients.rb
|