gooddata 2.1.9 → 2.1.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -0
- data/.travis.yml +1 -3
- data/CHANGELOG.md +24 -0
- data/Dockerfile +12 -4
- data/SDK_VERSION +1 -1
- data/VERSION +1 -1
- data/ci/bigquery/pom.xml +54 -0
- data/ci/redshift/pom.xml +73 -0
- data/ci/snowflake/pom.xml +57 -0
- data/lcm.rake +10 -6
- data/lib/gooddata/cloud_resources/bigquery/bigquery_client.rb +86 -0
- data/lib/gooddata/cloud_resources/bigquery/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/redshift/redshift_client.rb +3 -2
- data/lib/gooddata/cloud_resources/snowflake/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +84 -0
- data/lib/gooddata/helpers/data_helper.rb +1 -1
- data/lib/gooddata/lcm/actions/collect_clients.rb +6 -6
- data/lib/gooddata/lcm/actions/collect_dynamic_schedule_params.rb +6 -6
- data/lib/gooddata/lcm/actions/collect_users_brick_users.rb +7 -6
- data/lib/gooddata/lcm/actions/set_master_project.rb +76 -0
- data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +2 -2
- data/lib/gooddata/lcm/actions/synchronize_users.rb +31 -30
- data/lib/gooddata/lcm/lcm2.rb +21 -1
- data/lib/gooddata/models/domain.rb +16 -14
- data/lib/gooddata/models/project.rb +19 -17
- metadata +10 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 911d520db2175b4916716c0c72680eb1008805d4d93b4fc88aa1be824228d908
|
|
4
|
+
data.tar.gz: 8fb1fbdd9224f65fe507e4286e71bdb809ed3b14dc161b6ed4ae01d234a0c953
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ad36dd5b4d73e251bb4244e8360afefdd11a762a41316bfe4a8af93a263c1272ac14e3a2c9f82443be3efb7cac9b95f20c9fd5c1318b088b9346b8b300caeecb
|
|
7
|
+
data.tar.gz: 4e86ebf472e756d13e85ce222def94d9bd5ba2ba0f8adc4cc0a03b66bb94c31ab3c2066bb60c906a604f1e64232ae6e16e948d2c8ea072bf3a5ed1c786470110
|
data/.rubocop.yml
CHANGED
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
|
@@ -1,4 +1,28 @@
|
|
|
1
1
|
# GoodData Ruby SDK Changelog
|
|
2
|
+
## 2.1.10
|
|
3
|
+
- BUGFIX: TMA-1653 fix performance issue in functions project.users and domain.users
|
|
4
|
+
- BUGFIX: TMA-1643 Don't convert null value to empty string
|
|
5
|
+
- BUGFIX: TMA-1620 Users Brick, sometimes update users are conflicted
|
|
6
|
+
- BUGFIX: TMA-1642 parse csv from input source with case insensitive
|
|
7
|
+
- BUGFIX: TMA-1528: remove CollectSegments and CollectDataProduct for add and remove in users brick
|
|
8
|
+
- FEATURE: TMA-1629 Add parameter "set_master_project" to support reset latest master project
|
|
9
|
+
- FEATURE: TMA-1630 Support Snowflake, BigQuery as input source
|
|
10
|
+
## 2.1.9
|
|
11
|
+
- FEATURE: TMA-1076 support new version api 2
|
|
12
|
+
- BUGFIX: TMA-1637 handle input_source of dynamic params
|
|
13
|
+
- BUGFIX: TMA-1636 Build csv file with force_quotes
|
|
14
|
+
- FEATURE: TMA-1614 Support redshift input source
|
|
15
|
+
- FEATURE: TMA-1259 Start using dataproduct in NFS release table
|
|
16
|
+
- FEATURE: MSF-16455 support yellow WARNING status into RubySDK
|
|
17
|
+
- CONFIG: TMA-1625 update version lock for test docker images
|
|
18
|
+
- BUGFIX: TMA-1602 User filter brick failed - K8s bricks don't show error properly
|
|
19
|
+
- BUGFIX: TMA-1593 Increase java heap space during execute bricks
|
|
20
|
+
- BUGFIX: TMA-1558 K8s bricks don't show error when config invalid
|
|
21
|
+
- BUGFIX: TMA-1596 The error propagated from a LCM brick is shown multiple times in the log
|
|
22
|
+
- BUGFIX: TMA-1582 show synchronize ldm mode is running
|
|
23
|
+
- FEATURE: TMA-1588 support schedule param include deprecated
|
|
24
|
+
- FEATURE: TMA-1597 Logging lcm execution result
|
|
25
|
+
|
|
2
26
|
## 2.1.8
|
|
3
27
|
- FEATURE: TMA-1604 Upgrade Restforce version to 3.x
|
|
4
28
|
|
data/Dockerfile
CHANGED
|
@@ -43,16 +43,24 @@ RUN groupadd -g 48 apache \
|
|
|
43
43
|
USER apache
|
|
44
44
|
|
|
45
45
|
ADD ./bin ./bin
|
|
46
|
+
ADD --chown=apache:apache ./ci ./ci
|
|
46
47
|
ADD --chown=apache:apache ./lib ./lib
|
|
47
48
|
ADD ./SDK_VERSION .
|
|
48
49
|
ADD ./VERSION .
|
|
49
50
|
ADD ./Gemfile .
|
|
50
51
|
ADD ./gooddata.gemspec .
|
|
51
52
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
RUN
|
|
55
|
-
|
|
53
|
+
#build redshift dependencies
|
|
54
|
+
RUN mvn -f ci/redshift/pom.xml clean install -P binary-packaging
|
|
55
|
+
RUN cp -rf ci/redshift/target/*.jar ./lib/gooddata/cloud_resources/redshift/drivers/
|
|
56
|
+
|
|
57
|
+
#build snowflake dependencies
|
|
58
|
+
RUN mvn -f ci/snowflake/pom.xml clean install -P binary-packaging
|
|
59
|
+
RUN cp -rf ci/snowflake/target/*.jar ./lib/gooddata/cloud_resources/snowflake/drivers/
|
|
60
|
+
|
|
61
|
+
#build bigquery dependencies
|
|
62
|
+
RUN mvn -f ci/bigquery/pom.xml clean install -P binary-packaging
|
|
63
|
+
RUN cp -rf ci/bigquery/target/*.jar ./lib/gooddata/cloud_resources/bigquery/drivers/
|
|
56
64
|
|
|
57
65
|
RUN bundle install
|
|
58
66
|
|
data/SDK_VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
2.1.
|
|
1
|
+
2.1.10
|
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
3.7.
|
|
1
|
+
3.7.17
|
data/ci/bigquery/pom.xml
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
|
3
|
+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
4
|
+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
5
|
+
<modelVersion>4.0.0</modelVersion>
|
|
6
|
+
|
|
7
|
+
<groupId>com.gooddata.lcm</groupId>
|
|
8
|
+
<artifactId>lcm-bigquery-driver</artifactId>
|
|
9
|
+
<version>1.0-SNAPSHOT</version>
|
|
10
|
+
|
|
11
|
+
<dependencies>
|
|
12
|
+
<dependency>
|
|
13
|
+
<groupId>com.google.auth</groupId>
|
|
14
|
+
<artifactId>google-auth-library-oauth2-http</artifactId>
|
|
15
|
+
<version>0.16.2</version>
|
|
16
|
+
</dependency>
|
|
17
|
+
<dependency>
|
|
18
|
+
<groupId>org.apache.commons</groupId>
|
|
19
|
+
<artifactId>commons-text</artifactId>
|
|
20
|
+
<version>1.7</version>
|
|
21
|
+
</dependency>
|
|
22
|
+
<dependency>
|
|
23
|
+
<groupId>com.google.cloud</groupId>
|
|
24
|
+
<artifactId>google-cloud-bigquery</artifactId>
|
|
25
|
+
<version>1.102.0</version>
|
|
26
|
+
</dependency>
|
|
27
|
+
</dependencies>
|
|
28
|
+
|
|
29
|
+
<profiles>
|
|
30
|
+
<profile>
|
|
31
|
+
<id>binary-packaging</id>
|
|
32
|
+
<build>
|
|
33
|
+
<plugins>
|
|
34
|
+
<plugin>
|
|
35
|
+
<artifactId>maven-dependency-plugin</artifactId>
|
|
36
|
+
<executions>
|
|
37
|
+
<execution>
|
|
38
|
+
<phase>package</phase>
|
|
39
|
+
<goals>
|
|
40
|
+
<goal>copy-dependencies</goal>
|
|
41
|
+
</goals>
|
|
42
|
+
<configuration>
|
|
43
|
+
<outputDirectory>${project.build.directory}</outputDirectory>
|
|
44
|
+
<!-- compile scope gives runtime and compile dependencies (skips test deps) -->
|
|
45
|
+
<includeScope>runtime</includeScope>
|
|
46
|
+
</configuration>
|
|
47
|
+
</execution>
|
|
48
|
+
</executions>
|
|
49
|
+
</plugin>
|
|
50
|
+
</plugins>
|
|
51
|
+
</build>
|
|
52
|
+
</profile>
|
|
53
|
+
</profiles>
|
|
54
|
+
</project>
|
data/ci/redshift/pom.xml
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
|
3
|
+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
4
|
+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
5
|
+
<modelVersion>4.0.0</modelVersion>
|
|
6
|
+
|
|
7
|
+
<groupId>com.gooddata.lcm</groupId>
|
|
8
|
+
<artifactId>lcm-redshift-driver</artifactId>
|
|
9
|
+
<version>1.0-SNAPSHOT</version>
|
|
10
|
+
|
|
11
|
+
<dependencies>
|
|
12
|
+
<dependency>
|
|
13
|
+
<groupId>com.amazon.redshift</groupId>
|
|
14
|
+
<artifactId>redshift-jdbc42-no-awssdk</artifactId>
|
|
15
|
+
<version>1.2.36.1060</version>
|
|
16
|
+
</dependency>
|
|
17
|
+
<dependency>
|
|
18
|
+
<groupId>com.amazonaws</groupId>
|
|
19
|
+
<artifactId>aws-java-sdk-core</artifactId>
|
|
20
|
+
<version>1.11.633</version>
|
|
21
|
+
</dependency>
|
|
22
|
+
<dependency>
|
|
23
|
+
<groupId>com.amazonaws</groupId>
|
|
24
|
+
<artifactId>aws-java-sdk-redshift</artifactId>
|
|
25
|
+
<version>1.11.633</version>
|
|
26
|
+
</dependency>
|
|
27
|
+
<dependency>
|
|
28
|
+
<groupId>com.amazonaws</groupId>
|
|
29
|
+
<artifactId>aws-java-sdk-sts</artifactId>
|
|
30
|
+
<version>1.11.633</version>
|
|
31
|
+
</dependency>
|
|
32
|
+
<dependency>
|
|
33
|
+
<groupId>log4j</groupId>
|
|
34
|
+
<artifactId>log4j</artifactId>
|
|
35
|
+
<version>1.2.17</version>
|
|
36
|
+
<scope>compile</scope>
|
|
37
|
+
</dependency>
|
|
38
|
+
</dependencies>
|
|
39
|
+
|
|
40
|
+
<profiles>
|
|
41
|
+
<profile>
|
|
42
|
+
<id>binary-packaging</id>
|
|
43
|
+
<build>
|
|
44
|
+
<plugins>
|
|
45
|
+
<plugin>
|
|
46
|
+
<artifactId>maven-dependency-plugin</artifactId>
|
|
47
|
+
<executions>
|
|
48
|
+
<execution>
|
|
49
|
+
<phase>package</phase>
|
|
50
|
+
<goals>
|
|
51
|
+
<goal>copy-dependencies</goal>
|
|
52
|
+
</goals>
|
|
53
|
+
<configuration>
|
|
54
|
+
<outputDirectory>${project.build.directory}</outputDirectory>
|
|
55
|
+
<!-- compile scope gives runtime and compile dependencies (skips test deps) -->
|
|
56
|
+
<includeScope>runtime</includeScope>
|
|
57
|
+
</configuration>
|
|
58
|
+
</execution>
|
|
59
|
+
</executions>
|
|
60
|
+
</plugin>
|
|
61
|
+
</plugins>
|
|
62
|
+
</build>
|
|
63
|
+
</profile>
|
|
64
|
+
</profiles>
|
|
65
|
+
|
|
66
|
+
<repositories>
|
|
67
|
+
<repository>
|
|
68
|
+
<id>my-repo1</id>
|
|
69
|
+
<name>my custom repo</name>
|
|
70
|
+
<url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
|
|
71
|
+
</repository>
|
|
72
|
+
</repositories>
|
|
73
|
+
</project>
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
|
3
|
+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
4
|
+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
5
|
+
<modelVersion>4.0.0</modelVersion>
|
|
6
|
+
|
|
7
|
+
<groupId>com.gooddata.lcm</groupId>
|
|
8
|
+
<artifactId>lcm-snowflake-driver</artifactId>
|
|
9
|
+
<version>1.0-SNAPSHOT</version>
|
|
10
|
+
|
|
11
|
+
<dependencies>
|
|
12
|
+
<dependency>
|
|
13
|
+
<groupId>net.snowflake</groupId>
|
|
14
|
+
<artifactId>snowflake-jdbc</artifactId>
|
|
15
|
+
<version>3.6.22</version>
|
|
16
|
+
</dependency>
|
|
17
|
+
<dependency>
|
|
18
|
+
<groupId>org.slf4j</groupId>
|
|
19
|
+
<artifactId>slf4j-api</artifactId>
|
|
20
|
+
<version>1.7.2</version>
|
|
21
|
+
</dependency>
|
|
22
|
+
</dependencies>
|
|
23
|
+
|
|
24
|
+
<profiles>
|
|
25
|
+
<profile>
|
|
26
|
+
<id>binary-packaging</id>
|
|
27
|
+
<build>
|
|
28
|
+
<plugins>
|
|
29
|
+
<plugin>
|
|
30
|
+
<artifactId>maven-dependency-plugin</artifactId>
|
|
31
|
+
<executions>
|
|
32
|
+
<execution>
|
|
33
|
+
<phase>package</phase>
|
|
34
|
+
<goals>
|
|
35
|
+
<goal>copy-dependencies</goal>
|
|
36
|
+
</goals>
|
|
37
|
+
<configuration>
|
|
38
|
+
<outputDirectory>${project.build.directory}</outputDirectory>
|
|
39
|
+
<!-- compile scope gives runtime and compile dependencies (skips test deps) -->
|
|
40
|
+
<includeScope>runtime</includeScope>
|
|
41
|
+
</configuration>
|
|
42
|
+
</execution>
|
|
43
|
+
</executions>
|
|
44
|
+
</plugin>
|
|
45
|
+
</plugins>
|
|
46
|
+
</build>
|
|
47
|
+
</profile>
|
|
48
|
+
</profiles>
|
|
49
|
+
|
|
50
|
+
<repositories>
|
|
51
|
+
<repository>
|
|
52
|
+
<id>my-repo1</id>
|
|
53
|
+
<name>my custom repo</name>
|
|
54
|
+
<url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
|
|
55
|
+
</repository>
|
|
56
|
+
</repositories>
|
|
57
|
+
</project>
|
data/lcm.rake
CHANGED
|
@@ -118,7 +118,7 @@ end
|
|
|
118
118
|
namespace :docker do
|
|
119
119
|
desc 'Build Docker image'
|
|
120
120
|
task :build do
|
|
121
|
-
Rake::Task["maven:
|
|
121
|
+
Rake::Task["maven:build_dependencies"].invoke
|
|
122
122
|
system('docker build -f Dockerfile.jruby -t gooddata/appstore .')
|
|
123
123
|
end
|
|
124
124
|
|
|
@@ -129,11 +129,15 @@ namespace :docker do
|
|
|
129
129
|
end
|
|
130
130
|
|
|
131
131
|
namespace :maven do
|
|
132
|
-
task :
|
|
133
|
-
system(
|
|
134
|
-
system('
|
|
135
|
-
|
|
136
|
-
system('
|
|
132
|
+
task :build_dependencies do
|
|
133
|
+
system('mvn -f ci/snowflake/pom.xml clean install -P binary-packaging')
|
|
134
|
+
system('cp -rf ci/snowflake/target/*.jar lib/gooddata/cloud_resources/snowflake/drivers/')
|
|
135
|
+
|
|
136
|
+
system('mvn -f ci/bigquery/pom.xml clean install -P binary-packaging')
|
|
137
|
+
system('cp -rf ci/bigquery/target/*.jar lib/gooddata/cloud_resources/bigquery/drivers/')
|
|
138
|
+
|
|
139
|
+
system('mvn -f ci/redshift/pom.xml clean install -P binary-packaging')
|
|
140
|
+
system('cp -rf ci/redshift/target/*.jar lib/gooddata/cloud_resources/redshift/drivers/')
|
|
137
141
|
end
|
|
138
142
|
end
|
|
139
143
|
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# encoding: UTF-8
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
#
|
|
4
|
+
# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
|
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
|
7
|
+
|
|
8
|
+
require 'securerandom'
|
|
9
|
+
require 'java'
|
|
10
|
+
require 'pathname'
|
|
11
|
+
require_relative '../cloud_resource_client'
|
|
12
|
+
|
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
|
15
|
+
require file unless file.start_with?('lcm-bigquery-driver')
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
java_import 'com.google.auth.oauth2.ServiceAccountCredentials'
|
|
19
|
+
java_import 'com.google.cloud.bigquery.BigQuery'
|
|
20
|
+
java_import 'com.google.cloud.bigquery.BigQueryOptions'
|
|
21
|
+
java_import 'com.google.cloud.bigquery.FieldList'
|
|
22
|
+
java_import 'com.google.cloud.bigquery.FieldValue'
|
|
23
|
+
java_import 'com.google.cloud.bigquery.FieldValueList'
|
|
24
|
+
java_import 'com.google.cloud.bigquery.QueryJobConfiguration'
|
|
25
|
+
java_import 'com.google.cloud.bigquery.TableResult'
|
|
26
|
+
java_import 'org.apache.commons.text.StringEscapeUtils'
|
|
27
|
+
|
|
28
|
+
module GoodData
|
|
29
|
+
module CloudResources
|
|
30
|
+
class BigQueryClient < CloudResourceClient
|
|
31
|
+
class << self
|
|
32
|
+
def accept?(type)
|
|
33
|
+
type == 'bigquery'
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def initialize(options = {})
|
|
38
|
+
raise("Data Source needs a client to BigQuery to be able to query the storage but 'bigquery_client' is empty.") unless options['bigquery_client']
|
|
39
|
+
|
|
40
|
+
if options['bigquery_client']['connection'].is_a?(Hash)
|
|
41
|
+
@project = options['bigquery_client']['connection']['project']
|
|
42
|
+
@schema = options['bigquery_client']['connection']['schema'] || 'public'
|
|
43
|
+
@authentication = options['bigquery_client']['connection']['authentication']
|
|
44
|
+
else
|
|
45
|
+
raise('Missing connection info for BigQuery client')
|
|
46
|
+
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def realize_query(query, _params)
|
|
51
|
+
GoodData.gd_logger.info("Realize SQL query: type=bigquery status=started")
|
|
52
|
+
|
|
53
|
+
client = create_client
|
|
54
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
|
55
|
+
measure = Benchmark.measure do
|
|
56
|
+
query_config = QueryJobConfiguration.newBuilder(query).setDefaultDataset(@schema).build
|
|
57
|
+
table_result = client.query(query_config)
|
|
58
|
+
|
|
59
|
+
if table_result.getTotalRows.positive?
|
|
60
|
+
result = table_result.iterateAll
|
|
61
|
+
field_list = table_result.getSchema.getFields
|
|
62
|
+
col_count = field_list.size
|
|
63
|
+
CSV.open(filename, 'wb') do |csv|
|
|
64
|
+
csv << Array(1..col_count).map { |i| field_list.get(i - 1).getName } # build the header
|
|
65
|
+
result.each do |row|
|
|
66
|
+
csv << Array(1..col_count).map { |i| row.get(i - 1).getValue&.to_s }
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
GoodData.gd_logger.info("Realize SQL query: type=bigquery status=finished duration=#{measure.real}")
|
|
72
|
+
filename
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
private
|
|
76
|
+
|
|
77
|
+
def create_client
|
|
78
|
+
GoodData.logger.info "Setting up connection to BigQuery"
|
|
79
|
+
client_email = @authentication['serviceAccount']['clientEmail']
|
|
80
|
+
private_key = @authentication['serviceAccount']['privateKey']
|
|
81
|
+
credentials = ServiceAccountCredentials.fromPkcs8(nil, client_email, StringEscapeUtils.unescapeJson(private_key), nil, nil)
|
|
82
|
+
BigQueryOptions.newBuilder.setProjectId(@project).setCredentials(credentials).build.getService
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
File without changes
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
# encoding: UTF-8
|
|
2
|
+
# frozen_string_literal: true
|
|
2
3
|
#
|
|
3
4
|
# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
|
|
4
5
|
# This source code is licensed under the BSD-style license found in the
|
|
@@ -57,9 +58,9 @@ module GoodData
|
|
|
57
58
|
result = statement.get_result_set
|
|
58
59
|
metadata = result.get_meta_data
|
|
59
60
|
col_count = metadata.column_count
|
|
60
|
-
CSV.open(filename, 'wb'
|
|
61
|
+
CSV.open(filename, 'wb') do |csv|
|
|
61
62
|
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
|
62
|
-
csv << Array(1..col_count).map { |i| result.get_string(i) } while result.next
|
|
63
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
|
63
64
|
end
|
|
64
65
|
end
|
|
65
66
|
end
|
|
File without changes
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# encoding: UTF-8
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
#
|
|
4
|
+
# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
|
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
|
7
|
+
|
|
8
|
+
require 'securerandom'
|
|
9
|
+
require 'java'
|
|
10
|
+
require 'pathname'
|
|
11
|
+
require_relative '../cloud_resource_client'
|
|
12
|
+
|
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
|
15
|
+
require file unless file.start_with?('lcm-snowflake-driver')
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
module GoodData
|
|
19
|
+
module CloudResources
|
|
20
|
+
class SnowflakeClient < CloudResourceClient
|
|
21
|
+
class << self
|
|
22
|
+
def accept?(type)
|
|
23
|
+
type == 'snowflake'
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def initialize(options = {})
|
|
28
|
+
raise("Data Source needs a client to Snowflake to be able to query the storage but 'snowflake_client' is empty.") unless options['snowflake_client']
|
|
29
|
+
|
|
30
|
+
if options['snowflake_client']['connection'].is_a?(Hash)
|
|
31
|
+
@database = options['snowflake_client']['connection']['database']
|
|
32
|
+
@schema = options['snowflake_client']['connection']['schema'] || 'public'
|
|
33
|
+
@warehouse = options['snowflake_client']['connection']['warehouse']
|
|
34
|
+
@url = options['snowflake_client']['connection']['url']
|
|
35
|
+
@authentication = options['snowflake_client']['connection']['authentication']
|
|
36
|
+
else
|
|
37
|
+
raise('Missing connection info for Snowflake client')
|
|
38
|
+
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
Java.net.snowflake.client.jdbc.SnowflakeDriver
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def realize_query(query, _params)
|
|
45
|
+
GoodData.gd_logger.info("Realize SQL query: type=snowflake status=started")
|
|
46
|
+
|
|
47
|
+
connect
|
|
48
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
|
49
|
+
measure = Benchmark.measure do
|
|
50
|
+
statement = @connection.create_statement
|
|
51
|
+
|
|
52
|
+
has_result = statement.execute(query)
|
|
53
|
+
if has_result
|
|
54
|
+
result = statement.get_result_set
|
|
55
|
+
metadata = result.get_meta_data
|
|
56
|
+
col_count = metadata.column_count
|
|
57
|
+
CSV.open(filename, 'wb') do |csv|
|
|
58
|
+
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
|
59
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
GoodData.gd_logger.info("Realize SQL query: type=snowflake status=finished duration=#{measure.real}")
|
|
64
|
+
filename
|
|
65
|
+
ensure
|
|
66
|
+
@connection&.close
|
|
67
|
+
@connection = nil
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def connect
|
|
71
|
+
GoodData.logger.info "Setting up connection to Snowflake #{@url}"
|
|
72
|
+
|
|
73
|
+
prop = java.util.Properties.new
|
|
74
|
+
prop.setProperty('user', @authentication['basic']['userName'])
|
|
75
|
+
prop.setProperty('password', @authentication['basic']['password'])
|
|
76
|
+
prop.setProperty('schema', @schema)
|
|
77
|
+
prop.setProperty('warehouse', @warehouse)
|
|
78
|
+
prop.setProperty('db', @database)
|
|
79
|
+
|
|
80
|
+
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
@@ -44,7 +44,7 @@ module GoodData
|
|
|
44
44
|
realize_link
|
|
45
45
|
when 's3'
|
|
46
46
|
realize_s3(params)
|
|
47
|
-
when 'redshift'
|
|
47
|
+
when 'redshift', 'snowflake', 'bigquery'
|
|
48
48
|
raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
|
|
49
49
|
|
|
50
50
|
require_relative '../cloud_resources/cloud_resources'
|
|
@@ -67,11 +67,11 @@ module GoodData
|
|
|
67
67
|
end
|
|
68
68
|
|
|
69
69
|
def collect_clients(params, segment_names = nil)
|
|
70
|
-
client_id_column = params.client_id_column || 'client_id'
|
|
71
|
-
segment_id_column = params.segment_id_column || 'segment_id'
|
|
72
|
-
project_id_column = params.project_id_column || 'project_id'
|
|
73
|
-
project_title_column = params.project_title_column || 'project_title'
|
|
74
|
-
project_token_column = params.project_token_column || 'project_token'
|
|
70
|
+
client_id_column = params.client_id_column&.downcase || 'client_id'
|
|
71
|
+
segment_id_column = params.segment_id_column&.downcase || 'segment_id'
|
|
72
|
+
project_id_column = params.project_id_column&.downcase || 'project_id'
|
|
73
|
+
project_title_column = params.project_title_column&.downcase || 'project_title'
|
|
74
|
+
project_token_column = params.project_token_column&.downcase || 'project_token'
|
|
75
75
|
client = params.gdc_gd_client
|
|
76
76
|
|
|
77
77
|
clients = []
|
|
@@ -82,7 +82,7 @@ module GoodData
|
|
|
82
82
|
end
|
|
83
83
|
GoodData.logger.debug("Input data: #{input_data.read}")
|
|
84
84
|
GoodData.logger.debug("Segment names: #{segment_names}")
|
|
85
|
-
CSV.foreach(input_data, :headers => true, :return_headers => false,
|
|
85
|
+
CSV.foreach(input_data, :headers => true, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
|
|
86
86
|
GoodData.logger.debug("Processing row: #{row}")
|
|
87
87
|
segment_name = row[segment_id_column]
|
|
88
88
|
GoodData.logger.debug("Segment name: #{segment_name}")
|
|
@@ -38,11 +38,11 @@ module GoodData
|
|
|
38
38
|
def call(params)
|
|
39
39
|
return [] unless params.dynamic_params
|
|
40
40
|
|
|
41
|
-
schedule_title_column = params.schedule_title_column || 'schedule_title'
|
|
42
|
-
client_id_column = params.client_id_column || 'client_id'
|
|
43
|
-
param_name_column = params.param_name_column || 'param_name'
|
|
44
|
-
param_value_column = params.param_value_column || 'param_value'
|
|
45
|
-
param_secure_column = params.param_secure_column || 'param_secure'
|
|
41
|
+
schedule_title_column = params.schedule_title_column&.downcase || 'schedule_title'
|
|
42
|
+
client_id_column = params.client_id_column&.downcase || 'client_id'
|
|
43
|
+
param_name_column = params.param_name_column&.downcase || 'param_name'
|
|
44
|
+
param_value_column = params.param_value_column&.downcase || 'param_value'
|
|
45
|
+
param_secure_column = params.param_secure_column&.downcase || 'param_secure'
|
|
46
46
|
|
|
47
47
|
encryption_key = params.dynamic_params_encryption_key || ''
|
|
48
48
|
exist_encryption_key = encryption_key.blank? ? false : true
|
|
@@ -59,7 +59,7 @@ module GoodData
|
|
|
59
59
|
schedule_hidden_params = {}
|
|
60
60
|
exist_param_secure = false
|
|
61
61
|
|
|
62
|
-
CSV.foreach(input_data, :headers => true, :return_headers => false,
|
|
62
|
+
CSV.foreach(input_data, :headers => true, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
|
|
63
63
|
is_param_secure = row[param_secure_column] == 'true'
|
|
64
64
|
is_decrypt_secure_value = is_param_secure && exist_encryption_key ? true : false
|
|
65
65
|
exist_param_secure = true if is_param_secure
|
|
@@ -35,7 +35,7 @@ module GoodData
|
|
|
35
35
|
class << self
|
|
36
36
|
def call(params)
|
|
37
37
|
users_brick_users = []
|
|
38
|
-
login_column = params.users_brick_config.login_column || 'login'
|
|
38
|
+
login_column = params.users_brick_config.login_column&.downcase || 'login'
|
|
39
39
|
users_brick_data_source = GoodData::Helpers::DataSource.new(params.users_brick_config.input_source)
|
|
40
40
|
|
|
41
41
|
users_brick_data_source_file = without_check(PARAMS, params) do
|
|
@@ -45,14 +45,15 @@ module GoodData
|
|
|
45
45
|
)
|
|
46
46
|
end
|
|
47
47
|
CSV.foreach(users_brick_data_source_file,
|
|
48
|
-
headers
|
|
49
|
-
return_headers
|
|
50
|
-
|
|
51
|
-
|
|
48
|
+
:headers => true,
|
|
49
|
+
:return_headers => false,
|
|
50
|
+
:header_converters => :downcase,
|
|
51
|
+
:encoding => 'utf-8') do |row|
|
|
52
|
+
pid = row[params.multiple_projects_column&.downcase]
|
|
52
53
|
fail "The set multiple_projects_column '#{params.multiple_projects_column}' of the users input is empty" if !pid && MULTIPLE_COLUMN_MODES.include?(params.sync_mode)
|
|
53
54
|
|
|
54
55
|
users_brick_users << {
|
|
55
|
-
login: row[login_column].downcase,
|
|
56
|
+
login: row[login_column].nil? ? nil : row[login_column].strip.downcase,
|
|
56
57
|
pid: pid
|
|
57
58
|
}
|
|
58
59
|
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# encoding: UTF-8
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
#
|
|
4
|
+
# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
|
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
|
7
|
+
|
|
8
|
+
require_relative 'base_action'
|
|
9
|
+
|
|
10
|
+
module GoodData
|
|
11
|
+
module LCM2
|
|
12
|
+
class SetMasterProject < BaseAction
|
|
13
|
+
DESCRIPTION = 'Set master project'
|
|
14
|
+
|
|
15
|
+
PARAMS = define_params(self) do
|
|
16
|
+
description 'Organization Name'
|
|
17
|
+
param :organization, instance_of(Type::StringType), required: false
|
|
18
|
+
|
|
19
|
+
description 'Domain'
|
|
20
|
+
param :domain, instance_of(Type::StringType), required: false
|
|
21
|
+
|
|
22
|
+
description 'ADS Client'
|
|
23
|
+
param :ads_client, instance_of(Type::AdsClientType), required: false
|
|
24
|
+
|
|
25
|
+
description 'Table Name'
|
|
26
|
+
param :release_table_name, instance_of(Type::StringType), required: false
|
|
27
|
+
|
|
28
|
+
description 'Segments to manage'
|
|
29
|
+
param :segments, array_of(instance_of(Type::SegmentType)), required: true
|
|
30
|
+
|
|
31
|
+
description 'DataProduct to manage'
|
|
32
|
+
param :data_product, instance_of(Type::GDDataProductType), required: false
|
|
33
|
+
|
|
34
|
+
description 'Released master project should be used in next rollout'
|
|
35
|
+
param :set_master_project, instance_of(Type::StringType), required: false
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
class << self
|
|
39
|
+
def call(params)
|
|
40
|
+
results = []
|
|
41
|
+
domain_name = params.organization || params.domain
|
|
42
|
+
data_product = params.data_product
|
|
43
|
+
params.segments.each do |segment_in|
|
|
44
|
+
version = get_latest_version(params, domain_name, data_product.data_product_id, segment_in.segment_id) + 1
|
|
45
|
+
segment_in[:data_product_id] = data_product.data_product_id
|
|
46
|
+
segment_in[:master_pid] = params.set_master_project
|
|
47
|
+
segment_in[:version] = version
|
|
48
|
+
segment_in[:timestamp] = Time.now.utc.iso8601
|
|
49
|
+
|
|
50
|
+
results << {
|
|
51
|
+
data_product_id: data_product.data_product_id,
|
|
52
|
+
segment_id: segment_in.segment_id,
|
|
53
|
+
version: version
|
|
54
|
+
}
|
|
55
|
+
end
|
|
56
|
+
results
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def get_latest_version(params, domain_name, data_product_id, segment_id)
|
|
60
|
+
if params.ads_client
|
|
61
|
+
current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
|
|
62
|
+
params.release_table_name,
|
|
63
|
+
params.ads_client,
|
|
64
|
+
segment_id
|
|
65
|
+
)
|
|
66
|
+
else
|
|
67
|
+
current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product_id, segment_id)
|
|
68
|
+
end
|
|
69
|
+
return 0 unless current_master
|
|
70
|
+
|
|
71
|
+
current_master[:version].to_i
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -230,8 +230,8 @@ module GoodData
|
|
|
230
230
|
begin
|
|
231
231
|
GoodData.logger.info('Start reading data')
|
|
232
232
|
row_count = 0
|
|
233
|
-
CSV.foreach(tmp, headers
|
|
234
|
-
filters << row.to_hash.merge(pid: row[multiple_projects_column])
|
|
233
|
+
CSV.foreach(tmp, :headers => csv_with_headers, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
|
|
234
|
+
filters << row.to_hash.merge(pid: row[multiple_projects_column.downcase])
|
|
235
235
|
row_count += 1
|
|
236
236
|
GoodData.logger.info("Read #{row_count} rows") if (row_count % 50_000).zero?
|
|
237
237
|
end
|
|
@@ -347,38 +347,33 @@ module GoodData
|
|
|
347
347
|
end
|
|
348
348
|
|
|
349
349
|
def load_data(params, data_source)
|
|
350
|
-
first_name_column = params.first_name_column || 'first_name'
|
|
351
|
-
last_name_column = params.last_name_column || 'last_name'
|
|
352
|
-
login_column = params.login_column || 'login'
|
|
353
|
-
password_column = params.password_column || 'password'
|
|
354
|
-
email_column = params.email_column || 'email'
|
|
355
|
-
role_column = params.role_column || 'role'
|
|
356
|
-
sso_provider_column = params.sso_provider_column || 'sso_provider'
|
|
357
|
-
authentication_modes_column = params.authentication_modes_column || 'authentication_modes'
|
|
358
|
-
user_groups_column = params.user_groups_column || 'user_groups'
|
|
359
|
-
language_column = params.language_column || 'language'
|
|
360
|
-
company_column = params.company_column || 'company'
|
|
361
|
-
position_column = params.position_column || 'position'
|
|
362
|
-
country_column = params.country_column || 'country'
|
|
363
|
-
phone_column = params.phone_column || 'phone'
|
|
364
|
-
ip_whitelist_column = params.ip_whitelist_column || 'ip_whitelist'
|
|
350
|
+
first_name_column = params.first_name_column&.downcase || 'first_name'
|
|
351
|
+
last_name_column = params.last_name_column&.downcase || 'last_name'
|
|
352
|
+
login_column = params.login_column&.downcase || 'login'
|
|
353
|
+
password_column = params.password_column&.downcase || 'password'
|
|
354
|
+
email_column = params.email_column&.downcase || 'email'
|
|
355
|
+
role_column = params.role_column&.downcase || 'role'
|
|
356
|
+
sso_provider_column = params.sso_provider_column&.downcase || 'sso_provider'
|
|
357
|
+
authentication_modes_column = params.authentication_modes_column&.downcase || 'authentication_modes'
|
|
358
|
+
user_groups_column = params.user_groups_column&.downcase || 'user_groups'
|
|
359
|
+
language_column = params.language_column&.downcase || 'language'
|
|
360
|
+
company_column = params.company_column&.downcase || 'company'
|
|
361
|
+
position_column = params.position_column&.downcase || 'position'
|
|
362
|
+
country_column = params.country_column&.downcase || 'country'
|
|
363
|
+
phone_column = params.phone_column&.downcase || 'phone'
|
|
364
|
+
ip_whitelist_column = params.ip_whitelist_column&.downcase || 'ip_whitelist'
|
|
365
365
|
|
|
366
366
|
sso_provider = params.sso_provider
|
|
367
367
|
authentication_modes = params.authentication_modes || []
|
|
368
368
|
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
else
|
|
373
|
-
tmp = without_check(PARAMS, params) do
|
|
374
|
-
File.open(data_source.realize(params), 'r:UTF-8')
|
|
375
|
-
end
|
|
369
|
+
tmp = without_check(PARAMS, params) do
|
|
370
|
+
File.open(data_source.realize(params), 'r:UTF-8')
|
|
371
|
+
end
|
|
376
372
|
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
end
|
|
373
|
+
begin
|
|
374
|
+
data = read_csv_file(tmp)
|
|
375
|
+
rescue Exception => e # rubocop:disable RescueException
|
|
376
|
+
fail "There was an error during loading users from csv file. Message: #{e.message}. Error: #{e}"
|
|
382
377
|
end
|
|
383
378
|
|
|
384
379
|
data.map do |row|
|
|
@@ -398,12 +393,18 @@ module GoodData
|
|
|
398
393
|
ip_whitelist = row[ip_whitelist_column] || row[ip_whitelist_column.to_sym]
|
|
399
394
|
ip_whitelist = ip_whitelist.split(',').map(&:strip) if ip_whitelist
|
|
400
395
|
|
|
396
|
+
user_login = row[login_column] || row[login_column.to_sym]
|
|
397
|
+
user_login = user_login.strip unless user_login.nil?
|
|
398
|
+
|
|
399
|
+
user_email = row[email_column] || row[login_column] || row[email_column.to_sym] || row[login_column.to_sym]
|
|
400
|
+
user_email = user_email.strip unless user_email.nil?
|
|
401
|
+
|
|
401
402
|
{
|
|
402
403
|
:first_name => row[first_name_column] || row[first_name_column.to_sym],
|
|
403
404
|
:last_name => row[last_name_column] || row[last_name_column.to_sym],
|
|
404
|
-
:login =>
|
|
405
|
+
:login => user_login,
|
|
405
406
|
:password => row[password_column] || row[password_column.to_sym],
|
|
406
|
-
:email =>
|
|
407
|
+
:email => user_email,
|
|
407
408
|
:role => row[role_column] || row[role_column.to_sym],
|
|
408
409
|
:sso_provider => sso_provider || row[sso_provider_column] || row[sso_provider_column.to_sym],
|
|
409
410
|
:authentication_modes => modes,
|
|
@@ -424,7 +425,7 @@ module GoodData
|
|
|
424
425
|
res = []
|
|
425
426
|
row_count = 0
|
|
426
427
|
|
|
427
|
-
CSV.foreach(path, :headers => true) do |row|
|
|
428
|
+
CSV.foreach(path, :headers => true, :header_converters => :downcase, :encoding => 'utf-8') do |row|
|
|
428
429
|
if block_given?
|
|
429
430
|
data = yield row
|
|
430
431
|
else
|
data/lib/gooddata/lcm/lcm2.rb
CHANGED
|
@@ -106,6 +106,14 @@ module GoodData
|
|
|
106
106
|
UpdateReleaseTable
|
|
107
107
|
],
|
|
108
108
|
|
|
109
|
+
release_set_master_project: [
|
|
110
|
+
EnsureReleaseTable,
|
|
111
|
+
CollectDataProduct,
|
|
112
|
+
SegmentsFilter,
|
|
113
|
+
SetMasterProject,
|
|
114
|
+
UpdateReleaseTable
|
|
115
|
+
],
|
|
116
|
+
|
|
109
117
|
provision: [
|
|
110
118
|
EnsureReleaseTable,
|
|
111
119
|
CollectDataProduct,
|
|
@@ -271,8 +279,14 @@ module GoodData
|
|
|
271
279
|
|
|
272
280
|
GoodData.gd_logger.brick = mode
|
|
273
281
|
|
|
282
|
+
final_mode = if params.set_master_project && mode == 'release'
|
|
283
|
+
'release_set_master_project'
|
|
284
|
+
else
|
|
285
|
+
mode
|
|
286
|
+
end
|
|
287
|
+
|
|
274
288
|
# Get actions for mode specified
|
|
275
|
-
actions = get_mode_actions(
|
|
289
|
+
actions = get_mode_actions(final_mode)
|
|
276
290
|
|
|
277
291
|
if params.actions
|
|
278
292
|
actions = params.actions.map do |action|
|
|
@@ -305,6 +319,12 @@ module GoodData
|
|
|
305
319
|
skip_actions.include?(action.name.split('::').last)
|
|
306
320
|
end
|
|
307
321
|
|
|
322
|
+
sync_mode = params.fetch(:sync_mode, nil)
|
|
323
|
+
if mode == 'users' && %w[add_to_organization remove_from_organization].include?(sync_mode)
|
|
324
|
+
actions = actions.reject do |action|
|
|
325
|
+
%w[CollectDataProduct CollectSegments].include?(action.name.split('::').last)
|
|
326
|
+
end
|
|
327
|
+
end
|
|
308
328
|
check_unused_params(actions, params)
|
|
309
329
|
print_action_names(mode, actions)
|
|
310
330
|
|
|
@@ -222,22 +222,24 @@ module GoodData
|
|
|
222
222
|
domain = client.domain(domain)
|
|
223
223
|
if id == :all
|
|
224
224
|
GoodData.logger.warn("Retrieving all users from domain #{domain.name}")
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
end
|
|
232
|
-
|
|
233
|
-
tmp['accountSettings']['items'].each do |user_data|
|
|
234
|
-
user = client.create(GoodData::Profile, user_data)
|
|
235
|
-
y << user if user
|
|
236
|
-
end
|
|
237
|
-
break if tmp['accountSettings']['items'].count < page_limit
|
|
238
|
-
offset += page_limit
|
|
225
|
+
all_users = []
|
|
226
|
+
page_limit = opts[:page_limit] || 1000
|
|
227
|
+
offset = opts[:offset] || 0
|
|
228
|
+
loop do
|
|
229
|
+
begin
|
|
230
|
+
tmp = client(opts).get("#{domain.uri}/users", params: { offset: offset, limit: page_limit })
|
|
239
231
|
end
|
|
232
|
+
|
|
233
|
+
tmp['accountSettings']['items'].each do |user_data|
|
|
234
|
+
user = client.create(GoodData::Profile, user_data)
|
|
235
|
+
all_users << user if user
|
|
236
|
+
end
|
|
237
|
+
break if tmp['accountSettings']['items'].count < page_limit
|
|
238
|
+
|
|
239
|
+
offset += page_limit
|
|
240
240
|
end
|
|
241
|
+
|
|
242
|
+
all_users
|
|
241
243
|
else
|
|
242
244
|
find_user_by_login(domain, id)
|
|
243
245
|
end
|
|
@@ -1546,26 +1546,28 @@ module GoodData
|
|
|
1546
1546
|
# @return [Array<GoodData::User>] List of users
|
|
1547
1547
|
def users(opts = {})
|
|
1548
1548
|
client = client(opts)
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
end
|
|
1549
|
+
all_users = []
|
|
1550
|
+
offset = opts[:offset] || 0
|
|
1551
|
+
limit = opts[:limit] || 1_000
|
|
1552
|
+
loop do
|
|
1553
|
+
tmp = client.get("/gdc/projects/#{pid}/users", params: { offset: offset, limit: limit })
|
|
1554
|
+
tmp['users'].each do |user_data|
|
|
1555
|
+
user = client.create(GoodData::Membership, user_data, project: self)
|
|
1556
|
+
|
|
1557
|
+
if opts[:all]
|
|
1558
|
+
all_users << user
|
|
1559
|
+
elsif opts[:disabled]
|
|
1560
|
+
all_users << user if user&.disabled?
|
|
1561
|
+
else
|
|
1562
|
+
all_users << user if user&.enabled?
|
|
1564
1563
|
end
|
|
1565
|
-
break if tmp['users'].count < limit
|
|
1566
|
-
offset += limit
|
|
1567
1564
|
end
|
|
1565
|
+
break if tmp['users'].count < limit
|
|
1566
|
+
|
|
1567
|
+
offset += limit
|
|
1568
1568
|
end
|
|
1569
|
+
|
|
1570
|
+
all_users
|
|
1569
1571
|
end
|
|
1570
1572
|
|
|
1571
1573
|
alias_method :members, :users
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: gooddata
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.1.
|
|
4
|
+
version: 2.1.10
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Pavel Kolesnikov
|
|
@@ -14,7 +14,7 @@ authors:
|
|
|
14
14
|
autorequire:
|
|
15
15
|
bindir: bin
|
|
16
16
|
cert_chain: []
|
|
17
|
-
date: 2020-
|
|
17
|
+
date: 2020-05-08 00:00:00.000000000 Z
|
|
18
18
|
dependencies:
|
|
19
19
|
- !ruby/object:Gem::Dependency
|
|
20
20
|
name: license_finder
|
|
@@ -630,6 +630,9 @@ files:
|
|
|
630
630
|
- bin/user_filters.sh
|
|
631
631
|
- bin/users.sh
|
|
632
632
|
- ci.rake
|
|
633
|
+
- ci/bigquery/pom.xml
|
|
634
|
+
- ci/redshift/pom.xml
|
|
635
|
+
- ci/snowflake/pom.xml
|
|
633
636
|
- data/2008.crt
|
|
634
637
|
- data/new_ca.cer
|
|
635
638
|
- data/new_prodgdc_ca.crt
|
|
@@ -686,11 +689,15 @@ files:
|
|
|
686
689
|
- lib/gooddata/cli/shared.rb
|
|
687
690
|
- lib/gooddata/cli/terminal.rb
|
|
688
691
|
- lib/gooddata/client.rb
|
|
692
|
+
- lib/gooddata/cloud_resources/bigquery/bigquery_client.rb
|
|
693
|
+
- lib/gooddata/cloud_resources/bigquery/drivers/.gitkeepme
|
|
689
694
|
- lib/gooddata/cloud_resources/cloud_resouce_factory.rb
|
|
690
695
|
- lib/gooddata/cloud_resources/cloud_resource_client.rb
|
|
691
696
|
- lib/gooddata/cloud_resources/cloud_resources.rb
|
|
692
697
|
- lib/gooddata/cloud_resources/redshift/drivers/log4j.properties
|
|
693
698
|
- lib/gooddata/cloud_resources/redshift/redshift_client.rb
|
|
699
|
+
- lib/gooddata/cloud_resources/snowflake/drivers/.gitkeepme
|
|
700
|
+
- lib/gooddata/cloud_resources/snowflake/snowflake_client.rb
|
|
694
701
|
- lib/gooddata/commands/api.rb
|
|
695
702
|
- lib/gooddata/commands/auth.rb
|
|
696
703
|
- lib/gooddata/commands/base.rb
|
|
@@ -780,6 +787,7 @@ files:
|
|
|
780
787
|
- lib/gooddata/lcm/actions/purge_clients.rb
|
|
781
788
|
- lib/gooddata/lcm/actions/rename_existing_client_projects.rb
|
|
782
789
|
- lib/gooddata/lcm/actions/segments_filter.rb
|
|
790
|
+
- lib/gooddata/lcm/actions/set_master_project.rb
|
|
783
791
|
- lib/gooddata/lcm/actions/synchronize_attribute_drillpaths.rb
|
|
784
792
|
- lib/gooddata/lcm/actions/synchronize_cas.rb
|
|
785
793
|
- lib/gooddata/lcm/actions/synchronize_clients.rb
|