gooddata 2.1.9-java → 2.1.10-java

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cc6c846aa117ae4e5809d836ccdd2e0a9b5bd0a223837328c3f0e07065ee6b0f
4
- data.tar.gz: d77bc24dd2ce6f01a6c0685cf58d099117a442113dfbb6cfb1c615d71af033a3
3
+ metadata.gz: 00e8b6ecab4ffb8da4f9904c24a4d6fe9e5d4a5144da33a41539a82412494756
4
+ data.tar.gz: f4f0aec0b4d7d38f5a63ee742e5e4eb37104aa91f7bfcbc7e95060eaa98e83b5
5
5
  SHA512:
6
- metadata.gz: 36a5cae65532cae267891feecd76562e83082c2f812201577c30a42a97d93ddcf2689f4f35166089e9262d9afd08f6564c114e6ad769081a3422909d26b5b8ec
7
- data.tar.gz: e77339f7f2a67fa32213aa42619e71ac85f3495a1c9f1110458f71eb15e825e0014c5b9f06c72f85311ad0dcfad4b6dee0700464f6088350beee94ec02256de7
6
+ metadata.gz: 11a8f24cb89b5e2832d58ce34b5e209a90f820c8b50a0b68355d8938dff2bab9bece0c251e5688ddfaa7ef4838ebc74e7ebc975966a67065fde3c0da72851bbe
7
+ data.tar.gz: 2173449f161e26e975f7a46aadc35e4d4b546facaaf67b9ff3a6e43286ecabe287f9cc6962b03e2d49066835a1e5d6b08e15d40c55a79f793d7c7e327658c576
@@ -2,6 +2,7 @@ AllCops:
2
2
  DisplayCopNames: true
3
3
  Exclude:
4
4
  - 'spec/helpers/appstore_project_helper.rb'
5
+ TargetRubyVersion: 2.3
5
6
 
6
7
  AbcSize:
7
8
  Enabled: false
@@ -15,11 +15,9 @@ stages:
15
15
  git:
16
16
  depth: false # this is needed for pronto
17
17
 
18
- matrix:
18
+ jobs:
19
19
  allow_failures:
20
20
  - rvm: jruby-9.1.14
21
-
22
- jobs:
23
21
  include:
24
22
  # BEFORE MERGE
25
23
  - name: pronto code review
@@ -1,4 +1,28 @@
1
1
  # GoodData Ruby SDK Changelog
2
+ ## 2.1.10
3
+ - BUGFIX: TMA-1653 fix performance issue in functions project.users and domain.users
4
+ - BUGFIX: TMA-1643 Don't convert null value to empty string
5
+ - BUGFIX: TMA-1620 Users Brick, sometimes update users are conflicted
6
+ - BUGFIX: TMA-1642 parse csv from input source with case insensitive
7
+ - BUGFIX: TMA-1528: remove CollectSegments and CollectDataProduct for add and remove in users brick
8
+ - FEATURE: TMA-1629 Add parameter "set_master_project" to support reset latest master project
9
+ - FEATURE: TMA-1630 Support Snowflake, BigQuery as input source
10
+ ## 2.1.9
11
+ - FEATURE: TMA-1076 support new version api 2
12
+ - BUGFIX: TMA-1637 handle input_source of dynamic params
13
+ - BUGFIX: TMA-1636 Build csv file with force_quotes
14
+ - FEATURE: TMA-1614 Support redshift input source
15
+ - FEATURE: TMA-1259 Start using dataproduct in NFS release table
16
+ - FEATURE: MSF-16455 support yellow WARNING status into RubySDK
17
+ - CONFIG: TMA-1625 update version lock for test docker images
18
+ - BUGFIX: TMA-1602 User filter brick failed - K8s bricks don't show error properly
19
+ - BUGFIX: TMA-1593 Increase java heap space during execute bricks
20
+ - BUGFIX: TMA-1558 K8s bricks don't show error when config invalid
21
+ - BUGFIX: TMA-1596 The error propagated from a LCM brick is shown multiple times in the log
22
+ - BUGFIX: TMA-1582 show synchronize ldm mode is running
23
+ - FEATURE: TMA-1588 support schedule param include deprecated
24
+ - FEATURE: TMA-1597 Logging lcm execution result
25
+
2
26
  ## 2.1.8
3
27
  - FEATURE: TMA-1604 Upgrade Restforce version to 3.x
4
28
 
data/Dockerfile CHANGED
@@ -43,16 +43,24 @@ RUN groupadd -g 48 apache \
43
43
  USER apache
44
44
 
45
45
  ADD ./bin ./bin
46
+ ADD --chown=apache:apache ./ci ./ci
46
47
  ADD --chown=apache:apache ./lib ./lib
47
48
  ADD ./SDK_VERSION .
48
49
  ADD ./VERSION .
49
50
  ADD ./Gemfile .
50
51
  ADD ./gooddata.gemspec .
51
52
 
52
- RUN mkdir -p tmp
53
- COPY spec/lcm/redshift_driver_pom.xml tmp/pom.xml
54
- RUN mvn -f tmp/pom.xml clean install -P binary-packaging
55
- RUN cp -rf tmp/target/*.jar ./lib/gooddata/cloud_resources/redshift/drivers/
53
+ #build redshift dependencies
54
+ RUN mvn -f ci/redshift/pom.xml clean install -P binary-packaging
55
+ RUN cp -rf ci/redshift/target/*.jar ./lib/gooddata/cloud_resources/redshift/drivers/
56
+
57
+ #build snowflake dependencies
58
+ RUN mvn -f ci/snowflake/pom.xml clean install -P binary-packaging
59
+ RUN cp -rf ci/snowflake/target/*.jar ./lib/gooddata/cloud_resources/snowflake/drivers/
60
+
61
+ #build bigquery dependencies
62
+ RUN mvn -f ci/bigquery/pom.xml clean install -P binary-packaging
63
+ RUN cp -rf ci/bigquery/target/*.jar ./lib/gooddata/cloud_resources/bigquery/drivers/
56
64
 
57
65
  RUN bundle install
58
66
 
@@ -1 +1 @@
1
- 2.1.9
1
+ 2.1.10
data/VERSION CHANGED
@@ -1 +1 @@
1
- 3.7.14
1
+ 3.7.17
@@ -0,0 +1,54 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project xmlns="http://maven.apache.org/POM/4.0.0"
3
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5
+ <modelVersion>4.0.0</modelVersion>
6
+
7
+ <groupId>com.gooddata.lcm</groupId>
8
+ <artifactId>lcm-bigquery-driver</artifactId>
9
+ <version>1.0-SNAPSHOT</version>
10
+
11
+ <dependencies>
12
+ <dependency>
13
+ <groupId>com.google.auth</groupId>
14
+ <artifactId>google-auth-library-oauth2-http</artifactId>
15
+ <version>0.16.2</version>
16
+ </dependency>
17
+ <dependency>
18
+ <groupId>org.apache.commons</groupId>
19
+ <artifactId>commons-text</artifactId>
20
+ <version>1.7</version>
21
+ </dependency>
22
+ <dependency>
23
+ <groupId>com.google.cloud</groupId>
24
+ <artifactId>google-cloud-bigquery</artifactId>
25
+ <version>1.102.0</version>
26
+ </dependency>
27
+ </dependencies>
28
+
29
+ <profiles>
30
+ <profile>
31
+ <id>binary-packaging</id>
32
+ <build>
33
+ <plugins>
34
+ <plugin>
35
+ <artifactId>maven-dependency-plugin</artifactId>
36
+ <executions>
37
+ <execution>
38
+ <phase>package</phase>
39
+ <goals>
40
+ <goal>copy-dependencies</goal>
41
+ </goals>
42
+ <configuration>
43
+ <outputDirectory>${project.build.directory}</outputDirectory>
44
+ <!-- compile scope gives runtime and compile dependencies (skips test deps) -->
45
+ <includeScope>runtime</includeScope>
46
+ </configuration>
47
+ </execution>
48
+ </executions>
49
+ </plugin>
50
+ </plugins>
51
+ </build>
52
+ </profile>
53
+ </profiles>
54
+ </project>
@@ -0,0 +1,73 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project xmlns="http://maven.apache.org/POM/4.0.0"
3
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5
+ <modelVersion>4.0.0</modelVersion>
6
+
7
+ <groupId>com.gooddata.lcm</groupId>
8
+ <artifactId>lcm-redshift-driver</artifactId>
9
+ <version>1.0-SNAPSHOT</version>
10
+
11
+ <dependencies>
12
+ <dependency>
13
+ <groupId>com.amazon.redshift</groupId>
14
+ <artifactId>redshift-jdbc42-no-awssdk</artifactId>
15
+ <version>1.2.36.1060</version>
16
+ </dependency>
17
+ <dependency>
18
+ <groupId>com.amazonaws</groupId>
19
+ <artifactId>aws-java-sdk-core</artifactId>
20
+ <version>1.11.633</version>
21
+ </dependency>
22
+ <dependency>
23
+ <groupId>com.amazonaws</groupId>
24
+ <artifactId>aws-java-sdk-redshift</artifactId>
25
+ <version>1.11.633</version>
26
+ </dependency>
27
+ <dependency>
28
+ <groupId>com.amazonaws</groupId>
29
+ <artifactId>aws-java-sdk-sts</artifactId>
30
+ <version>1.11.633</version>
31
+ </dependency>
32
+ <dependency>
33
+ <groupId>log4j</groupId>
34
+ <artifactId>log4j</artifactId>
35
+ <version>1.2.17</version>
36
+ <scope>compile</scope>
37
+ </dependency>
38
+ </dependencies>
39
+
40
+ <profiles>
41
+ <profile>
42
+ <id>binary-packaging</id>
43
+ <build>
44
+ <plugins>
45
+ <plugin>
46
+ <artifactId>maven-dependency-plugin</artifactId>
47
+ <executions>
48
+ <execution>
49
+ <phase>package</phase>
50
+ <goals>
51
+ <goal>copy-dependencies</goal>
52
+ </goals>
53
+ <configuration>
54
+ <outputDirectory>${project.build.directory}</outputDirectory>
55
+ <!-- compile scope gives runtime and compile dependencies (skips test deps) -->
56
+ <includeScope>runtime</includeScope>
57
+ </configuration>
58
+ </execution>
59
+ </executions>
60
+ </plugin>
61
+ </plugins>
62
+ </build>
63
+ </profile>
64
+ </profiles>
65
+
66
+ <repositories>
67
+ <repository>
68
+ <id>my-repo1</id>
69
+ <name>my custom repo</name>
70
+ <url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
71
+ </repository>
72
+ </repositories>
73
+ </project>
@@ -0,0 +1,57 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project xmlns="http://maven.apache.org/POM/4.0.0"
3
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5
+ <modelVersion>4.0.0</modelVersion>
6
+
7
+ <groupId>com.gooddata.lcm</groupId>
8
+ <artifactId>lcm-snowflake-driver</artifactId>
9
+ <version>1.0-SNAPSHOT</version>
10
+
11
+ <dependencies>
12
+ <dependency>
13
+ <groupId>net.snowflake</groupId>
14
+ <artifactId>snowflake-jdbc</artifactId>
15
+ <version>3.6.22</version>
16
+ </dependency>
17
+ <dependency>
18
+ <groupId>org.slf4j</groupId>
19
+ <artifactId>slf4j-api</artifactId>
20
+ <version>1.7.2</version>
21
+ </dependency>
22
+ </dependencies>
23
+
24
+ <profiles>
25
+ <profile>
26
+ <id>binary-packaging</id>
27
+ <build>
28
+ <plugins>
29
+ <plugin>
30
+ <artifactId>maven-dependency-plugin</artifactId>
31
+ <executions>
32
+ <execution>
33
+ <phase>package</phase>
34
+ <goals>
35
+ <goal>copy-dependencies</goal>
36
+ </goals>
37
+ <configuration>
38
+ <outputDirectory>${project.build.directory}</outputDirectory>
39
+ <!-- compile scope gives runtime and compile dependencies (skips test deps) -->
40
+ <includeScope>runtime</includeScope>
41
+ </configuration>
42
+ </execution>
43
+ </executions>
44
+ </plugin>
45
+ </plugins>
46
+ </build>
47
+ </profile>
48
+ </profiles>
49
+
50
+ <repositories>
51
+ <repository>
52
+ <id>my-repo1</id>
53
+ <name>my custom repo</name>
54
+ <url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
55
+ </repository>
56
+ </repositories>
57
+ </project>
data/lcm.rake CHANGED
@@ -118,7 +118,7 @@ end
118
118
  namespace :docker do
119
119
  desc 'Build Docker image'
120
120
  task :build do
121
- Rake::Task["maven:build_redshift"].invoke
121
+ Rake::Task["maven:build_dependencies"].invoke
122
122
  system('docker build -f Dockerfile.jruby -t gooddata/appstore .')
123
123
  end
124
124
 
@@ -129,11 +129,15 @@ namespace :docker do
129
129
  end
130
130
 
131
131
  namespace :maven do
132
- task :build_redshift do
133
- system("cp -rf spec/lcm/redshift_driver_pom.xml tmp/pom.xml")
134
- system('mvn -f tmp/pom.xml clean install -P binary-packaging')
135
- system('cp -rf tmp/target/*.jar lib/gooddata/cloud_resources/redshift/drivers/')
136
- system('rm -rf lib/gooddata/cloud_resources/redshift/drivers/lcm-redshift-driver*.jar')
132
+ task :build_dependencies do
133
+ system('mvn -f ci/snowflake/pom.xml clean install -P binary-packaging')
134
+ system('cp -rf ci/snowflake/target/*.jar lib/gooddata/cloud_resources/snowflake/drivers/')
135
+
136
+ system('mvn -f ci/bigquery/pom.xml clean install -P binary-packaging')
137
+ system('cp -rf ci/bigquery/target/*.jar lib/gooddata/cloud_resources/bigquery/drivers/')
138
+
139
+ system('mvn -f ci/redshift/pom.xml clean install -P binary-packaging')
140
+ system('cp -rf ci/redshift/target/*.jar lib/gooddata/cloud_resources/redshift/drivers/')
137
141
  end
138
142
  end
139
143
 
@@ -0,0 +1,86 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'java'
10
+ require 'pathname'
11
+ require_relative '../cloud_resource_client'
12
+
13
+ base = Pathname(__FILE__).dirname.expand_path
14
+ Dir.glob(base + 'drivers/*.jar').each do |file|
15
+ require file unless file.start_with?('lcm-bigquery-driver')
16
+ end
17
+
18
+ java_import 'com.google.auth.oauth2.ServiceAccountCredentials'
19
+ java_import 'com.google.cloud.bigquery.BigQuery'
20
+ java_import 'com.google.cloud.bigquery.BigQueryOptions'
21
+ java_import 'com.google.cloud.bigquery.FieldList'
22
+ java_import 'com.google.cloud.bigquery.FieldValue'
23
+ java_import 'com.google.cloud.bigquery.FieldValueList'
24
+ java_import 'com.google.cloud.bigquery.QueryJobConfiguration'
25
+ java_import 'com.google.cloud.bigquery.TableResult'
26
+ java_import 'org.apache.commons.text.StringEscapeUtils'
27
+
28
+ module GoodData
29
+ module CloudResources
30
+ class BigQueryClient < CloudResourceClient
31
+ class << self
32
+ def accept?(type)
33
+ type == 'bigquery'
34
+ end
35
+ end
36
+
37
+ def initialize(options = {})
38
+ raise("Data Source needs a client to BigQuery to be able to query the storage but 'bigquery_client' is empty.") unless options['bigquery_client']
39
+
40
+ if options['bigquery_client']['connection'].is_a?(Hash)
41
+ @project = options['bigquery_client']['connection']['project']
42
+ @schema = options['bigquery_client']['connection']['schema'] || 'public'
43
+ @authentication = options['bigquery_client']['connection']['authentication']
44
+ else
45
+ raise('Missing connection info for BigQuery client')
46
+
47
+ end
48
+ end
49
+
50
+ def realize_query(query, _params)
51
+ GoodData.gd_logger.info("Realize SQL query: type=bigquery status=started")
52
+
53
+ client = create_client
54
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
55
+ measure = Benchmark.measure do
56
+ query_config = QueryJobConfiguration.newBuilder(query).setDefaultDataset(@schema).build
57
+ table_result = client.query(query_config)
58
+
59
+ if table_result.getTotalRows.positive?
60
+ result = table_result.iterateAll
61
+ field_list = table_result.getSchema.getFields
62
+ col_count = field_list.size
63
+ CSV.open(filename, 'wb') do |csv|
64
+ csv << Array(1..col_count).map { |i| field_list.get(i - 1).getName } # build the header
65
+ result.each do |row|
66
+ csv << Array(1..col_count).map { |i| row.get(i - 1).getValue&.to_s }
67
+ end
68
+ end
69
+ end
70
+ end
71
+ GoodData.gd_logger.info("Realize SQL query: type=bigquery status=finished duration=#{measure.real}")
72
+ filename
73
+ end
74
+
75
+ private
76
+
77
+ def create_client
78
+ GoodData.logger.info "Setting up connection to BigQuery"
79
+ client_email = @authentication['serviceAccount']['clientEmail']
80
+ private_key = @authentication['serviceAccount']['privateKey']
81
+ credentials = ServiceAccountCredentials.fromPkcs8(nil, client_email, StringEscapeUtils.unescapeJson(private_key), nil, nil)
82
+ BigQueryOptions.newBuilder.setProjectId(@project).setCredentials(credentials).build.getService
83
+ end
84
+ end
85
+ end
86
+ end
@@ -1,4 +1,5 @@
1
1
  # encoding: UTF-8
2
+ # frozen_string_literal: true
2
3
  #
3
4
  # Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
4
5
  # This source code is licensed under the BSD-style license found in the
@@ -57,9 +58,9 @@ module GoodData
57
58
  result = statement.get_result_set
58
59
  metadata = result.get_meta_data
59
60
  col_count = metadata.column_count
60
- CSV.open(filename, 'wb', :force_quotes => true) do |csv|
61
+ CSV.open(filename, 'wb') do |csv|
61
62
  csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
62
- csv << Array(1..col_count).map { |i| result.get_string(i) } while result.next
63
+ csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
63
64
  end
64
65
  end
65
66
  end
@@ -0,0 +1,84 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'java'
10
+ require 'pathname'
11
+ require_relative '../cloud_resource_client'
12
+
13
+ base = Pathname(__FILE__).dirname.expand_path
14
+ Dir.glob(base + 'drivers/*.jar').each do |file|
15
+ require file unless file.start_with?('lcm-snowflake-driver')
16
+ end
17
+
18
+ module GoodData
19
+ module CloudResources
20
+ class SnowflakeClient < CloudResourceClient
21
+ class << self
22
+ def accept?(type)
23
+ type == 'snowflake'
24
+ end
25
+ end
26
+
27
+ def initialize(options = {})
28
+ raise("Data Source needs a client to Snowflake to be able to query the storage but 'snowflake_client' is empty.") unless options['snowflake_client']
29
+
30
+ if options['snowflake_client']['connection'].is_a?(Hash)
31
+ @database = options['snowflake_client']['connection']['database']
32
+ @schema = options['snowflake_client']['connection']['schema'] || 'public'
33
+ @warehouse = options['snowflake_client']['connection']['warehouse']
34
+ @url = options['snowflake_client']['connection']['url']
35
+ @authentication = options['snowflake_client']['connection']['authentication']
36
+ else
37
+ raise('Missing connection info for Snowflake client')
38
+
39
+ end
40
+
41
+ Java.net.snowflake.client.jdbc.SnowflakeDriver
42
+ end
43
+
44
+ def realize_query(query, _params)
45
+ GoodData.gd_logger.info("Realize SQL query: type=snowflake status=started")
46
+
47
+ connect
48
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
49
+ measure = Benchmark.measure do
50
+ statement = @connection.create_statement
51
+
52
+ has_result = statement.execute(query)
53
+ if has_result
54
+ result = statement.get_result_set
55
+ metadata = result.get_meta_data
56
+ col_count = metadata.column_count
57
+ CSV.open(filename, 'wb') do |csv|
58
+ csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
59
+ csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
60
+ end
61
+ end
62
+ end
63
+ GoodData.gd_logger.info("Realize SQL query: type=snowflake status=finished duration=#{measure.real}")
64
+ filename
65
+ ensure
66
+ @connection&.close
67
+ @connection = nil
68
+ end
69
+
70
+ def connect
71
+ GoodData.logger.info "Setting up connection to Snowflake #{@url}"
72
+
73
+ prop = java.util.Properties.new
74
+ prop.setProperty('user', @authentication['basic']['userName'])
75
+ prop.setProperty('password', @authentication['basic']['password'])
76
+ prop.setProperty('schema', @schema)
77
+ prop.setProperty('warehouse', @warehouse)
78
+ prop.setProperty('db', @database)
79
+
80
+ @connection = java.sql.DriverManager.getConnection(@url, prop)
81
+ end
82
+ end
83
+ end
84
+ end
@@ -44,7 +44,7 @@ module GoodData
44
44
  realize_link
45
45
  when 's3'
46
46
  realize_s3(params)
47
- when 'redshift'
47
+ when 'redshift', 'snowflake', 'bigquery'
48
48
  raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
49
49
 
50
50
  require_relative '../cloud_resources/cloud_resources'
@@ -67,11 +67,11 @@ module GoodData
67
67
  end
68
68
 
69
69
  def collect_clients(params, segment_names = nil)
70
- client_id_column = params.client_id_column || 'client_id'
71
- segment_id_column = params.segment_id_column || 'segment_id'
72
- project_id_column = params.project_id_column || 'project_id'
73
- project_title_column = params.project_title_column || 'project_title'
74
- project_token_column = params.project_token_column || 'project_token'
70
+ client_id_column = params.client_id_column&.downcase || 'client_id'
71
+ segment_id_column = params.segment_id_column&.downcase || 'segment_id'
72
+ project_id_column = params.project_id_column&.downcase || 'project_id'
73
+ project_title_column = params.project_title_column&.downcase || 'project_title'
74
+ project_token_column = params.project_token_column&.downcase || 'project_token'
75
75
  client = params.gdc_gd_client
76
76
 
77
77
  clients = []
@@ -82,7 +82,7 @@ module GoodData
82
82
  end
83
83
  GoodData.logger.debug("Input data: #{input_data.read}")
84
84
  GoodData.logger.debug("Segment names: #{segment_names}")
85
- CSV.foreach(input_data, :headers => true, :return_headers => false, encoding: 'utf-8') do |row|
85
+ CSV.foreach(input_data, :headers => true, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
86
86
  GoodData.logger.debug("Processing row: #{row}")
87
87
  segment_name = row[segment_id_column]
88
88
  GoodData.logger.debug("Segment name: #{segment_name}")
@@ -38,11 +38,11 @@ module GoodData
38
38
  def call(params)
39
39
  return [] unless params.dynamic_params
40
40
 
41
- schedule_title_column = params.schedule_title_column || 'schedule_title'
42
- client_id_column = params.client_id_column || 'client_id'
43
- param_name_column = params.param_name_column || 'param_name'
44
- param_value_column = params.param_value_column || 'param_value'
45
- param_secure_column = params.param_secure_column || 'param_secure'
41
+ schedule_title_column = params.schedule_title_column&.downcase || 'schedule_title'
42
+ client_id_column = params.client_id_column&.downcase || 'client_id'
43
+ param_name_column = params.param_name_column&.downcase || 'param_name'
44
+ param_value_column = params.param_value_column&.downcase || 'param_value'
45
+ param_secure_column = params.param_secure_column&.downcase || 'param_secure'
46
46
 
47
47
  encryption_key = params.dynamic_params_encryption_key || ''
48
48
  exist_encryption_key = encryption_key.blank? ? false : true
@@ -59,7 +59,7 @@ module GoodData
59
59
  schedule_hidden_params = {}
60
60
  exist_param_secure = false
61
61
 
62
- CSV.foreach(input_data, :headers => true, :return_headers => false, encoding: 'utf-8') do |row|
62
+ CSV.foreach(input_data, :headers => true, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
63
63
  is_param_secure = row[param_secure_column] == 'true'
64
64
  is_decrypt_secure_value = is_param_secure && exist_encryption_key ? true : false
65
65
  exist_param_secure = true if is_param_secure
@@ -35,7 +35,7 @@ module GoodData
35
35
  class << self
36
36
  def call(params)
37
37
  users_brick_users = []
38
- login_column = params.users_brick_config.login_column || 'login'
38
+ login_column = params.users_brick_config.login_column&.downcase || 'login'
39
39
  users_brick_data_source = GoodData::Helpers::DataSource.new(params.users_brick_config.input_source)
40
40
 
41
41
  users_brick_data_source_file = without_check(PARAMS, params) do
@@ -45,14 +45,15 @@ module GoodData
45
45
  )
46
46
  end
47
47
  CSV.foreach(users_brick_data_source_file,
48
- headers: true,
49
- return_headers: false,
50
- encoding: 'utf-8') do |row|
51
- pid = row[params.multiple_projects_column]
48
+ :headers => true,
49
+ :return_headers => false,
50
+ :header_converters => :downcase,
51
+ :encoding => 'utf-8') do |row|
52
+ pid = row[params.multiple_projects_column&.downcase]
52
53
  fail "The set multiple_projects_column '#{params.multiple_projects_column}' of the users input is empty" if !pid && MULTIPLE_COLUMN_MODES.include?(params.sync_mode)
53
54
 
54
55
  users_brick_users << {
55
- login: row[login_column].downcase,
56
+ login: row[login_column].nil? ? nil : row[login_column].strip.downcase,
56
57
  pid: pid
57
58
  }
58
59
  end
@@ -0,0 +1,76 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require_relative 'base_action'
9
+
10
+ module GoodData
11
+ module LCM2
12
+ class SetMasterProject < BaseAction
13
+ DESCRIPTION = 'Set master project'
14
+
15
+ PARAMS = define_params(self) do
16
+ description 'Organization Name'
17
+ param :organization, instance_of(Type::StringType), required: false
18
+
19
+ description 'Domain'
20
+ param :domain, instance_of(Type::StringType), required: false
21
+
22
+ description 'ADS Client'
23
+ param :ads_client, instance_of(Type::AdsClientType), required: false
24
+
25
+ description 'Table Name'
26
+ param :release_table_name, instance_of(Type::StringType), required: false
27
+
28
+ description 'Segments to manage'
29
+ param :segments, array_of(instance_of(Type::SegmentType)), required: true
30
+
31
+ description 'DataProduct to manage'
32
+ param :data_product, instance_of(Type::GDDataProductType), required: false
33
+
34
+ description 'Released master project should be used in next rollout'
35
+ param :set_master_project, instance_of(Type::StringType), required: false
36
+ end
37
+
38
+ class << self
39
+ def call(params)
40
+ results = []
41
+ domain_name = params.organization || params.domain
42
+ data_product = params.data_product
43
+ params.segments.each do |segment_in|
44
+ version = get_latest_version(params, domain_name, data_product.data_product_id, segment_in.segment_id) + 1
45
+ segment_in[:data_product_id] = data_product.data_product_id
46
+ segment_in[:master_pid] = params.set_master_project
47
+ segment_in[:version] = version
48
+ segment_in[:timestamp] = Time.now.utc.iso8601
49
+
50
+ results << {
51
+ data_product_id: data_product.data_product_id,
52
+ segment_id: segment_in.segment_id,
53
+ version: version
54
+ }
55
+ end
56
+ results
57
+ end
58
+
59
+ def get_latest_version(params, domain_name, data_product_id, segment_id)
60
+ if params.ads_client
61
+ current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
62
+ params.release_table_name,
63
+ params.ads_client,
64
+ segment_id
65
+ )
66
+ else
67
+ current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product_id, segment_id)
68
+ end
69
+ return 0 unless current_master
70
+
71
+ current_master[:version].to_i
72
+ end
73
+ end
74
+ end
75
+ end
76
+ end
@@ -230,8 +230,8 @@ module GoodData
230
230
  begin
231
231
  GoodData.logger.info('Start reading data')
232
232
  row_count = 0
233
- CSV.foreach(tmp, headers: csv_with_headers, return_headers: false, encoding: 'utf-8') do |row|
234
- filters << row.to_hash.merge(pid: row[multiple_projects_column])
233
+ CSV.foreach(tmp, :headers => csv_with_headers, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
234
+ filters << row.to_hash.merge(pid: row[multiple_projects_column.downcase])
235
235
  row_count += 1
236
236
  GoodData.logger.info("Read #{row_count} rows") if (row_count % 50_000).zero?
237
237
  end
@@ -347,38 +347,33 @@ module GoodData
347
347
  end
348
348
 
349
349
  def load_data(params, data_source)
350
- first_name_column = params.first_name_column || 'first_name'
351
- last_name_column = params.last_name_column || 'last_name'
352
- login_column = params.login_column || 'login'
353
- password_column = params.password_column || 'password'
354
- email_column = params.email_column || 'email'
355
- role_column = params.role_column || 'role'
356
- sso_provider_column = params.sso_provider_column || 'sso_provider'
357
- authentication_modes_column = params.authentication_modes_column || 'authentication_modes'
358
- user_groups_column = params.user_groups_column || 'user_groups'
359
- language_column = params.language_column || 'language'
360
- company_column = params.company_column || 'company'
361
- position_column = params.position_column || 'position'
362
- country_column = params.country_column || 'country'
363
- phone_column = params.phone_column || 'phone'
364
- ip_whitelist_column = params.ip_whitelist_column || 'ip_whitelist'
350
+ first_name_column = params.first_name_column&.downcase || 'first_name'
351
+ last_name_column = params.last_name_column&.downcase || 'last_name'
352
+ login_column = params.login_column&.downcase || 'login'
353
+ password_column = params.password_column&.downcase || 'password'
354
+ email_column = params.email_column&.downcase || 'email'
355
+ role_column = params.role_column&.downcase || 'role'
356
+ sso_provider_column = params.sso_provider_column&.downcase || 'sso_provider'
357
+ authentication_modes_column = params.authentication_modes_column&.downcase || 'authentication_modes'
358
+ user_groups_column = params.user_groups_column&.downcase || 'user_groups'
359
+ language_column = params.language_column&.downcase || 'language'
360
+ company_column = params.company_column&.downcase || 'company'
361
+ position_column = params.position_column&.downcase || 'position'
362
+ country_column = params.country_column&.downcase || 'country'
363
+ phone_column = params.phone_column&.downcase || 'phone'
364
+ ip_whitelist_column = params.ip_whitelist_column&.downcase || 'ip_whitelist'
365
365
 
366
366
  sso_provider = params.sso_provider
367
367
  authentication_modes = params.authentication_modes || []
368
368
 
369
- dwh = params.ads_client
370
- if dwh
371
- data = dwh.execute_select(params.input_source.query)
372
- else
373
- tmp = without_check(PARAMS, params) do
374
- File.open(data_source.realize(params), 'r:UTF-8')
375
- end
369
+ tmp = without_check(PARAMS, params) do
370
+ File.open(data_source.realize(params), 'r:UTF-8')
371
+ end
376
372
 
377
- begin
378
- data = read_csv_file(tmp)
379
- rescue Exception => e # rubocop:disable RescueException
380
- fail "There was an error during loading users from csv file. Message: #{e.message}. Error: #{e}"
381
- end
373
+ begin
374
+ data = read_csv_file(tmp)
375
+ rescue Exception => e # rubocop:disable RescueException
376
+ fail "There was an error during loading users from csv file. Message: #{e.message}. Error: #{e}"
382
377
  end
383
378
 
384
379
  data.map do |row|
@@ -398,12 +393,18 @@ module GoodData
398
393
  ip_whitelist = row[ip_whitelist_column] || row[ip_whitelist_column.to_sym]
399
394
  ip_whitelist = ip_whitelist.split(',').map(&:strip) if ip_whitelist
400
395
 
396
+ user_login = row[login_column] || row[login_column.to_sym]
397
+ user_login = user_login.strip unless user_login.nil?
398
+
399
+ user_email = row[email_column] || row[login_column] || row[email_column.to_sym] || row[login_column.to_sym]
400
+ user_email = user_email.strip unless user_email.nil?
401
+
401
402
  {
402
403
  :first_name => row[first_name_column] || row[first_name_column.to_sym],
403
404
  :last_name => row[last_name_column] || row[last_name_column.to_sym],
404
- :login => row[login_column] || row[login_column.to_sym],
405
+ :login => user_login,
405
406
  :password => row[password_column] || row[password_column.to_sym],
406
- :email => row[email_column] || row[login_column] || row[email_column.to_sym] || row[login_column.to_sym],
407
+ :email => user_email,
407
408
  :role => row[role_column] || row[role_column.to_sym],
408
409
  :sso_provider => sso_provider || row[sso_provider_column] || row[sso_provider_column.to_sym],
409
410
  :authentication_modes => modes,
@@ -424,7 +425,7 @@ module GoodData
424
425
  res = []
425
426
  row_count = 0
426
427
 
427
- CSV.foreach(path, :headers => true) do |row|
428
+ CSV.foreach(path, :headers => true, :header_converters => :downcase, :encoding => 'utf-8') do |row|
428
429
  if block_given?
429
430
  data = yield row
430
431
  else
@@ -106,6 +106,14 @@ module GoodData
106
106
  UpdateReleaseTable
107
107
  ],
108
108
 
109
+ release_set_master_project: [
110
+ EnsureReleaseTable,
111
+ CollectDataProduct,
112
+ SegmentsFilter,
113
+ SetMasterProject,
114
+ UpdateReleaseTable
115
+ ],
116
+
109
117
  provision: [
110
118
  EnsureReleaseTable,
111
119
  CollectDataProduct,
@@ -271,8 +279,14 @@ module GoodData
271
279
 
272
280
  GoodData.gd_logger.brick = mode
273
281
 
282
+ final_mode = if params.set_master_project && mode == 'release'
283
+ 'release_set_master_project'
284
+ else
285
+ mode
286
+ end
287
+
274
288
  # Get actions for mode specified
275
- actions = get_mode_actions(mode)
289
+ actions = get_mode_actions(final_mode)
276
290
 
277
291
  if params.actions
278
292
  actions = params.actions.map do |action|
@@ -305,6 +319,12 @@ module GoodData
305
319
  skip_actions.include?(action.name.split('::').last)
306
320
  end
307
321
 
322
+ sync_mode = params.fetch(:sync_mode, nil)
323
+ if mode == 'users' && %w[add_to_organization remove_from_organization].include?(sync_mode)
324
+ actions = actions.reject do |action|
325
+ %w[CollectDataProduct CollectSegments].include?(action.name.split('::').last)
326
+ end
327
+ end
308
328
  check_unused_params(actions, params)
309
329
  print_action_names(mode, actions)
310
330
 
@@ -222,22 +222,24 @@ module GoodData
222
222
  domain = client.domain(domain)
223
223
  if id == :all
224
224
  GoodData.logger.warn("Retrieving all users from domain #{domain.name}")
225
- Enumerator.new do |y|
226
- page_limit = opts[:page_limit] || 1000
227
- offset = opts[:offset] || 0
228
- loop do
229
- begin
230
- tmp = client(opts).get("#{domain.uri}/users", params: { offset: offset, limit: page_limit })
231
- end
232
-
233
- tmp['accountSettings']['items'].each do |user_data|
234
- user = client.create(GoodData::Profile, user_data)
235
- y << user if user
236
- end
237
- break if tmp['accountSettings']['items'].count < page_limit
238
- offset += page_limit
225
+ all_users = []
226
+ page_limit = opts[:page_limit] || 1000
227
+ offset = opts[:offset] || 0
228
+ loop do
229
+ begin
230
+ tmp = client(opts).get("#{domain.uri}/users", params: { offset: offset, limit: page_limit })
239
231
  end
232
+
233
+ tmp['accountSettings']['items'].each do |user_data|
234
+ user = client.create(GoodData::Profile, user_data)
235
+ all_users << user if user
236
+ end
237
+ break if tmp['accountSettings']['items'].count < page_limit
238
+
239
+ offset += page_limit
240
240
  end
241
+
242
+ all_users
241
243
  else
242
244
  find_user_by_login(domain, id)
243
245
  end
@@ -1546,26 +1546,28 @@ module GoodData
1546
1546
  # @return [Array<GoodData::User>] List of users
1547
1547
  def users(opts = {})
1548
1548
  client = client(opts)
1549
- Enumerator.new do |y|
1550
- offset = opts[:offset] || 0
1551
- limit = opts[:limit] || 1_000
1552
- loop do
1553
- tmp = client.get("/gdc/projects/#{pid}/users", params: { offset: offset, limit: limit })
1554
- tmp['users'].each do |user_data|
1555
- user = client.create(GoodData::Membership, user_data, project: self)
1556
-
1557
- if opts[:all]
1558
- y << user
1559
- elsif opts[:disabled]
1560
- y << user if user && user.disabled?
1561
- else
1562
- y << user if user && user.enabled?
1563
- end
1549
+ all_users = []
1550
+ offset = opts[:offset] || 0
1551
+ limit = opts[:limit] || 1_000
1552
+ loop do
1553
+ tmp = client.get("/gdc/projects/#{pid}/users", params: { offset: offset, limit: limit })
1554
+ tmp['users'].each do |user_data|
1555
+ user = client.create(GoodData::Membership, user_data, project: self)
1556
+
1557
+ if opts[:all]
1558
+ all_users << user
1559
+ elsif opts[:disabled]
1560
+ all_users << user if user&.disabled?
1561
+ else
1562
+ all_users << user if user&.enabled?
1564
1563
  end
1565
- break if tmp['users'].count < limit
1566
- offset += limit
1567
1564
  end
1565
+ break if tmp['users'].count < limit
1566
+
1567
+ offset += limit
1568
1568
  end
1569
+
1570
+ all_users
1569
1571
  end
1570
1572
 
1571
1573
  alias_method :members, :users
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gooddata
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.1.9
4
+ version: 2.1.10
5
5
  platform: java
6
6
  authors:
7
7
  - Pavel Kolesnikov
@@ -14,7 +14,7 @@ authors:
14
14
  autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
- date: 2020-02-11 00:00:00.000000000 Z
17
+ date: 2020-05-08 00:00:00.000000000 Z
18
18
  dependencies:
19
19
  - !ruby/object:Gem::Dependency
20
20
  requirement: !ruby/object:Gem::Requirement
@@ -560,6 +560,9 @@ files:
560
560
  - bin/user_filters.sh
561
561
  - bin/users.sh
562
562
  - ci.rake
563
+ - ci/bigquery/pom.xml
564
+ - ci/redshift/pom.xml
565
+ - ci/snowflake/pom.xml
563
566
  - data/2008.crt
564
567
  - data/new_ca.cer
565
568
  - data/new_prodgdc_ca.crt
@@ -616,11 +619,15 @@ files:
616
619
  - lib/gooddata/cli/shared.rb
617
620
  - lib/gooddata/cli/terminal.rb
618
621
  - lib/gooddata/client.rb
622
+ - lib/gooddata/cloud_resources/bigquery/bigquery_client.rb
623
+ - lib/gooddata/cloud_resources/bigquery/drivers/.gitkeepme
619
624
  - lib/gooddata/cloud_resources/cloud_resouce_factory.rb
620
625
  - lib/gooddata/cloud_resources/cloud_resource_client.rb
621
626
  - lib/gooddata/cloud_resources/cloud_resources.rb
622
627
  - lib/gooddata/cloud_resources/redshift/drivers/log4j.properties
623
628
  - lib/gooddata/cloud_resources/redshift/redshift_client.rb
629
+ - lib/gooddata/cloud_resources/snowflake/drivers/.gitkeepme
630
+ - lib/gooddata/cloud_resources/snowflake/snowflake_client.rb
624
631
  - lib/gooddata/commands/api.rb
625
632
  - lib/gooddata/commands/auth.rb
626
633
  - lib/gooddata/commands/base.rb
@@ -710,6 +717,7 @@ files:
710
717
  - lib/gooddata/lcm/actions/purge_clients.rb
711
718
  - lib/gooddata/lcm/actions/rename_existing_client_projects.rb
712
719
  - lib/gooddata/lcm/actions/segments_filter.rb
720
+ - lib/gooddata/lcm/actions/set_master_project.rb
713
721
  - lib/gooddata/lcm/actions/synchronize_attribute_drillpaths.rb
714
722
  - lib/gooddata/lcm/actions/synchronize_cas.rb
715
723
  - lib/gooddata/lcm/actions/synchronize_clients.rb