gooddata 2.1.19 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +5 -5
  2. data/.gdc-ii-config.yaml +1 -1
  3. data/.github/workflows/build.yml +66 -0
  4. data/.github/workflows/pre-merge.yml +72 -0
  5. data/CHANGELOG.md +38 -0
  6. data/Dockerfile +21 -14
  7. data/Dockerfile.jruby +1 -11
  8. data/LICENSE +4409 -16
  9. data/README.md +1 -2
  10. data/SDK_VERSION +1 -1
  11. data/VERSION +1 -1
  12. data/ci/mssql/pom.xml +62 -0
  13. data/ci/mysql/pom.xml +57 -0
  14. data/ci/redshift/pom.xml +1 -1
  15. data/docker-compose.lcm.yml +0 -3
  16. data/gooddata.gemspec +2 -1
  17. data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
  18. data/lcm.rake +2 -8
  19. data/lib/gooddata/bricks/middleware/aws_middleware.rb +35 -9
  20. data/lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb +98 -0
  21. data/lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme +0 -0
  22. data/lib/gooddata/cloud_resources/mssql/mssql_client.rb +122 -0
  23. data/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme +0 -0
  24. data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +111 -0
  25. data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +0 -1
  26. data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +18 -1
  27. data/lib/gooddata/helpers/data_helper.rb +9 -4
  28. data/lib/gooddata/lcm/actions/collect_meta.rb +3 -1
  29. data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +3 -2
  30. data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -7
  31. data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +64 -0
  32. data/lib/gooddata/lcm/actions/synchronize_ldm.rb +19 -8
  33. data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +12 -9
  34. data/lib/gooddata/lcm/actions/update_metric_formats.rb +185 -0
  35. data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb +5 -0
  36. data/lib/gooddata/lcm/helpers/release_table_helper.rb +42 -8
  37. data/lib/gooddata/lcm/lcm2.rb +5 -0
  38. data/lib/gooddata/mixins/md_object_query.rb +1 -0
  39. data/lib/gooddata/models/data_source.rb +5 -1
  40. data/lib/gooddata/models/dataset_mapping.rb +36 -0
  41. data/lib/gooddata/models/metadata/label.rb +26 -27
  42. data/lib/gooddata/models/project.rb +34 -9
  43. data/lib/gooddata/models/schedule.rb +13 -1
  44. data/lib/gooddata/models/user_filters/user_filter_builder.rb +58 -53
  45. data/lib/gooddata/rest/phmap.rb +1 -0
  46. metadata +45 -18
  47. data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb +0 -37
data/README.md CHANGED
@@ -18,8 +18,7 @@ Feel free to check out the [GoodData community website](http://community.gooddat
18
18
  [![Downloads](http://img.shields.io/gem/dt/gooddata.svg)](http://rubygems.org/gems/gooddata)
19
19
  [![Dependency Status](https://gemnasium.com/gooddata/gooddata-ruby.png)](https://gemnasium.com/gooddata/gooddata-ruby)
20
20
  [![Code Climate](https://codeclimate.com/github/gooddata/gooddata-ruby.png)](https://codeclimate.com/github/gooddata/gooddata-ruby)
21
- [![Build Status](https://travis-ci.org/gooddata/gooddata-ruby.png)](https://travis-ci.org/gooddata/gooddata-ruby)
22
- [![Coverage Status](https://coveralls.io/repos/gooddata/gooddata-ruby/badge.png)](https://coveralls.io/r/gooddata/gooddata-ruby)
21
+ [![Build Status](https://github.com/gooddata/gooddata-ruby/actions/workflows/build.yml/badge.svg)](https://github.com/gooddata/gooddata-ruby/actions/workflows/build.yml/)
23
22
 
24
23
  ## Supported versions
25
24
 
data/SDK_VERSION CHANGED
@@ -1 +1 @@
1
- 2.1.19
1
+ 2.2.0
data/VERSION CHANGED
@@ -1 +1 @@
1
- 3.7.26
1
+ 3.7.43
data/ci/mssql/pom.xml ADDED
@@ -0,0 +1,62 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project xmlns="http://maven.apache.org/POM/4.0.0"
3
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5
+ <modelVersion>4.0.0</modelVersion>
6
+
7
+ <groupId>com.gooddata.lcm</groupId>
8
+ <artifactId>lcm-mssql-driver</artifactId>
9
+ <version>1.0-SNAPSHOT</version>
10
+
11
+ <dependencies>
12
+ <dependency>
13
+ <groupId>com.microsoft.sqlserver</groupId>
14
+ <artifactId>mssql-jdbc</artifactId>
15
+ <version>9.3.1.jre8-preview</version>
16
+ </dependency>
17
+ <dependency>
18
+ <groupId>com.microsoft.azure</groupId>
19
+ <artifactId>msal4j</artifactId>
20
+ <version>1.10.1</version>
21
+ </dependency>
22
+ <dependency>
23
+ <groupId>org.slf4j</groupId>
24
+ <artifactId>slf4j-api</artifactId>
25
+ <version>1.7.2</version>
26
+ </dependency>
27
+ </dependencies>
28
+
29
+ <profiles>
30
+ <profile>
31
+ <id>binary-packaging</id>
32
+ <build>
33
+ <plugins>
34
+ <plugin>
35
+ <artifactId>maven-dependency-plugin</artifactId>
36
+ <executions>
37
+ <execution>
38
+ <phase>package</phase>
39
+ <goals>
40
+ <goal>copy-dependencies</goal>
41
+ </goals>
42
+ <configuration>
43
+ <outputDirectory>${project.build.directory}</outputDirectory>
44
+ <!-- compile scope gives runtime and compile dependencies (skips test deps) -->
45
+ <includeScope>runtime</includeScope>
46
+ </configuration>
47
+ </execution>
48
+ </executions>
49
+ </plugin>
50
+ </plugins>
51
+ </build>
52
+ </profile>
53
+ </profiles>
54
+
55
+ <repositories>
56
+ <repository>
57
+ <id>my-repo1</id>
58
+ <name>my custom repo</name>
59
+ <url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
60
+ </repository>
61
+ </repositories>
62
+ </project>
data/ci/mysql/pom.xml ADDED
@@ -0,0 +1,57 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project xmlns="http://maven.apache.org/POM/4.0.0"
3
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5
+ <modelVersion>4.0.0</modelVersion>
6
+
7
+ <groupId>com.gooddata.lcm</groupId>
8
+ <artifactId>lcm-mysql-driver</artifactId>
9
+ <version>1.0-SNAPSHOT</version>
10
+
11
+ <dependencies>
12
+ <dependency>
13
+ <groupId>mysql</groupId>
14
+ <artifactId>mysql-connector-java</artifactId>
15
+ <version>8.0.25</version>
16
+ </dependency>
17
+ <dependency>
18
+ <groupId>org.slf4j</groupId>
19
+ <artifactId>slf4j-api</artifactId>
20
+ <version>1.7.2</version>
21
+ </dependency>
22
+ </dependencies>
23
+
24
+ <profiles>
25
+ <profile>
26
+ <id>binary-packaging</id>
27
+ <build>
28
+ <plugins>
29
+ <plugin>
30
+ <artifactId>maven-dependency-plugin</artifactId>
31
+ <executions>
32
+ <execution>
33
+ <phase>package</phase>
34
+ <goals>
35
+ <goal>copy-dependencies</goal>
36
+ </goals>
37
+ <configuration>
38
+ <outputDirectory>${project.build.directory}</outputDirectory>
39
+ <!-- compile scope gives runtime and compile dependencies (skips test deps) -->
40
+ <includeScope>runtime</includeScope>
41
+ </configuration>
42
+ </execution>
43
+ </executions>
44
+ </plugin>
45
+ </plugins>
46
+ </build>
47
+ </profile>
48
+ </profiles>
49
+
50
+ <repositories>
51
+ <repository>
52
+ <id>my-repo1</id>
53
+ <name>my custom repo</name>
54
+ <url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
55
+ </repository>
56
+ </repositories>
57
+ </project>
data/ci/redshift/pom.xml CHANGED
@@ -67,7 +67,7 @@
67
67
  <repository>
68
68
  <id>my-repo1</id>
69
69
  <name>my custom repo</name>
70
- <url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
70
+ <url>https://sonatype-nexus.intgdc.com/repository/public/</url>
71
71
  </repository>
72
72
  </repositories>
73
73
  </project>
@@ -13,9 +13,6 @@ services:
13
13
  - JRUBY_OPTS=-J-Xmx1g
14
14
  - GD_SPEC_PASSWORD
15
15
  - VCR_ON
16
- - POSTGRES_DB=gooddata
17
- - POSTGRES_USER=gooddata
18
- - POSTGRES_PASSWORD=changeit
19
16
  volumes:
20
17
  - .:/src
21
18
  volumes_from:
data/gooddata.gemspec CHANGED
@@ -66,6 +66,8 @@ Gem::Specification.new do |s|
66
66
  else
67
67
  s.add_dependency 'docile', '> 1.1', '< 1.4.0'
68
68
  end
69
+ s.add_dependency 'azure-storage-blob', '~> 1.1.0'
70
+ s.add_dependency 'nokogiri', '~> 1.10.0'
69
71
  s.add_dependency 'gli', '~> 2.15'
70
72
  s.add_dependency 'gooddata_datawarehouse', '~> 0.0.10' if RUBY_PLATFORM == 'java'
71
73
  s.add_dependency 'highline', '= 2.0.0.pre.develop.14'
@@ -77,7 +79,6 @@ Gem::Specification.new do |s|
77
79
  s.add_dependency 'restforce', '>= 2.4', '< 4.0'
78
80
  s.add_dependency 'rest-client', '~> 2.0'
79
81
  s.add_dependency 'rubyzip', '~> 1.2', '>= 1.2.1'
80
- s.add_dependency 'salesforce_bulk_query', '~> 0.2'
81
82
  s.add_dependency 'terminal-table', '~> 1.7'
82
83
  s.add_dependency 'thread_safe'
83
84
  s.add_dependency 'backports'
@@ -1,4 +1,4 @@
1
1
  apiVersion: v1
2
2
  name: lcm-bricks
3
3
  description: LCM Bricks
4
- version: 2.0.6
4
+ version: 2.0.8
data/lcm.rake CHANGED
@@ -101,14 +101,8 @@ namespace :test do
101
101
  test_cases.each do |t|
102
102
  desc "Run #{t} tests in Docker"
103
103
  task t do
104
- if t.to_s == 'integration'
105
- system("docker-compose -f docker-compose.lcm.yml run --rm appstore /bin/bash -c ./spec/integration_with_postgresql.sh") ||
106
- fail('Test execution failed!')
107
- else
108
- system("docker-compose -f docker-compose.lcm.yml run --rm appstore bundle exec rake -f lcm.rake test:#{t}") ||
109
- fail('Test execution failed!')
110
- end
111
-
104
+ system("docker-compose -f docker-compose.lcm.yml run --rm appstore bundle exec rake -f lcm.rake test:#{t}") ||
105
+ fail('Test execution failed!')
112
106
  end
113
107
  end
114
108
  end
@@ -1,6 +1,6 @@
1
1
  # encoding: UTF-8
2
2
  #
3
- # Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
3
+ # Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
4
4
  # This source code is licensed under the BSD-style license found in the
5
5
  # LICENSE file in the root directory of this source tree.
6
6
 
@@ -12,21 +12,47 @@ module GoodData
12
12
  class AWSMiddleware < Bricks::Middleware
13
13
  def call(params)
14
14
  params = params.to_hash
15
- if params.key?('aws_client')
15
+ s3_config = get_s3_config(params)
16
+
17
+ unless s3_config.empty?
16
18
  GoodData.logger.info('Setting up AWS-S3 connection')
17
- raise 'Unable to connect to AWS. Parameter "aws_client" seems to be empty' unless params['aws_client']
18
- raise 'Unable to connect to AWS. Parameter "access_key_id" is missing' if params['aws_client']['access_key_id'].blank?
19
- raise 'Unable to connect to AWS. Parameter "secret_access_key" is missing' if params['aws_client']['secret_access_key'].blank?
20
- params['aws_client'] = rewrite_for_aws_sdk_v2(params['aws_client'])
21
- symbolized_config = GoodData::Helpers.symbolize_keys(params['aws_client'])
22
- s3 = Aws::S3::Resource.new(symbolized_config)
23
- params['aws_client']['s3_client'] = s3
19
+ if params.key?('aws_client')
20
+ params['s3_client'] = {}
21
+ elsif params.key?('s3_client')
22
+ params['input_source'] = {} unless params.key?('input_source')
23
+ params['input_source']['bucket'] = params['s3_client']['bucket']
24
+ end
25
+ s3_config = rewrite_for_aws_sdk_v2(s3_config)
26
+ symbolized_config = GoodData::Helpers.symbolize_keys(s3_config)
27
+ params['s3_client']['client'] = Aws::S3::Resource.new(symbolized_config)
24
28
  end
25
29
  @app.call(params)
26
30
  end
27
31
 
28
32
  private
29
33
 
34
+ def get_s3_config(params)
35
+ s3_config = {}
36
+ if params.key?('aws_client')
37
+ GoodData.logger.warn('Found two configuration aws_client and s3_client for S3 input source, use aws_client configuration') if params.key?('s3_client')
38
+ raise 'Unable to connect to AWS. Parameter "aws_client" seems to be empty' unless params['aws_client']
39
+ raise 'Unable to connect to AWS. Parameter "access_key_id" is missing' if params['aws_client']['access_key_id'].blank?
40
+ raise 'Unable to connect to AWS. Parameter "secret_access_key" is missing' if params['aws_client']['secret_access_key'].blank?
41
+
42
+ s3_config = params['aws_client']
43
+ elsif params.key?('s3_client')
44
+ raise 'Unable to connect to AWS. Parameter "s3_client" seems to be empty' unless params['s3_client']
45
+ raise 'Unable to connect to AWS. Parameter "accessKey" is missing' if params['s3_client']['accessKey'].blank?
46
+ raise 'Unable to connect to AWS. Parameter "secretKey" is missing' if params['s3_client']['secretKey'].blank?
47
+ raise 'Unable to connect to AWS. Parameter "bucket" is missing' if params['s3_client']['bucket'].blank?
48
+
49
+ s3_config['access_key_id'] = params['s3_client']['accessKey']
50
+ s3_config['secret_access_key'] = params['s3_client']['secretKey']
51
+ s3_config['region'] = params['s3_client']['region']
52
+ end
53
+ s3_config
54
+ end
55
+
30
56
  def rewrite_for_aws_sdk_v2(config)
31
57
  config['region'] = 'us-west-2' unless config['region']
32
58
  if config['use_ssl']
@@ -0,0 +1,98 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'pathname'
10
+ require "azure/storage/blob"
11
+
12
+ module GoodData
13
+ class BlobStorageClient
14
+ SAS_URL_PATTERN = %r{(^https?:\/\/[^\/]*)\/.*\?(.*)}
15
+ INVALID_BLOB_GENERAL_MESSAGE = "The connection string is not valid."
16
+ INVALID_BLOB_SIG_WELL_FORMED_MESSAGE = "The signature format is not valid."
17
+ INVALID_BLOB_CONTAINER_MESSAGE = "ContainerNotFound"
18
+ INVALID_BLOB_CONTAINER_FORMED_MESSAGE = "The container with the specified name is not found."
19
+ INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE = "Signature not valid in the specified time frame"
20
+ INVALID_BLOB_EXPIRED_MESSAGE = "The signature expired."
21
+ INVALID_BLOB_INVALID_CONNECTION_STRING_MESSAGE = "The connection string is not valid."
22
+ INVALID_BLOB_PATH_MESSAGE = "BlobNotFound"
23
+ INVALID_BLOB_INVALID_PATH_MESSAGE = "The path to the data is not found."
24
+
25
+ attr_reader :use_sas
26
+
27
+ def initialize(options = {})
28
+ raise("Data Source needs a client to Blob Storage to be able to get blob file but 'blobStorage_client' is empty.") unless options['blobStorage_client']
29
+
30
+ if options['blobStorage_client']['connectionString'] && options['blobStorage_client']['container']
31
+ @connection_string = options['blobStorage_client']['connectionString']
32
+ @container = options['blobStorage_client']['container']
33
+ @path = options['blobStorage_client']['path']
34
+ @use_sas = false
35
+ build_sas(@connection_string)
36
+ else
37
+ raise('Missing connection info for Blob Storage client')
38
+ end
39
+ end
40
+
41
+ def realize_blob(file, _params)
42
+ GoodData.gd_logger.info("Realizing download from Blob Storage. Container #{@container}.")
43
+ filename = ''
44
+ begin
45
+ connect
46
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
47
+ blob_name = get_blob_name(@path, file)
48
+
49
+ measure = Benchmark.measure do
50
+ _blob, content = @client.get_blob(@container, blob_name)
51
+ File.open(filename, "wb") { |f| f.write(content) }
52
+ end
53
+ rescue => e
54
+ raise_error(e)
55
+ end
56
+ GoodData.gd_logger.info("Done downloading file type=blobStorage status=finished duration=#{measure.real}")
57
+ filename
58
+ end
59
+
60
+ def connect
61
+ GoodData.logger.info "Setting up connection to Blob Storage"
62
+ if use_sas
63
+ @client = Azure::Storage::Blob::BlobService.create(:storage_blob_host => @host, :storage_sas_token => @sas_token)
64
+ else
65
+ @client = Azure::Storage::Blob::BlobService.create_from_connection_string(@connection_string)
66
+ end
67
+ end
68
+
69
+ def build_sas(url)
70
+ matches = url.scan(SAS_URL_PATTERN)
71
+ return unless matches && matches[0]
72
+
73
+ @use_sas = true
74
+ @host = matches[0][0]
75
+ @sas_token = matches[0][1]
76
+ end
77
+
78
+ def raise_error(e)
79
+ if e.message && e.message.include?(INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE)
80
+ raise INVALID_BLOB_EXPIRED_MESSAGE
81
+ elsif e.message && e.message.include?(INVALID_BLOB_SIG_WELL_FORMED_MESSAGE)
82
+ raise INVALID_BLOB_SIG_WELL_FORMED_MESSAGE
83
+ elsif e.message && e.message.include?(INVALID_BLOB_CONTAINER_MESSAGE)
84
+ raise INVALID_BLOB_CONTAINER_FORMED_MESSAGE
85
+ elsif e.message && e.message.include?(INVALID_BLOB_PATH_MESSAGE)
86
+ raise INVALID_BLOB_INVALID_PATH_MESSAGE
87
+ else
88
+ raise INVALID_BLOB_GENERAL_MESSAGE
89
+ end
90
+ end
91
+
92
+ def get_blob_name(path, file)
93
+ return file unless path
94
+
95
+ path.rindex('/') == path.length - 1 ? "#{path}#{file}" : "#{path}/#{file}"
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,122 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'java'
10
+ require 'pathname'
11
+ require_relative '../cloud_resource_client'
12
+
13
+ base = Pathname(__FILE__).dirname.expand_path
14
+ Dir.glob(base + 'drivers/*.jar').each do |file|
15
+ require file unless file.start_with?('lcm-mssql-driver')
16
+ end
17
+
18
+ module GoodData
19
+ module CloudResources
20
+ class MSSQLClient < CloudResourceClient
21
+ MSSQL_SEPARATOR_PARAM = ";"
22
+ MSSQL_URL_PATTERN = %r{jdbc:sqlserver://([^:]+)(:([0-9]+))?(/)?}
23
+ MSSQL_DEFAULT_PORT = 1433
24
+ LOGIN_TIME_OUT = 30
25
+ MSSQL_FETCH_SIZE = 10_000
26
+ VERIFY_FULL = 'verify-full'
27
+ PREFER = 'prefer'
28
+ REQUIRE = 'require'
29
+
30
+ class << self
31
+ def accept?(type)
32
+ type == 'mssql'
33
+ end
34
+ end
35
+
36
+ def initialize(options = {})
37
+ raise("Data Source needs a client to MSSQL to be able to query the storage but 'mssql_client' is empty.") unless options['mssql_client']
38
+
39
+ connection = options['mssql_client']['connection']
40
+ if connection.is_a?(Hash)
41
+ @database = connection['database']
42
+ @schema = connection['schema']
43
+ @authentication = connection['authentication']
44
+ @ssl_mode = connection['sslMode']
45
+ @url = connection['url']
46
+
47
+ validate
48
+ else
49
+ raise('Missing connection info for MSSQL client')
50
+ end
51
+
52
+ Java.com.microsoft.sqlserver.jdbc.SQLServerDriver
53
+ end
54
+
55
+ def realize_query(query, _params)
56
+ GoodData.gd_logger.info("Realize SQL query: type=mssql status=started")
57
+
58
+ connect
59
+
60
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
61
+ measure = Benchmark.measure do
62
+ statement = @connection.create_statement
63
+ statement.set_fetch_size(MSSQL_FETCH_SIZE)
64
+ has_result = statement.execute(query)
65
+ if has_result
66
+ result = statement.get_result_set
67
+ metadata = result.get_meta_data
68
+ col_count = metadata.column_count
69
+ CSV.open(filename, 'wb') do |csv|
70
+ csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
71
+ csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
72
+ end
73
+ end
74
+ end
75
+
76
+ GoodData.gd_logger.info("Realize SQL query: type=mssql status=finished duration=#{measure.real}")
77
+ filename
78
+ ensure
79
+ @connection&.close
80
+ @connection = nil
81
+ end
82
+
83
+ def connect
84
+ connection_string = build_connection_string
85
+ GoodData.logger.info "Setting up connection to MSSQL #{connection_string}"
86
+
87
+ authentication = @authentication['basic'] || @authentication['activeDirectoryPassword']
88
+
89
+ prop = java.util.Properties.new
90
+ prop.setProperty('userName', authentication['userName'])
91
+ prop.setProperty('password', authentication['password'])
92
+
93
+ @connection = java.sql.DriverManager.getConnection(connection_string, prop)
94
+ end
95
+
96
+ def validate
97
+ raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
98
+
99
+ raise "Instance name is not supported" if @url !~ /^[^\\]*$/
100
+
101
+ raise "The connection url is invalid. Parameter is not supported." if @url.include? MSSQL_SEPARATOR_PARAM
102
+
103
+ url_matches = @url.scan(MSSQL_URL_PATTERN)
104
+ raise "Cannot reach the url" if url_matches.nil? || url_matches.length.zero?
105
+
106
+ raise "The authentication method is not supported." unless @authentication['basic'] || @authentication['activeDirectoryPassword']
107
+ end
108
+
109
+ def build_connection_string
110
+ encrypt = @ssl_mode != PREFER
111
+ trust_server_certificate = @ssl_mode == REQUIRE
112
+
113
+ "#{@url};" \
114
+ "database=#{@database};" \
115
+ "encrypt=#{encrypt};" \
116
+ "trustServerCertificate=#{trust_server_certificate};" \
117
+ "loginTimeout=#{LOGIN_TIME_OUT};" \
118
+ "#{'authentication=ActiveDirectoryPassword;' if @authentication['activeDirectoryPassword']}"
119
+ end
120
+ end
121
+ end
122
+ end
@@ -0,0 +1,111 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'java'
10
+ require 'pathname'
11
+ require_relative '../cloud_resource_client'
12
+
13
+ base = Pathname(__FILE__).dirname.expand_path
14
+ Dir.glob(base + 'drivers/*.jar').each do |file|
15
+ require file unless file.start_with?('lcm-mysql-driver')
16
+ end
17
+
18
+ module GoodData
19
+ module CloudResources
20
+ class MysqlClient < CloudResourceClient
21
+ JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
22
+ MYSQL_DEFAULT_PORT = 3306
23
+ JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://'
24
+ VERIFY_FULL = 'VERIFY_IDENTITY'
25
+ PREFER = 'PREFERRED'
26
+ REQUIRE = 'REQUIRED'
27
+ MYSQL_FETCH_SIZE = 1000
28
+
29
+ class << self
30
+ def accept?(type)
31
+ type == 'mysql'
32
+ end
33
+ end
34
+
35
+ def initialize(options = {})
36
+ raise("Data Source needs a client to Mysql to be able to query the storage but 'mysql_client' is empty.") unless options['mysql_client']
37
+
38
+ if options['mysql_client']['connection'].is_a?(Hash)
39
+ @database = options['mysql_client']['connection']['database']
40
+ @authentication = options['mysql_client']['connection']['authentication']
41
+ @ssl_mode = options['mysql_client']['connection']['sslMode']
42
+ raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
43
+
44
+ @url = build_url(options['mysql_client']['connection']['url'])
45
+ else
46
+ raise('Missing connection info for Mysql client')
47
+ end
48
+
49
+ Java.com.mysql.cj.jdbc.Driver
50
+ end
51
+
52
+ def realize_query(query, _params)
53
+ GoodData.gd_logger.info("Realize SQL query: type=mysql status=started")
54
+
55
+ connect
56
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
57
+ measure = Benchmark.measure do
58
+ statement = @connection.create_statement
59
+ statement.set_fetch_size(MYSQL_FETCH_SIZE)
60
+ has_result = statement.execute(query)
61
+ if has_result
62
+ result = statement.get_result_set
63
+ metadata = result.get_meta_data
64
+ col_count = metadata.column_count
65
+ CSV.open(filename, 'wb') do |csv|
66
+ csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
67
+ csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
68
+ end
69
+ end
70
+ end
71
+ GoodData.gd_logger.info("Realize SQL query: type=mysql status=finished duration=#{measure.real}")
72
+ filename
73
+ ensure
74
+ @connection&.close
75
+ @connection = nil
76
+ end
77
+
78
+ def connect
79
+ GoodData.logger.info "Setting up connection to Mysql #{@url}"
80
+
81
+ prop = java.util.Properties.new
82
+ prop.setProperty('user', @authentication['basic']['userName'])
83
+ prop.setProperty('password', @authentication['basic']['password'])
84
+
85
+ @connection = java.sql.DriverManager.getConnection(@url, prop)
86
+ @connection.set_auto_commit(false)
87
+ end
88
+
89
+ def build_url(url)
90
+ matches = url.scan(JDBC_MYSQL_PATTERN)
91
+ raise 'Cannot reach the url' unless matches
92
+
93
+ host = matches[0][0]
94
+ port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT
95
+
96
+ "#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{get_ssl_mode(@ssl_mode)}&useCursorFetch=true&enabledTLSProtocols=TLSv1.2"
97
+ end
98
+
99
+ def get_ssl_mode(ssl_mode)
100
+ mode = PREFER
101
+ if ssl_mode == 'verify-full'
102
+ mode = VERIFY_FULL
103
+ elsif ssl_mode == 'require'
104
+ mode = REQUIRE
105
+ end
106
+
107
+ mode
108
+ end
109
+ end
110
+ end
111
+ end
@@ -98,7 +98,6 @@ module GoodData
98
98
 
99
99
  host = matches[0][0]
100
100
  port = matches[0][2]&.to_i || POSTGRES_DEFAULT_PORT
101
- raise "Custom port #{port} is not supported. Remove it or use the default port '5432'" if POSTGRES_DEFAULT_PORT != port
102
101
 
103
102
  "#{JDBC_POSTGRES_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{@ssl_mode}#{VERIFY_FULL == @ssl_mode ? SSL_JAVA_FACTORY : ''}"
104
103
  end
@@ -18,6 +18,9 @@ end
18
18
  module GoodData
19
19
  module CloudResources
20
20
  class SnowflakeClient < CloudResourceClient
21
+ SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=GoodData_Platform'
22
+ SNOWFLAKE_SEPARATOR_PARAM = '?'
23
+
21
24
  class << self
22
25
  def accept?(type)
23
26
  type == 'snowflake'
@@ -31,7 +34,7 @@ module GoodData
31
34
  @database = options['snowflake_client']['connection']['database']
32
35
  @schema = options['snowflake_client']['connection']['schema'] || 'public'
33
36
  @warehouse = options['snowflake_client']['connection']['warehouse']
34
- @url = options['snowflake_client']['connection']['url']
37
+ @url = build_url(options['snowflake_client']['connection']['url'])
35
38
  @authentication = options['snowflake_client']['connection']['authentication']
36
39
  else
37
40
  raise('Missing connection info for Snowflake client')
@@ -79,6 +82,20 @@ module GoodData
79
82
 
80
83
  @connection = java.sql.DriverManager.getConnection(@url, prop)
81
84
  end
85
+
86
+ def build_url(url)
87
+ is_contain = url.include?(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
88
+ unless is_contain
89
+ if url.include?(SNOWFLAKE_SEPARATOR_PARAM)
90
+ url.concat("&")
91
+ else
92
+ url.concat(SNOWFLAKE_SEPARATOR_PARAM)
93
+ end
94
+ url.concat(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
95
+ end
96
+
97
+ url
98
+ end
82
99
  end
83
100
  end
84
101
  end