gooddata 2.1.19 → 2.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (47) hide show
  1. checksums.yaml +5 -5
  2. data/.gdc-ii-config.yaml +1 -1
  3. data/.github/workflows/build.yml +66 -0
  4. data/.github/workflows/pre-merge.yml +72 -0
  5. data/CHANGELOG.md +38 -0
  6. data/Dockerfile +21 -14
  7. data/Dockerfile.jruby +1 -11
  8. data/LICENSE +4409 -16
  9. data/README.md +1 -2
  10. data/SDK_VERSION +1 -1
  11. data/VERSION +1 -1
  12. data/ci/mssql/pom.xml +62 -0
  13. data/ci/mysql/pom.xml +57 -0
  14. data/ci/redshift/pom.xml +1 -1
  15. data/docker-compose.lcm.yml +0 -3
  16. data/gooddata.gemspec +2 -1
  17. data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
  18. data/lcm.rake +2 -8
  19. data/lib/gooddata/bricks/middleware/aws_middleware.rb +35 -9
  20. data/lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb +98 -0
  21. data/lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme +0 -0
  22. data/lib/gooddata/cloud_resources/mssql/mssql_client.rb +122 -0
  23. data/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme +0 -0
  24. data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +111 -0
  25. data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +0 -1
  26. data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +18 -1
  27. data/lib/gooddata/helpers/data_helper.rb +9 -4
  28. data/lib/gooddata/lcm/actions/collect_meta.rb +3 -1
  29. data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +3 -2
  30. data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -7
  31. data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +64 -0
  32. data/lib/gooddata/lcm/actions/synchronize_ldm.rb +19 -8
  33. data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +12 -9
  34. data/lib/gooddata/lcm/actions/update_metric_formats.rb +185 -0
  35. data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb +5 -0
  36. data/lib/gooddata/lcm/helpers/release_table_helper.rb +42 -8
  37. data/lib/gooddata/lcm/lcm2.rb +5 -0
  38. data/lib/gooddata/mixins/md_object_query.rb +1 -0
  39. data/lib/gooddata/models/data_source.rb +5 -1
  40. data/lib/gooddata/models/dataset_mapping.rb +36 -0
  41. data/lib/gooddata/models/metadata/label.rb +26 -27
  42. data/lib/gooddata/models/project.rb +34 -9
  43. data/lib/gooddata/models/schedule.rb +13 -1
  44. data/lib/gooddata/models/user_filters/user_filter_builder.rb +58 -53
  45. data/lib/gooddata/rest/phmap.rb +1 -0
  46. metadata +45 -18
  47. data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb +0 -37
data/README.md CHANGED
@@ -18,8 +18,7 @@ Feel free to check out the [GoodData community website](http://community.gooddat
18
18
  [![Downloads](http://img.shields.io/gem/dt/gooddata.svg)](http://rubygems.org/gems/gooddata)
19
19
  [![Dependency Status](https://gemnasium.com/gooddata/gooddata-ruby.png)](https://gemnasium.com/gooddata/gooddata-ruby)
20
20
  [![Code Climate](https://codeclimate.com/github/gooddata/gooddata-ruby.png)](https://codeclimate.com/github/gooddata/gooddata-ruby)
21
- [![Build Status](https://travis-ci.org/gooddata/gooddata-ruby.png)](https://travis-ci.org/gooddata/gooddata-ruby)
22
- [![Coverage Status](https://coveralls.io/repos/gooddata/gooddata-ruby/badge.png)](https://coveralls.io/r/gooddata/gooddata-ruby)
21
+ [![Build Status](https://github.com/gooddata/gooddata-ruby/actions/workflows/build.yml/badge.svg)](https://github.com/gooddata/gooddata-ruby/actions/workflows/build.yml/)
23
22
 
24
23
  ## Supported versions
25
24
 
data/SDK_VERSION CHANGED
@@ -1 +1 @@
1
- 2.1.19
1
+ 2.2.0
data/VERSION CHANGED
@@ -1 +1 @@
1
- 3.7.26
1
+ 3.7.43
data/ci/mssql/pom.xml ADDED
@@ -0,0 +1,62 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project xmlns="http://maven.apache.org/POM/4.0.0"
3
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5
+ <modelVersion>4.0.0</modelVersion>
6
+
7
+ <groupId>com.gooddata.lcm</groupId>
8
+ <artifactId>lcm-mssql-driver</artifactId>
9
+ <version>1.0-SNAPSHOT</version>
10
+
11
+ <dependencies>
12
+ <dependency>
13
+ <groupId>com.microsoft.sqlserver</groupId>
14
+ <artifactId>mssql-jdbc</artifactId>
15
+ <version>9.3.1.jre8-preview</version>
16
+ </dependency>
17
+ <dependency>
18
+ <groupId>com.microsoft.azure</groupId>
19
+ <artifactId>msal4j</artifactId>
20
+ <version>1.10.1</version>
21
+ </dependency>
22
+ <dependency>
23
+ <groupId>org.slf4j</groupId>
24
+ <artifactId>slf4j-api</artifactId>
25
+ <version>1.7.2</version>
26
+ </dependency>
27
+ </dependencies>
28
+
29
+ <profiles>
30
+ <profile>
31
+ <id>binary-packaging</id>
32
+ <build>
33
+ <plugins>
34
+ <plugin>
35
+ <artifactId>maven-dependency-plugin</artifactId>
36
+ <executions>
37
+ <execution>
38
+ <phase>package</phase>
39
+ <goals>
40
+ <goal>copy-dependencies</goal>
41
+ </goals>
42
+ <configuration>
43
+ <outputDirectory>${project.build.directory}</outputDirectory>
44
+ <!-- compile scope gives runtime and compile dependencies (skips test deps) -->
45
+ <includeScope>runtime</includeScope>
46
+ </configuration>
47
+ </execution>
48
+ </executions>
49
+ </plugin>
50
+ </plugins>
51
+ </build>
52
+ </profile>
53
+ </profiles>
54
+
55
+ <repositories>
56
+ <repository>
57
+ <id>my-repo1</id>
58
+ <name>my custom repo</name>
59
+ <url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
60
+ </repository>
61
+ </repositories>
62
+ </project>
data/ci/mysql/pom.xml ADDED
@@ -0,0 +1,57 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project xmlns="http://maven.apache.org/POM/4.0.0"
3
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5
+ <modelVersion>4.0.0</modelVersion>
6
+
7
+ <groupId>com.gooddata.lcm</groupId>
8
+ <artifactId>lcm-mysql-driver</artifactId>
9
+ <version>1.0-SNAPSHOT</version>
10
+
11
+ <dependencies>
12
+ <dependency>
13
+ <groupId>mysql</groupId>
14
+ <artifactId>mysql-connector-java</artifactId>
15
+ <version>8.0.25</version>
16
+ </dependency>
17
+ <dependency>
18
+ <groupId>org.slf4j</groupId>
19
+ <artifactId>slf4j-api</artifactId>
20
+ <version>1.7.2</version>
21
+ </dependency>
22
+ </dependencies>
23
+
24
+ <profiles>
25
+ <profile>
26
+ <id>binary-packaging</id>
27
+ <build>
28
+ <plugins>
29
+ <plugin>
30
+ <artifactId>maven-dependency-plugin</artifactId>
31
+ <executions>
32
+ <execution>
33
+ <phase>package</phase>
34
+ <goals>
35
+ <goal>copy-dependencies</goal>
36
+ </goals>
37
+ <configuration>
38
+ <outputDirectory>${project.build.directory}</outputDirectory>
39
+ <!-- compile scope gives runtime and compile dependencies (skips test deps) -->
40
+ <includeScope>runtime</includeScope>
41
+ </configuration>
42
+ </execution>
43
+ </executions>
44
+ </plugin>
45
+ </plugins>
46
+ </build>
47
+ </profile>
48
+ </profiles>
49
+
50
+ <repositories>
51
+ <repository>
52
+ <id>my-repo1</id>
53
+ <name>my custom repo</name>
54
+ <url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
55
+ </repository>
56
+ </repositories>
57
+ </project>
data/ci/redshift/pom.xml CHANGED
@@ -67,7 +67,7 @@
67
67
  <repository>
68
68
  <id>my-repo1</id>
69
69
  <name>my custom repo</name>
70
- <url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
70
+ <url>https://sonatype-nexus.intgdc.com/repository/public/</url>
71
71
  </repository>
72
72
  </repositories>
73
73
  </project>
@@ -13,9 +13,6 @@ services:
13
13
  - JRUBY_OPTS=-J-Xmx1g
14
14
  - GD_SPEC_PASSWORD
15
15
  - VCR_ON
16
- - POSTGRES_DB=gooddata
17
- - POSTGRES_USER=gooddata
18
- - POSTGRES_PASSWORD=changeit
19
16
  volumes:
20
17
  - .:/src
21
18
  volumes_from:
data/gooddata.gemspec CHANGED
@@ -66,6 +66,8 @@ Gem::Specification.new do |s|
66
66
  else
67
67
  s.add_dependency 'docile', '> 1.1', '< 1.4.0'
68
68
  end
69
+ s.add_dependency 'azure-storage-blob', '~> 1.1.0'
70
+ s.add_dependency 'nokogiri', '~> 1.10.0'
69
71
  s.add_dependency 'gli', '~> 2.15'
70
72
  s.add_dependency 'gooddata_datawarehouse', '~> 0.0.10' if RUBY_PLATFORM == 'java'
71
73
  s.add_dependency 'highline', '= 2.0.0.pre.develop.14'
@@ -77,7 +79,6 @@ Gem::Specification.new do |s|
77
79
  s.add_dependency 'restforce', '>= 2.4', '< 4.0'
78
80
  s.add_dependency 'rest-client', '~> 2.0'
79
81
  s.add_dependency 'rubyzip', '~> 1.2', '>= 1.2.1'
80
- s.add_dependency 'salesforce_bulk_query', '~> 0.2'
81
82
  s.add_dependency 'terminal-table', '~> 1.7'
82
83
  s.add_dependency 'thread_safe'
83
84
  s.add_dependency 'backports'
@@ -1,4 +1,4 @@
1
1
  apiVersion: v1
2
2
  name: lcm-bricks
3
3
  description: LCM Bricks
4
- version: 2.0.6
4
+ version: 2.0.8
data/lcm.rake CHANGED
@@ -101,14 +101,8 @@ namespace :test do
101
101
  test_cases.each do |t|
102
102
  desc "Run #{t} tests in Docker"
103
103
  task t do
104
- if t.to_s == 'integration'
105
- system("docker-compose -f docker-compose.lcm.yml run --rm appstore /bin/bash -c ./spec/integration_with_postgresql.sh") ||
106
- fail('Test execution failed!')
107
- else
108
- system("docker-compose -f docker-compose.lcm.yml run --rm appstore bundle exec rake -f lcm.rake test:#{t}") ||
109
- fail('Test execution failed!')
110
- end
111
-
104
+ system("docker-compose -f docker-compose.lcm.yml run --rm appstore bundle exec rake -f lcm.rake test:#{t}") ||
105
+ fail('Test execution failed!')
112
106
  end
113
107
  end
114
108
  end
@@ -1,6 +1,6 @@
1
1
  # encoding: UTF-8
2
2
  #
3
- # Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
3
+ # Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
4
4
  # This source code is licensed under the BSD-style license found in the
5
5
  # LICENSE file in the root directory of this source tree.
6
6
 
@@ -12,21 +12,47 @@ module GoodData
12
12
  class AWSMiddleware < Bricks::Middleware
13
13
  def call(params)
14
14
  params = params.to_hash
15
- if params.key?('aws_client')
15
+ s3_config = get_s3_config(params)
16
+
17
+ unless s3_config.empty?
16
18
  GoodData.logger.info('Setting up AWS-S3 connection')
17
- raise 'Unable to connect to AWS. Parameter "aws_client" seems to be empty' unless params['aws_client']
18
- raise 'Unable to connect to AWS. Parameter "access_key_id" is missing' if params['aws_client']['access_key_id'].blank?
19
- raise 'Unable to connect to AWS. Parameter "secret_access_key" is missing' if params['aws_client']['secret_access_key'].blank?
20
- params['aws_client'] = rewrite_for_aws_sdk_v2(params['aws_client'])
21
- symbolized_config = GoodData::Helpers.symbolize_keys(params['aws_client'])
22
- s3 = Aws::S3::Resource.new(symbolized_config)
23
- params['aws_client']['s3_client'] = s3
19
+ if params.key?('aws_client')
20
+ params['s3_client'] = {}
21
+ elsif params.key?('s3_client')
22
+ params['input_source'] = {} unless params.key?('input_source')
23
+ params['input_source']['bucket'] = params['s3_client']['bucket']
24
+ end
25
+ s3_config = rewrite_for_aws_sdk_v2(s3_config)
26
+ symbolized_config = GoodData::Helpers.symbolize_keys(s3_config)
27
+ params['s3_client']['client'] = Aws::S3::Resource.new(symbolized_config)
24
28
  end
25
29
  @app.call(params)
26
30
  end
27
31
 
28
32
  private
29
33
 
34
+ def get_s3_config(params)
35
+ s3_config = {}
36
+ if params.key?('aws_client')
37
+ GoodData.logger.warn('Found two configuration aws_client and s3_client for S3 input source, use aws_client configuration') if params.key?('s3_client')
38
+ raise 'Unable to connect to AWS. Parameter "aws_client" seems to be empty' unless params['aws_client']
39
+ raise 'Unable to connect to AWS. Parameter "access_key_id" is missing' if params['aws_client']['access_key_id'].blank?
40
+ raise 'Unable to connect to AWS. Parameter "secret_access_key" is missing' if params['aws_client']['secret_access_key'].blank?
41
+
42
+ s3_config = params['aws_client']
43
+ elsif params.key?('s3_client')
44
+ raise 'Unable to connect to AWS. Parameter "s3_client" seems to be empty' unless params['s3_client']
45
+ raise 'Unable to connect to AWS. Parameter "accessKey" is missing' if params['s3_client']['accessKey'].blank?
46
+ raise 'Unable to connect to AWS. Parameter "secretKey" is missing' if params['s3_client']['secretKey'].blank?
47
+ raise 'Unable to connect to AWS. Parameter "bucket" is missing' if params['s3_client']['bucket'].blank?
48
+
49
+ s3_config['access_key_id'] = params['s3_client']['accessKey']
50
+ s3_config['secret_access_key'] = params['s3_client']['secretKey']
51
+ s3_config['region'] = params['s3_client']['region']
52
+ end
53
+ s3_config
54
+ end
55
+
30
56
  def rewrite_for_aws_sdk_v2(config)
31
57
  config['region'] = 'us-west-2' unless config['region']
32
58
  if config['use_ssl']
@@ -0,0 +1,98 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'pathname'
10
+ require "azure/storage/blob"
11
+
12
+ module GoodData
13
+ class BlobStorageClient
14
+ SAS_URL_PATTERN = %r{(^https?:\/\/[^\/]*)\/.*\?(.*)}
15
+ INVALID_BLOB_GENERAL_MESSAGE = "The connection string is not valid."
16
+ INVALID_BLOB_SIG_WELL_FORMED_MESSAGE = "The signature format is not valid."
17
+ INVALID_BLOB_CONTAINER_MESSAGE = "ContainerNotFound"
18
+ INVALID_BLOB_CONTAINER_FORMED_MESSAGE = "The container with the specified name is not found."
19
+ INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE = "Signature not valid in the specified time frame"
20
+ INVALID_BLOB_EXPIRED_MESSAGE = "The signature expired."
21
+ INVALID_BLOB_INVALID_CONNECTION_STRING_MESSAGE = "The connection string is not valid."
22
+ INVALID_BLOB_PATH_MESSAGE = "BlobNotFound"
23
+ INVALID_BLOB_INVALID_PATH_MESSAGE = "The path to the data is not found."
24
+
25
+ attr_reader :use_sas
26
+
27
+ def initialize(options = {})
28
+ raise("Data Source needs a client to Blob Storage to be able to get blob file but 'blobStorage_client' is empty.") unless options['blobStorage_client']
29
+
30
+ if options['blobStorage_client']['connectionString'] && options['blobStorage_client']['container']
31
+ @connection_string = options['blobStorage_client']['connectionString']
32
+ @container = options['blobStorage_client']['container']
33
+ @path = options['blobStorage_client']['path']
34
+ @use_sas = false
35
+ build_sas(@connection_string)
36
+ else
37
+ raise('Missing connection info for Blob Storage client')
38
+ end
39
+ end
40
+
41
+ def realize_blob(file, _params)
42
+ GoodData.gd_logger.info("Realizing download from Blob Storage. Container #{@container}.")
43
+ filename = ''
44
+ begin
45
+ connect
46
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
47
+ blob_name = get_blob_name(@path, file)
48
+
49
+ measure = Benchmark.measure do
50
+ _blob, content = @client.get_blob(@container, blob_name)
51
+ File.open(filename, "wb") { |f| f.write(content) }
52
+ end
53
+ rescue => e
54
+ raise_error(e)
55
+ end
56
+ GoodData.gd_logger.info("Done downloading file type=blobStorage status=finished duration=#{measure.real}")
57
+ filename
58
+ end
59
+
60
+ def connect
61
+ GoodData.logger.info "Setting up connection to Blob Storage"
62
+ if use_sas
63
+ @client = Azure::Storage::Blob::BlobService.create(:storage_blob_host => @host, :storage_sas_token => @sas_token)
64
+ else
65
+ @client = Azure::Storage::Blob::BlobService.create_from_connection_string(@connection_string)
66
+ end
67
+ end
68
+
69
+ def build_sas(url)
70
+ matches = url.scan(SAS_URL_PATTERN)
71
+ return unless matches && matches[0]
72
+
73
+ @use_sas = true
74
+ @host = matches[0][0]
75
+ @sas_token = matches[0][1]
76
+ end
77
+
78
+ def raise_error(e)
79
+ if e.message && e.message.include?(INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE)
80
+ raise INVALID_BLOB_EXPIRED_MESSAGE
81
+ elsif e.message && e.message.include?(INVALID_BLOB_SIG_WELL_FORMED_MESSAGE)
82
+ raise INVALID_BLOB_SIG_WELL_FORMED_MESSAGE
83
+ elsif e.message && e.message.include?(INVALID_BLOB_CONTAINER_MESSAGE)
84
+ raise INVALID_BLOB_CONTAINER_FORMED_MESSAGE
85
+ elsif e.message && e.message.include?(INVALID_BLOB_PATH_MESSAGE)
86
+ raise INVALID_BLOB_INVALID_PATH_MESSAGE
87
+ else
88
+ raise INVALID_BLOB_GENERAL_MESSAGE
89
+ end
90
+ end
91
+
92
+ def get_blob_name(path, file)
93
+ return file unless path
94
+
95
+ path.rindex('/') == path.length - 1 ? "#{path}#{file}" : "#{path}/#{file}"
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,122 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'java'
10
+ require 'pathname'
11
+ require_relative '../cloud_resource_client'
12
+
13
+ base = Pathname(__FILE__).dirname.expand_path
14
+ Dir.glob(base + 'drivers/*.jar').each do |file|
15
+ require file unless file.start_with?('lcm-mssql-driver')
16
+ end
17
+
18
+ module GoodData
19
+ module CloudResources
20
+ class MSSQLClient < CloudResourceClient
21
+ MSSQL_SEPARATOR_PARAM = ";"
22
+ MSSQL_URL_PATTERN = %r{jdbc:sqlserver://([^:]+)(:([0-9]+))?(/)?}
23
+ MSSQL_DEFAULT_PORT = 1433
24
+ LOGIN_TIME_OUT = 30
25
+ MSSQL_FETCH_SIZE = 10_000
26
+ VERIFY_FULL = 'verify-full'
27
+ PREFER = 'prefer'
28
+ REQUIRE = 'require'
29
+
30
+ class << self
31
+ def accept?(type)
32
+ type == 'mssql'
33
+ end
34
+ end
35
+
36
+ def initialize(options = {})
37
+ raise("Data Source needs a client to MSSQL to be able to query the storage but 'mssql_client' is empty.") unless options['mssql_client']
38
+
39
+ connection = options['mssql_client']['connection']
40
+ if connection.is_a?(Hash)
41
+ @database = connection['database']
42
+ @schema = connection['schema']
43
+ @authentication = connection['authentication']
44
+ @ssl_mode = connection['sslMode']
45
+ @url = connection['url']
46
+
47
+ validate
48
+ else
49
+ raise('Missing connection info for MSSQL client')
50
+ end
51
+
52
+ Java.com.microsoft.sqlserver.jdbc.SQLServerDriver
53
+ end
54
+
55
+ def realize_query(query, _params)
56
+ GoodData.gd_logger.info("Realize SQL query: type=mssql status=started")
57
+
58
+ connect
59
+
60
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
61
+ measure = Benchmark.measure do
62
+ statement = @connection.create_statement
63
+ statement.set_fetch_size(MSSQL_FETCH_SIZE)
64
+ has_result = statement.execute(query)
65
+ if has_result
66
+ result = statement.get_result_set
67
+ metadata = result.get_meta_data
68
+ col_count = metadata.column_count
69
+ CSV.open(filename, 'wb') do |csv|
70
+ csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
71
+ csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
72
+ end
73
+ end
74
+ end
75
+
76
+ GoodData.gd_logger.info("Realize SQL query: type=mssql status=finished duration=#{measure.real}")
77
+ filename
78
+ ensure
79
+ @connection&.close
80
+ @connection = nil
81
+ end
82
+
83
+ def connect
84
+ connection_string = build_connection_string
85
+ GoodData.logger.info "Setting up connection to MSSQL #{connection_string}"
86
+
87
+ authentication = @authentication['basic'] || @authentication['activeDirectoryPassword']
88
+
89
+ prop = java.util.Properties.new
90
+ prop.setProperty('userName', authentication['userName'])
91
+ prop.setProperty('password', authentication['password'])
92
+
93
+ @connection = java.sql.DriverManager.getConnection(connection_string, prop)
94
+ end
95
+
96
+ def validate
97
+ raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
98
+
99
+ raise "Instance name is not supported" if @url !~ /^[^\\]*$/
100
+
101
+ raise "The connection url is invalid. Parameter is not supported." if @url.include? MSSQL_SEPARATOR_PARAM
102
+
103
+ url_matches = @url.scan(MSSQL_URL_PATTERN)
104
+ raise "Cannot reach the url" if url_matches.nil? || url_matches.length.zero?
105
+
106
+ raise "The authentication method is not supported." unless @authentication['basic'] || @authentication['activeDirectoryPassword']
107
+ end
108
+
109
+ def build_connection_string
110
+ encrypt = @ssl_mode != PREFER
111
+ trust_server_certificate = @ssl_mode == REQUIRE
112
+
113
+ "#{@url};" \
114
+ "database=#{@database};" \
115
+ "encrypt=#{encrypt};" \
116
+ "trustServerCertificate=#{trust_server_certificate};" \
117
+ "loginTimeout=#{LOGIN_TIME_OUT};" \
118
+ "#{'authentication=ActiveDirectoryPassword;' if @authentication['activeDirectoryPassword']}"
119
+ end
120
+ end
121
+ end
122
+ end
@@ -0,0 +1,111 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'java'
10
+ require 'pathname'
11
+ require_relative '../cloud_resource_client'
12
+
13
+ base = Pathname(__FILE__).dirname.expand_path
14
+ Dir.glob(base + 'drivers/*.jar').each do |file|
15
+ require file unless file.start_with?('lcm-mysql-driver')
16
+ end
17
+
18
+ module GoodData
19
+ module CloudResources
20
+ class MysqlClient < CloudResourceClient
21
+ JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
22
+ MYSQL_DEFAULT_PORT = 3306
23
+ JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://'
24
+ VERIFY_FULL = 'VERIFY_IDENTITY'
25
+ PREFER = 'PREFERRED'
26
+ REQUIRE = 'REQUIRED'
27
+ MYSQL_FETCH_SIZE = 1000
28
+
29
+ class << self
30
+ def accept?(type)
31
+ type == 'mysql'
32
+ end
33
+ end
34
+
35
+ def initialize(options = {})
36
+ raise("Data Source needs a client to Mysql to be able to query the storage but 'mysql_client' is empty.") unless options['mysql_client']
37
+
38
+ if options['mysql_client']['connection'].is_a?(Hash)
39
+ @database = options['mysql_client']['connection']['database']
40
+ @authentication = options['mysql_client']['connection']['authentication']
41
+ @ssl_mode = options['mysql_client']['connection']['sslMode']
42
+ raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
43
+
44
+ @url = build_url(options['mysql_client']['connection']['url'])
45
+ else
46
+ raise('Missing connection info for Mysql client')
47
+ end
48
+
49
+ Java.com.mysql.cj.jdbc.Driver
50
+ end
51
+
52
+ def realize_query(query, _params)
53
+ GoodData.gd_logger.info("Realize SQL query: type=mysql status=started")
54
+
55
+ connect
56
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
57
+ measure = Benchmark.measure do
58
+ statement = @connection.create_statement
59
+ statement.set_fetch_size(MYSQL_FETCH_SIZE)
60
+ has_result = statement.execute(query)
61
+ if has_result
62
+ result = statement.get_result_set
63
+ metadata = result.get_meta_data
64
+ col_count = metadata.column_count
65
+ CSV.open(filename, 'wb') do |csv|
66
+ csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
67
+ csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
68
+ end
69
+ end
70
+ end
71
+ GoodData.gd_logger.info("Realize SQL query: type=mysql status=finished duration=#{measure.real}")
72
+ filename
73
+ ensure
74
+ @connection&.close
75
+ @connection = nil
76
+ end
77
+
78
+ def connect
79
+ GoodData.logger.info "Setting up connection to Mysql #{@url}"
80
+
81
+ prop = java.util.Properties.new
82
+ prop.setProperty('user', @authentication['basic']['userName'])
83
+ prop.setProperty('password', @authentication['basic']['password'])
84
+
85
+ @connection = java.sql.DriverManager.getConnection(@url, prop)
86
+ @connection.set_auto_commit(false)
87
+ end
88
+
89
+ def build_url(url)
90
+ matches = url.scan(JDBC_MYSQL_PATTERN)
91
+ raise 'Cannot reach the url' unless matches
92
+
93
+ host = matches[0][0]
94
+ port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT
95
+
96
+ "#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{get_ssl_mode(@ssl_mode)}&useCursorFetch=true&enabledTLSProtocols=TLSv1.2"
97
+ end
98
+
99
+ def get_ssl_mode(ssl_mode)
100
+ mode = PREFER
101
+ if ssl_mode == 'verify-full'
102
+ mode = VERIFY_FULL
103
+ elsif ssl_mode == 'require'
104
+ mode = REQUIRE
105
+ end
106
+
107
+ mode
108
+ end
109
+ end
110
+ end
111
+ end
@@ -98,7 +98,6 @@ module GoodData
98
98
 
99
99
  host = matches[0][0]
100
100
  port = matches[0][2]&.to_i || POSTGRES_DEFAULT_PORT
101
- raise "Custom port #{port} is not supported. Remove it or use the default port '5432'" if POSTGRES_DEFAULT_PORT != port
102
101
 
103
102
  "#{JDBC_POSTGRES_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{@ssl_mode}#{VERIFY_FULL == @ssl_mode ? SSL_JAVA_FACTORY : ''}"
104
103
  end
@@ -18,6 +18,9 @@ end
18
18
  module GoodData
19
19
  module CloudResources
20
20
  class SnowflakeClient < CloudResourceClient
21
+ SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=GoodData_Platform'
22
+ SNOWFLAKE_SEPARATOR_PARAM = '?'
23
+
21
24
  class << self
22
25
  def accept?(type)
23
26
  type == 'snowflake'
@@ -31,7 +34,7 @@ module GoodData
31
34
  @database = options['snowflake_client']['connection']['database']
32
35
  @schema = options['snowflake_client']['connection']['schema'] || 'public'
33
36
  @warehouse = options['snowflake_client']['connection']['warehouse']
34
- @url = options['snowflake_client']['connection']['url']
37
+ @url = build_url(options['snowflake_client']['connection']['url'])
35
38
  @authentication = options['snowflake_client']['connection']['authentication']
36
39
  else
37
40
  raise('Missing connection info for Snowflake client')
@@ -79,6 +82,20 @@ module GoodData
79
82
 
80
83
  @connection = java.sql.DriverManager.getConnection(@url, prop)
81
84
  end
85
+
86
+ def build_url(url)
87
+ is_contain = url.include?(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
88
+ unless is_contain
89
+ if url.include?(SNOWFLAKE_SEPARATOR_PARAM)
90
+ url.concat("&")
91
+ else
92
+ url.concat(SNOWFLAKE_SEPARATOR_PARAM)
93
+ end
94
+ url.concat(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
95
+ end
96
+
97
+ url
98
+ end
82
99
  end
83
100
  end
84
101
  end