gooddata 2.1.19 → 2.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gdc-ii-config.yaml +1 -1
- data/.github/workflows/build.yml +66 -0
- data/.github/workflows/pre-merge.yml +72 -0
- data/CHANGELOG.md +38 -0
- data/Dockerfile +21 -14
- data/Dockerfile.jruby +1 -11
- data/LICENSE +4409 -16
- data/README.md +1 -2
- data/SDK_VERSION +1 -1
- data/VERSION +1 -1
- data/ci/mssql/pom.xml +62 -0
- data/ci/mysql/pom.xml +57 -0
- data/ci/redshift/pom.xml +1 -1
- data/docker-compose.lcm.yml +0 -3
- data/gooddata.gemspec +2 -1
- data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
- data/lcm.rake +2 -8
- data/lib/gooddata/bricks/middleware/aws_middleware.rb +35 -9
- data/lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb +98 -0
- data/lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mssql/mssql_client.rb +122 -0
- data/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +111 -0
- data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +0 -1
- data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +18 -1
- data/lib/gooddata/helpers/data_helper.rb +9 -4
- data/lib/gooddata/lcm/actions/collect_meta.rb +3 -1
- data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +3 -2
- data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -7
- data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +64 -0
- data/lib/gooddata/lcm/actions/synchronize_ldm.rb +19 -8
- data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +12 -9
- data/lib/gooddata/lcm/actions/update_metric_formats.rb +185 -0
- data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb +5 -0
- data/lib/gooddata/lcm/helpers/release_table_helper.rb +42 -8
- data/lib/gooddata/lcm/lcm2.rb +5 -0
- data/lib/gooddata/mixins/md_object_query.rb +1 -0
- data/lib/gooddata/models/data_source.rb +5 -1
- data/lib/gooddata/models/dataset_mapping.rb +36 -0
- data/lib/gooddata/models/metadata/label.rb +26 -27
- data/lib/gooddata/models/project.rb +34 -9
- data/lib/gooddata/models/schedule.rb +13 -1
- data/lib/gooddata/models/user_filters/user_filter_builder.rb +58 -53
- data/lib/gooddata/rest/phmap.rb +1 -0
- metadata +45 -18
- data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb +0 -37
data/README.md
CHANGED
@@ -18,8 +18,7 @@ Feel free to check out the [GoodData community website](http://community.gooddat
|
|
18
18
|
[![Downloads](http://img.shields.io/gem/dt/gooddata.svg)](http://rubygems.org/gems/gooddata)
|
19
19
|
[![Dependency Status](https://gemnasium.com/gooddata/gooddata-ruby.png)](https://gemnasium.com/gooddata/gooddata-ruby)
|
20
20
|
[![Code Climate](https://codeclimate.com/github/gooddata/gooddata-ruby.png)](https://codeclimate.com/github/gooddata/gooddata-ruby)
|
21
|
-
[![Build Status](https://
|
22
|
-
[![Coverage Status](https://coveralls.io/repos/gooddata/gooddata-ruby/badge.png)](https://coveralls.io/r/gooddata/gooddata-ruby)
|
21
|
+
[![Build Status](https://github.com/gooddata/gooddata-ruby/actions/workflows/build.yml/badge.svg)](https://github.com/gooddata/gooddata-ruby/actions/workflows/build.yml/)
|
23
22
|
|
24
23
|
## Supported versions
|
25
24
|
|
data/SDK_VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.
|
1
|
+
2.2.0
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
3.7.
|
1
|
+
3.7.43
|
data/ci/mssql/pom.xml
ADDED
@@ -0,0 +1,62 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
+
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
3
|
+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
4
|
+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
5
|
+
<modelVersion>4.0.0</modelVersion>
|
6
|
+
|
7
|
+
<groupId>com.gooddata.lcm</groupId>
|
8
|
+
<artifactId>lcm-mssql-driver</artifactId>
|
9
|
+
<version>1.0-SNAPSHOT</version>
|
10
|
+
|
11
|
+
<dependencies>
|
12
|
+
<dependency>
|
13
|
+
<groupId>com.microsoft.sqlserver</groupId>
|
14
|
+
<artifactId>mssql-jdbc</artifactId>
|
15
|
+
<version>9.3.1.jre8-preview</version>
|
16
|
+
</dependency>
|
17
|
+
<dependency>
|
18
|
+
<groupId>com.microsoft.azure</groupId>
|
19
|
+
<artifactId>msal4j</artifactId>
|
20
|
+
<version>1.10.1</version>
|
21
|
+
</dependency>
|
22
|
+
<dependency>
|
23
|
+
<groupId>org.slf4j</groupId>
|
24
|
+
<artifactId>slf4j-api</artifactId>
|
25
|
+
<version>1.7.2</version>
|
26
|
+
</dependency>
|
27
|
+
</dependencies>
|
28
|
+
|
29
|
+
<profiles>
|
30
|
+
<profile>
|
31
|
+
<id>binary-packaging</id>
|
32
|
+
<build>
|
33
|
+
<plugins>
|
34
|
+
<plugin>
|
35
|
+
<artifactId>maven-dependency-plugin</artifactId>
|
36
|
+
<executions>
|
37
|
+
<execution>
|
38
|
+
<phase>package</phase>
|
39
|
+
<goals>
|
40
|
+
<goal>copy-dependencies</goal>
|
41
|
+
</goals>
|
42
|
+
<configuration>
|
43
|
+
<outputDirectory>${project.build.directory}</outputDirectory>
|
44
|
+
<!-- compile scope gives runtime and compile dependencies (skips test deps) -->
|
45
|
+
<includeScope>runtime</includeScope>
|
46
|
+
</configuration>
|
47
|
+
</execution>
|
48
|
+
</executions>
|
49
|
+
</plugin>
|
50
|
+
</plugins>
|
51
|
+
</build>
|
52
|
+
</profile>
|
53
|
+
</profiles>
|
54
|
+
|
55
|
+
<repositories>
|
56
|
+
<repository>
|
57
|
+
<id>my-repo1</id>
|
58
|
+
<name>my custom repo</name>
|
59
|
+
<url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
|
60
|
+
</repository>
|
61
|
+
</repositories>
|
62
|
+
</project>
|
data/ci/mysql/pom.xml
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
+
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
3
|
+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
4
|
+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
5
|
+
<modelVersion>4.0.0</modelVersion>
|
6
|
+
|
7
|
+
<groupId>com.gooddata.lcm</groupId>
|
8
|
+
<artifactId>lcm-mysql-driver</artifactId>
|
9
|
+
<version>1.0-SNAPSHOT</version>
|
10
|
+
|
11
|
+
<dependencies>
|
12
|
+
<dependency>
|
13
|
+
<groupId>mysql</groupId>
|
14
|
+
<artifactId>mysql-connector-java</artifactId>
|
15
|
+
<version>8.0.25</version>
|
16
|
+
</dependency>
|
17
|
+
<dependency>
|
18
|
+
<groupId>org.slf4j</groupId>
|
19
|
+
<artifactId>slf4j-api</artifactId>
|
20
|
+
<version>1.7.2</version>
|
21
|
+
</dependency>
|
22
|
+
</dependencies>
|
23
|
+
|
24
|
+
<profiles>
|
25
|
+
<profile>
|
26
|
+
<id>binary-packaging</id>
|
27
|
+
<build>
|
28
|
+
<plugins>
|
29
|
+
<plugin>
|
30
|
+
<artifactId>maven-dependency-plugin</artifactId>
|
31
|
+
<executions>
|
32
|
+
<execution>
|
33
|
+
<phase>package</phase>
|
34
|
+
<goals>
|
35
|
+
<goal>copy-dependencies</goal>
|
36
|
+
</goals>
|
37
|
+
<configuration>
|
38
|
+
<outputDirectory>${project.build.directory}</outputDirectory>
|
39
|
+
<!-- compile scope gives runtime and compile dependencies (skips test deps) -->
|
40
|
+
<includeScope>runtime</includeScope>
|
41
|
+
</configuration>
|
42
|
+
</execution>
|
43
|
+
</executions>
|
44
|
+
</plugin>
|
45
|
+
</plugins>
|
46
|
+
</build>
|
47
|
+
</profile>
|
48
|
+
</profiles>
|
49
|
+
|
50
|
+
<repositories>
|
51
|
+
<repository>
|
52
|
+
<id>my-repo1</id>
|
53
|
+
<name>my custom repo</name>
|
54
|
+
<url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
|
55
|
+
</repository>
|
56
|
+
</repositories>
|
57
|
+
</project>
|
data/ci/redshift/pom.xml
CHANGED
@@ -67,7 +67,7 @@
|
|
67
67
|
<repository>
|
68
68
|
<id>my-repo1</id>
|
69
69
|
<name>my custom repo</name>
|
70
|
-
<url>https://
|
70
|
+
<url>https://sonatype-nexus.intgdc.com/repository/public/</url>
|
71
71
|
</repository>
|
72
72
|
</repositories>
|
73
73
|
</project>
|
data/docker-compose.lcm.yml
CHANGED
data/gooddata.gemspec
CHANGED
@@ -66,6 +66,8 @@ Gem::Specification.new do |s|
|
|
66
66
|
else
|
67
67
|
s.add_dependency 'docile', '> 1.1', '< 1.4.0'
|
68
68
|
end
|
69
|
+
s.add_dependency 'azure-storage-blob', '~> 1.1.0'
|
70
|
+
s.add_dependency 'nokogiri', '~> 1.10.0'
|
69
71
|
s.add_dependency 'gli', '~> 2.15'
|
70
72
|
s.add_dependency 'gooddata_datawarehouse', '~> 0.0.10' if RUBY_PLATFORM == 'java'
|
71
73
|
s.add_dependency 'highline', '= 2.0.0.pre.develop.14'
|
@@ -77,7 +79,6 @@ Gem::Specification.new do |s|
|
|
77
79
|
s.add_dependency 'restforce', '>= 2.4', '< 4.0'
|
78
80
|
s.add_dependency 'rest-client', '~> 2.0'
|
79
81
|
s.add_dependency 'rubyzip', '~> 1.2', '>= 1.2.1'
|
80
|
-
s.add_dependency 'salesforce_bulk_query', '~> 0.2'
|
81
82
|
s.add_dependency 'terminal-table', '~> 1.7'
|
82
83
|
s.add_dependency 'thread_safe'
|
83
84
|
s.add_dependency 'backports'
|
data/lcm.rake
CHANGED
@@ -101,14 +101,8 @@ namespace :test do
|
|
101
101
|
test_cases.each do |t|
|
102
102
|
desc "Run #{t} tests in Docker"
|
103
103
|
task t do
|
104
|
-
|
105
|
-
|
106
|
-
fail('Test execution failed!')
|
107
|
-
else
|
108
|
-
system("docker-compose -f docker-compose.lcm.yml run --rm appstore bundle exec rake -f lcm.rake test:#{t}") ||
|
109
|
-
fail('Test execution failed!')
|
110
|
-
end
|
111
|
-
|
104
|
+
system("docker-compose -f docker-compose.lcm.yml run --rm appstore bundle exec rake -f lcm.rake test:#{t}") ||
|
105
|
+
fail('Test execution failed!')
|
112
106
|
end
|
113
107
|
end
|
114
108
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
#
|
3
|
-
# Copyright (c) 2010-
|
3
|
+
# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
|
4
4
|
# This source code is licensed under the BSD-style license found in the
|
5
5
|
# LICENSE file in the root directory of this source tree.
|
6
6
|
|
@@ -12,21 +12,47 @@ module GoodData
|
|
12
12
|
class AWSMiddleware < Bricks::Middleware
|
13
13
|
def call(params)
|
14
14
|
params = params.to_hash
|
15
|
-
|
15
|
+
s3_config = get_s3_config(params)
|
16
|
+
|
17
|
+
unless s3_config.empty?
|
16
18
|
GoodData.logger.info('Setting up AWS-S3 connection')
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
19
|
+
if params.key?('aws_client')
|
20
|
+
params['s3_client'] = {}
|
21
|
+
elsif params.key?('s3_client')
|
22
|
+
params['input_source'] = {} unless params.key?('input_source')
|
23
|
+
params['input_source']['bucket'] = params['s3_client']['bucket']
|
24
|
+
end
|
25
|
+
s3_config = rewrite_for_aws_sdk_v2(s3_config)
|
26
|
+
symbolized_config = GoodData::Helpers.symbolize_keys(s3_config)
|
27
|
+
params['s3_client']['client'] = Aws::S3::Resource.new(symbolized_config)
|
24
28
|
end
|
25
29
|
@app.call(params)
|
26
30
|
end
|
27
31
|
|
28
32
|
private
|
29
33
|
|
34
|
+
def get_s3_config(params)
|
35
|
+
s3_config = {}
|
36
|
+
if params.key?('aws_client')
|
37
|
+
GoodData.logger.warn('Found two configuration aws_client and s3_client for S3 input source, use aws_client configuration') if params.key?('s3_client')
|
38
|
+
raise 'Unable to connect to AWS. Parameter "aws_client" seems to be empty' unless params['aws_client']
|
39
|
+
raise 'Unable to connect to AWS. Parameter "access_key_id" is missing' if params['aws_client']['access_key_id'].blank?
|
40
|
+
raise 'Unable to connect to AWS. Parameter "secret_access_key" is missing' if params['aws_client']['secret_access_key'].blank?
|
41
|
+
|
42
|
+
s3_config = params['aws_client']
|
43
|
+
elsif params.key?('s3_client')
|
44
|
+
raise 'Unable to connect to AWS. Parameter "s3_client" seems to be empty' unless params['s3_client']
|
45
|
+
raise 'Unable to connect to AWS. Parameter "accessKey" is missing' if params['s3_client']['accessKey'].blank?
|
46
|
+
raise 'Unable to connect to AWS. Parameter "secretKey" is missing' if params['s3_client']['secretKey'].blank?
|
47
|
+
raise 'Unable to connect to AWS. Parameter "bucket" is missing' if params['s3_client']['bucket'].blank?
|
48
|
+
|
49
|
+
s3_config['access_key_id'] = params['s3_client']['accessKey']
|
50
|
+
s3_config['secret_access_key'] = params['s3_client']['secretKey']
|
51
|
+
s3_config['region'] = params['s3_client']['region']
|
52
|
+
end
|
53
|
+
s3_config
|
54
|
+
end
|
55
|
+
|
30
56
|
def rewrite_for_aws_sdk_v2(config)
|
31
57
|
config['region'] = 'us-west-2' unless config['region']
|
32
58
|
if config['use_ssl']
|
@@ -0,0 +1,98 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'pathname'
|
10
|
+
require "azure/storage/blob"
|
11
|
+
|
12
|
+
module GoodData
|
13
|
+
class BlobStorageClient
|
14
|
+
SAS_URL_PATTERN = %r{(^https?:\/\/[^\/]*)\/.*\?(.*)}
|
15
|
+
INVALID_BLOB_GENERAL_MESSAGE = "The connection string is not valid."
|
16
|
+
INVALID_BLOB_SIG_WELL_FORMED_MESSAGE = "The signature format is not valid."
|
17
|
+
INVALID_BLOB_CONTAINER_MESSAGE = "ContainerNotFound"
|
18
|
+
INVALID_BLOB_CONTAINER_FORMED_MESSAGE = "The container with the specified name is not found."
|
19
|
+
INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE = "Signature not valid in the specified time frame"
|
20
|
+
INVALID_BLOB_EXPIRED_MESSAGE = "The signature expired."
|
21
|
+
INVALID_BLOB_INVALID_CONNECTION_STRING_MESSAGE = "The connection string is not valid."
|
22
|
+
INVALID_BLOB_PATH_MESSAGE = "BlobNotFound"
|
23
|
+
INVALID_BLOB_INVALID_PATH_MESSAGE = "The path to the data is not found."
|
24
|
+
|
25
|
+
attr_reader :use_sas
|
26
|
+
|
27
|
+
def initialize(options = {})
|
28
|
+
raise("Data Source needs a client to Blob Storage to be able to get blob file but 'blobStorage_client' is empty.") unless options['blobStorage_client']
|
29
|
+
|
30
|
+
if options['blobStorage_client']['connectionString'] && options['blobStorage_client']['container']
|
31
|
+
@connection_string = options['blobStorage_client']['connectionString']
|
32
|
+
@container = options['blobStorage_client']['container']
|
33
|
+
@path = options['blobStorage_client']['path']
|
34
|
+
@use_sas = false
|
35
|
+
build_sas(@connection_string)
|
36
|
+
else
|
37
|
+
raise('Missing connection info for Blob Storage client')
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def realize_blob(file, _params)
|
42
|
+
GoodData.gd_logger.info("Realizing download from Blob Storage. Container #{@container}.")
|
43
|
+
filename = ''
|
44
|
+
begin
|
45
|
+
connect
|
46
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
47
|
+
blob_name = get_blob_name(@path, file)
|
48
|
+
|
49
|
+
measure = Benchmark.measure do
|
50
|
+
_blob, content = @client.get_blob(@container, blob_name)
|
51
|
+
File.open(filename, "wb") { |f| f.write(content) }
|
52
|
+
end
|
53
|
+
rescue => e
|
54
|
+
raise_error(e)
|
55
|
+
end
|
56
|
+
GoodData.gd_logger.info("Done downloading file type=blobStorage status=finished duration=#{measure.real}")
|
57
|
+
filename
|
58
|
+
end
|
59
|
+
|
60
|
+
def connect
|
61
|
+
GoodData.logger.info "Setting up connection to Blob Storage"
|
62
|
+
if use_sas
|
63
|
+
@client = Azure::Storage::Blob::BlobService.create(:storage_blob_host => @host, :storage_sas_token => @sas_token)
|
64
|
+
else
|
65
|
+
@client = Azure::Storage::Blob::BlobService.create_from_connection_string(@connection_string)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def build_sas(url)
|
70
|
+
matches = url.scan(SAS_URL_PATTERN)
|
71
|
+
return unless matches && matches[0]
|
72
|
+
|
73
|
+
@use_sas = true
|
74
|
+
@host = matches[0][0]
|
75
|
+
@sas_token = matches[0][1]
|
76
|
+
end
|
77
|
+
|
78
|
+
def raise_error(e)
|
79
|
+
if e.message && e.message.include?(INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE)
|
80
|
+
raise INVALID_BLOB_EXPIRED_MESSAGE
|
81
|
+
elsif e.message && e.message.include?(INVALID_BLOB_SIG_WELL_FORMED_MESSAGE)
|
82
|
+
raise INVALID_BLOB_SIG_WELL_FORMED_MESSAGE
|
83
|
+
elsif e.message && e.message.include?(INVALID_BLOB_CONTAINER_MESSAGE)
|
84
|
+
raise INVALID_BLOB_CONTAINER_FORMED_MESSAGE
|
85
|
+
elsif e.message && e.message.include?(INVALID_BLOB_PATH_MESSAGE)
|
86
|
+
raise INVALID_BLOB_INVALID_PATH_MESSAGE
|
87
|
+
else
|
88
|
+
raise INVALID_BLOB_GENERAL_MESSAGE
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def get_blob_name(path, file)
|
93
|
+
return file unless path
|
94
|
+
|
95
|
+
path.rindex('/') == path.length - 1 ? "#{path}#{file}" : "#{path}/#{file}"
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
File without changes
|
@@ -0,0 +1,122 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'java'
|
10
|
+
require 'pathname'
|
11
|
+
require_relative '../cloud_resource_client'
|
12
|
+
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
15
|
+
require file unless file.start_with?('lcm-mssql-driver')
|
16
|
+
end
|
17
|
+
|
18
|
+
module GoodData
|
19
|
+
module CloudResources
|
20
|
+
class MSSQLClient < CloudResourceClient
|
21
|
+
MSSQL_SEPARATOR_PARAM = ";"
|
22
|
+
MSSQL_URL_PATTERN = %r{jdbc:sqlserver://([^:]+)(:([0-9]+))?(/)?}
|
23
|
+
MSSQL_DEFAULT_PORT = 1433
|
24
|
+
LOGIN_TIME_OUT = 30
|
25
|
+
MSSQL_FETCH_SIZE = 10_000
|
26
|
+
VERIFY_FULL = 'verify-full'
|
27
|
+
PREFER = 'prefer'
|
28
|
+
REQUIRE = 'require'
|
29
|
+
|
30
|
+
class << self
|
31
|
+
def accept?(type)
|
32
|
+
type == 'mssql'
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def initialize(options = {})
|
37
|
+
raise("Data Source needs a client to MSSQL to be able to query the storage but 'mssql_client' is empty.") unless options['mssql_client']
|
38
|
+
|
39
|
+
connection = options['mssql_client']['connection']
|
40
|
+
if connection.is_a?(Hash)
|
41
|
+
@database = connection['database']
|
42
|
+
@schema = connection['schema']
|
43
|
+
@authentication = connection['authentication']
|
44
|
+
@ssl_mode = connection['sslMode']
|
45
|
+
@url = connection['url']
|
46
|
+
|
47
|
+
validate
|
48
|
+
else
|
49
|
+
raise('Missing connection info for MSSQL client')
|
50
|
+
end
|
51
|
+
|
52
|
+
Java.com.microsoft.sqlserver.jdbc.SQLServerDriver
|
53
|
+
end
|
54
|
+
|
55
|
+
def realize_query(query, _params)
|
56
|
+
GoodData.gd_logger.info("Realize SQL query: type=mssql status=started")
|
57
|
+
|
58
|
+
connect
|
59
|
+
|
60
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
61
|
+
measure = Benchmark.measure do
|
62
|
+
statement = @connection.create_statement
|
63
|
+
statement.set_fetch_size(MSSQL_FETCH_SIZE)
|
64
|
+
has_result = statement.execute(query)
|
65
|
+
if has_result
|
66
|
+
result = statement.get_result_set
|
67
|
+
metadata = result.get_meta_data
|
68
|
+
col_count = metadata.column_count
|
69
|
+
CSV.open(filename, 'wb') do |csv|
|
70
|
+
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
71
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
GoodData.gd_logger.info("Realize SQL query: type=mssql status=finished duration=#{measure.real}")
|
77
|
+
filename
|
78
|
+
ensure
|
79
|
+
@connection&.close
|
80
|
+
@connection = nil
|
81
|
+
end
|
82
|
+
|
83
|
+
def connect
|
84
|
+
connection_string = build_connection_string
|
85
|
+
GoodData.logger.info "Setting up connection to MSSQL #{connection_string}"
|
86
|
+
|
87
|
+
authentication = @authentication['basic'] || @authentication['activeDirectoryPassword']
|
88
|
+
|
89
|
+
prop = java.util.Properties.new
|
90
|
+
prop.setProperty('userName', authentication['userName'])
|
91
|
+
prop.setProperty('password', authentication['password'])
|
92
|
+
|
93
|
+
@connection = java.sql.DriverManager.getConnection(connection_string, prop)
|
94
|
+
end
|
95
|
+
|
96
|
+
def validate
|
97
|
+
raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
|
98
|
+
|
99
|
+
raise "Instance name is not supported" if @url !~ /^[^\\]*$/
|
100
|
+
|
101
|
+
raise "The connection url is invalid. Parameter is not supported." if @url.include? MSSQL_SEPARATOR_PARAM
|
102
|
+
|
103
|
+
url_matches = @url.scan(MSSQL_URL_PATTERN)
|
104
|
+
raise "Cannot reach the url" if url_matches.nil? || url_matches.length.zero?
|
105
|
+
|
106
|
+
raise "The authentication method is not supported." unless @authentication['basic'] || @authentication['activeDirectoryPassword']
|
107
|
+
end
|
108
|
+
|
109
|
+
def build_connection_string
|
110
|
+
encrypt = @ssl_mode != PREFER
|
111
|
+
trust_server_certificate = @ssl_mode == REQUIRE
|
112
|
+
|
113
|
+
"#{@url};" \
|
114
|
+
"database=#{@database};" \
|
115
|
+
"encrypt=#{encrypt};" \
|
116
|
+
"trustServerCertificate=#{trust_server_certificate};" \
|
117
|
+
"loginTimeout=#{LOGIN_TIME_OUT};" \
|
118
|
+
"#{'authentication=ActiveDirectoryPassword;' if @authentication['activeDirectoryPassword']}"
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
File without changes
|
@@ -0,0 +1,111 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'java'
|
10
|
+
require 'pathname'
|
11
|
+
require_relative '../cloud_resource_client'
|
12
|
+
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
15
|
+
require file unless file.start_with?('lcm-mysql-driver')
|
16
|
+
end
|
17
|
+
|
18
|
+
module GoodData
|
19
|
+
module CloudResources
|
20
|
+
class MysqlClient < CloudResourceClient
|
21
|
+
JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
|
22
|
+
MYSQL_DEFAULT_PORT = 3306
|
23
|
+
JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://'
|
24
|
+
VERIFY_FULL = 'VERIFY_IDENTITY'
|
25
|
+
PREFER = 'PREFERRED'
|
26
|
+
REQUIRE = 'REQUIRED'
|
27
|
+
MYSQL_FETCH_SIZE = 1000
|
28
|
+
|
29
|
+
class << self
|
30
|
+
def accept?(type)
|
31
|
+
type == 'mysql'
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def initialize(options = {})
|
36
|
+
raise("Data Source needs a client to Mysql to be able to query the storage but 'mysql_client' is empty.") unless options['mysql_client']
|
37
|
+
|
38
|
+
if options['mysql_client']['connection'].is_a?(Hash)
|
39
|
+
@database = options['mysql_client']['connection']['database']
|
40
|
+
@authentication = options['mysql_client']['connection']['authentication']
|
41
|
+
@ssl_mode = options['mysql_client']['connection']['sslMode']
|
42
|
+
raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
|
43
|
+
|
44
|
+
@url = build_url(options['mysql_client']['connection']['url'])
|
45
|
+
else
|
46
|
+
raise('Missing connection info for Mysql client')
|
47
|
+
end
|
48
|
+
|
49
|
+
Java.com.mysql.cj.jdbc.Driver
|
50
|
+
end
|
51
|
+
|
52
|
+
def realize_query(query, _params)
|
53
|
+
GoodData.gd_logger.info("Realize SQL query: type=mysql status=started")
|
54
|
+
|
55
|
+
connect
|
56
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
57
|
+
measure = Benchmark.measure do
|
58
|
+
statement = @connection.create_statement
|
59
|
+
statement.set_fetch_size(MYSQL_FETCH_SIZE)
|
60
|
+
has_result = statement.execute(query)
|
61
|
+
if has_result
|
62
|
+
result = statement.get_result_set
|
63
|
+
metadata = result.get_meta_data
|
64
|
+
col_count = metadata.column_count
|
65
|
+
CSV.open(filename, 'wb') do |csv|
|
66
|
+
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
67
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
GoodData.gd_logger.info("Realize SQL query: type=mysql status=finished duration=#{measure.real}")
|
72
|
+
filename
|
73
|
+
ensure
|
74
|
+
@connection&.close
|
75
|
+
@connection = nil
|
76
|
+
end
|
77
|
+
|
78
|
+
def connect
|
79
|
+
GoodData.logger.info "Setting up connection to Mysql #{@url}"
|
80
|
+
|
81
|
+
prop = java.util.Properties.new
|
82
|
+
prop.setProperty('user', @authentication['basic']['userName'])
|
83
|
+
prop.setProperty('password', @authentication['basic']['password'])
|
84
|
+
|
85
|
+
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
86
|
+
@connection.set_auto_commit(false)
|
87
|
+
end
|
88
|
+
|
89
|
+
def build_url(url)
|
90
|
+
matches = url.scan(JDBC_MYSQL_PATTERN)
|
91
|
+
raise 'Cannot reach the url' unless matches
|
92
|
+
|
93
|
+
host = matches[0][0]
|
94
|
+
port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT
|
95
|
+
|
96
|
+
"#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{get_ssl_mode(@ssl_mode)}&useCursorFetch=true&enabledTLSProtocols=TLSv1.2"
|
97
|
+
end
|
98
|
+
|
99
|
+
def get_ssl_mode(ssl_mode)
|
100
|
+
mode = PREFER
|
101
|
+
if ssl_mode == 'verify-full'
|
102
|
+
mode = VERIFY_FULL
|
103
|
+
elsif ssl_mode == 'require'
|
104
|
+
mode = REQUIRE
|
105
|
+
end
|
106
|
+
|
107
|
+
mode
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
@@ -98,7 +98,6 @@ module GoodData
|
|
98
98
|
|
99
99
|
host = matches[0][0]
|
100
100
|
port = matches[0][2]&.to_i || POSTGRES_DEFAULT_PORT
|
101
|
-
raise "Custom port #{port} is not supported. Remove it or use the default port '5432'" if POSTGRES_DEFAULT_PORT != port
|
102
101
|
|
103
102
|
"#{JDBC_POSTGRES_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{@ssl_mode}#{VERIFY_FULL == @ssl_mode ? SSL_JAVA_FACTORY : ''}"
|
104
103
|
end
|
@@ -18,6 +18,9 @@ end
|
|
18
18
|
module GoodData
|
19
19
|
module CloudResources
|
20
20
|
class SnowflakeClient < CloudResourceClient
|
21
|
+
SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=GoodData_Platform'
|
22
|
+
SNOWFLAKE_SEPARATOR_PARAM = '?'
|
23
|
+
|
21
24
|
class << self
|
22
25
|
def accept?(type)
|
23
26
|
type == 'snowflake'
|
@@ -31,7 +34,7 @@ module GoodData
|
|
31
34
|
@database = options['snowflake_client']['connection']['database']
|
32
35
|
@schema = options['snowflake_client']['connection']['schema'] || 'public'
|
33
36
|
@warehouse = options['snowflake_client']['connection']['warehouse']
|
34
|
-
@url = options['snowflake_client']['connection']['url']
|
37
|
+
@url = build_url(options['snowflake_client']['connection']['url'])
|
35
38
|
@authentication = options['snowflake_client']['connection']['authentication']
|
36
39
|
else
|
37
40
|
raise('Missing connection info for Snowflake client')
|
@@ -79,6 +82,20 @@ module GoodData
|
|
79
82
|
|
80
83
|
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
81
84
|
end
|
85
|
+
|
86
|
+
def build_url(url)
|
87
|
+
is_contain = url.include?(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
|
88
|
+
unless is_contain
|
89
|
+
if url.include?(SNOWFLAKE_SEPARATOR_PARAM)
|
90
|
+
url.concat("&")
|
91
|
+
else
|
92
|
+
url.concat(SNOWFLAKE_SEPARATOR_PARAM)
|
93
|
+
end
|
94
|
+
url.concat(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
|
95
|
+
end
|
96
|
+
|
97
|
+
url
|
98
|
+
end
|
82
99
|
end
|
83
100
|
end
|
84
101
|
end
|