gooddata 2.1.19 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gdc-ii-config.yaml +1 -1
- data/.github/workflows/build.yml +66 -0
- data/.github/workflows/pre-merge.yml +72 -0
- data/CHANGELOG.md +38 -0
- data/Dockerfile +21 -14
- data/Dockerfile.jruby +1 -11
- data/LICENSE +4409 -16
- data/README.md +1 -2
- data/SDK_VERSION +1 -1
- data/VERSION +1 -1
- data/ci/mssql/pom.xml +62 -0
- data/ci/mysql/pom.xml +57 -0
- data/ci/redshift/pom.xml +1 -1
- data/docker-compose.lcm.yml +0 -3
- data/gooddata.gemspec +2 -1
- data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
- data/lcm.rake +2 -8
- data/lib/gooddata/bricks/middleware/aws_middleware.rb +35 -9
- data/lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb +98 -0
- data/lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mssql/mssql_client.rb +122 -0
- data/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +111 -0
- data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +0 -1
- data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +18 -1
- data/lib/gooddata/helpers/data_helper.rb +9 -4
- data/lib/gooddata/lcm/actions/collect_meta.rb +3 -1
- data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +3 -2
- data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -7
- data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +64 -0
- data/lib/gooddata/lcm/actions/synchronize_ldm.rb +19 -8
- data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +12 -9
- data/lib/gooddata/lcm/actions/update_metric_formats.rb +185 -0
- data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb +5 -0
- data/lib/gooddata/lcm/helpers/release_table_helper.rb +42 -8
- data/lib/gooddata/lcm/lcm2.rb +5 -0
- data/lib/gooddata/mixins/md_object_query.rb +1 -0
- data/lib/gooddata/models/data_source.rb +5 -1
- data/lib/gooddata/models/dataset_mapping.rb +36 -0
- data/lib/gooddata/models/metadata/label.rb +26 -27
- data/lib/gooddata/models/project.rb +34 -9
- data/lib/gooddata/models/schedule.rb +13 -1
- data/lib/gooddata/models/user_filters/user_filter_builder.rb +58 -53
- data/lib/gooddata/rest/phmap.rb +1 -0
- metadata +45 -18
- data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb +0 -37
data/README.md
CHANGED
|
@@ -18,8 +18,7 @@ Feel free to check out the [GoodData community website](http://community.gooddat
|
|
|
18
18
|
[](http://rubygems.org/gems/gooddata)
|
|
19
19
|
[](https://gemnasium.com/gooddata/gooddata-ruby)
|
|
20
20
|
[](https://codeclimate.com/github/gooddata/gooddata-ruby)
|
|
21
|
-
[](https://coveralls.io/r/gooddata/gooddata-ruby)
|
|
21
|
+
[](https://github.com/gooddata/gooddata-ruby/actions/workflows/build.yml/)
|
|
23
22
|
|
|
24
23
|
## Supported versions
|
|
25
24
|
|
data/SDK_VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
2.
|
|
1
|
+
2.2.0
|
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
3.7.
|
|
1
|
+
3.7.43
|
data/ci/mssql/pom.xml
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
|
3
|
+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
4
|
+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
5
|
+
<modelVersion>4.0.0</modelVersion>
|
|
6
|
+
|
|
7
|
+
<groupId>com.gooddata.lcm</groupId>
|
|
8
|
+
<artifactId>lcm-mssql-driver</artifactId>
|
|
9
|
+
<version>1.0-SNAPSHOT</version>
|
|
10
|
+
|
|
11
|
+
<dependencies>
|
|
12
|
+
<dependency>
|
|
13
|
+
<groupId>com.microsoft.sqlserver</groupId>
|
|
14
|
+
<artifactId>mssql-jdbc</artifactId>
|
|
15
|
+
<version>9.3.1.jre8-preview</version>
|
|
16
|
+
</dependency>
|
|
17
|
+
<dependency>
|
|
18
|
+
<groupId>com.microsoft.azure</groupId>
|
|
19
|
+
<artifactId>msal4j</artifactId>
|
|
20
|
+
<version>1.10.1</version>
|
|
21
|
+
</dependency>
|
|
22
|
+
<dependency>
|
|
23
|
+
<groupId>org.slf4j</groupId>
|
|
24
|
+
<artifactId>slf4j-api</artifactId>
|
|
25
|
+
<version>1.7.2</version>
|
|
26
|
+
</dependency>
|
|
27
|
+
</dependencies>
|
|
28
|
+
|
|
29
|
+
<profiles>
|
|
30
|
+
<profile>
|
|
31
|
+
<id>binary-packaging</id>
|
|
32
|
+
<build>
|
|
33
|
+
<plugins>
|
|
34
|
+
<plugin>
|
|
35
|
+
<artifactId>maven-dependency-plugin</artifactId>
|
|
36
|
+
<executions>
|
|
37
|
+
<execution>
|
|
38
|
+
<phase>package</phase>
|
|
39
|
+
<goals>
|
|
40
|
+
<goal>copy-dependencies</goal>
|
|
41
|
+
</goals>
|
|
42
|
+
<configuration>
|
|
43
|
+
<outputDirectory>${project.build.directory}</outputDirectory>
|
|
44
|
+
<!-- compile scope gives runtime and compile dependencies (skips test deps) -->
|
|
45
|
+
<includeScope>runtime</includeScope>
|
|
46
|
+
</configuration>
|
|
47
|
+
</execution>
|
|
48
|
+
</executions>
|
|
49
|
+
</plugin>
|
|
50
|
+
</plugins>
|
|
51
|
+
</build>
|
|
52
|
+
</profile>
|
|
53
|
+
</profiles>
|
|
54
|
+
|
|
55
|
+
<repositories>
|
|
56
|
+
<repository>
|
|
57
|
+
<id>my-repo1</id>
|
|
58
|
+
<name>my custom repo</name>
|
|
59
|
+
<url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
|
|
60
|
+
</repository>
|
|
61
|
+
</repositories>
|
|
62
|
+
</project>
|
data/ci/mysql/pom.xml
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
|
3
|
+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
4
|
+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
5
|
+
<modelVersion>4.0.0</modelVersion>
|
|
6
|
+
|
|
7
|
+
<groupId>com.gooddata.lcm</groupId>
|
|
8
|
+
<artifactId>lcm-mysql-driver</artifactId>
|
|
9
|
+
<version>1.0-SNAPSHOT</version>
|
|
10
|
+
|
|
11
|
+
<dependencies>
|
|
12
|
+
<dependency>
|
|
13
|
+
<groupId>mysql</groupId>
|
|
14
|
+
<artifactId>mysql-connector-java</artifactId>
|
|
15
|
+
<version>8.0.25</version>
|
|
16
|
+
</dependency>
|
|
17
|
+
<dependency>
|
|
18
|
+
<groupId>org.slf4j</groupId>
|
|
19
|
+
<artifactId>slf4j-api</artifactId>
|
|
20
|
+
<version>1.7.2</version>
|
|
21
|
+
</dependency>
|
|
22
|
+
</dependencies>
|
|
23
|
+
|
|
24
|
+
<profiles>
|
|
25
|
+
<profile>
|
|
26
|
+
<id>binary-packaging</id>
|
|
27
|
+
<build>
|
|
28
|
+
<plugins>
|
|
29
|
+
<plugin>
|
|
30
|
+
<artifactId>maven-dependency-plugin</artifactId>
|
|
31
|
+
<executions>
|
|
32
|
+
<execution>
|
|
33
|
+
<phase>package</phase>
|
|
34
|
+
<goals>
|
|
35
|
+
<goal>copy-dependencies</goal>
|
|
36
|
+
</goals>
|
|
37
|
+
<configuration>
|
|
38
|
+
<outputDirectory>${project.build.directory}</outputDirectory>
|
|
39
|
+
<!-- compile scope gives runtime and compile dependencies (skips test deps) -->
|
|
40
|
+
<includeScope>runtime</includeScope>
|
|
41
|
+
</configuration>
|
|
42
|
+
</execution>
|
|
43
|
+
</executions>
|
|
44
|
+
</plugin>
|
|
45
|
+
</plugins>
|
|
46
|
+
</build>
|
|
47
|
+
</profile>
|
|
48
|
+
</profiles>
|
|
49
|
+
|
|
50
|
+
<repositories>
|
|
51
|
+
<repository>
|
|
52
|
+
<id>my-repo1</id>
|
|
53
|
+
<name>my custom repo</name>
|
|
54
|
+
<url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
|
|
55
|
+
</repository>
|
|
56
|
+
</repositories>
|
|
57
|
+
</project>
|
data/ci/redshift/pom.xml
CHANGED
|
@@ -67,7 +67,7 @@
|
|
|
67
67
|
<repository>
|
|
68
68
|
<id>my-repo1</id>
|
|
69
69
|
<name>my custom repo</name>
|
|
70
|
-
<url>https://
|
|
70
|
+
<url>https://sonatype-nexus.intgdc.com/repository/public/</url>
|
|
71
71
|
</repository>
|
|
72
72
|
</repositories>
|
|
73
73
|
</project>
|
data/docker-compose.lcm.yml
CHANGED
data/gooddata.gemspec
CHANGED
|
@@ -66,6 +66,8 @@ Gem::Specification.new do |s|
|
|
|
66
66
|
else
|
|
67
67
|
s.add_dependency 'docile', '> 1.1', '< 1.4.0'
|
|
68
68
|
end
|
|
69
|
+
s.add_dependency 'azure-storage-blob', '~> 1.1.0'
|
|
70
|
+
s.add_dependency 'nokogiri', '~> 1.10.0'
|
|
69
71
|
s.add_dependency 'gli', '~> 2.15'
|
|
70
72
|
s.add_dependency 'gooddata_datawarehouse', '~> 0.0.10' if RUBY_PLATFORM == 'java'
|
|
71
73
|
s.add_dependency 'highline', '= 2.0.0.pre.develop.14'
|
|
@@ -77,7 +79,6 @@ Gem::Specification.new do |s|
|
|
|
77
79
|
s.add_dependency 'restforce', '>= 2.4', '< 4.0'
|
|
78
80
|
s.add_dependency 'rest-client', '~> 2.0'
|
|
79
81
|
s.add_dependency 'rubyzip', '~> 1.2', '>= 1.2.1'
|
|
80
|
-
s.add_dependency 'salesforce_bulk_query', '~> 0.2'
|
|
81
82
|
s.add_dependency 'terminal-table', '~> 1.7'
|
|
82
83
|
s.add_dependency 'thread_safe'
|
|
83
84
|
s.add_dependency 'backports'
|
data/lcm.rake
CHANGED
|
@@ -101,14 +101,8 @@ namespace :test do
|
|
|
101
101
|
test_cases.each do |t|
|
|
102
102
|
desc "Run #{t} tests in Docker"
|
|
103
103
|
task t do
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
fail('Test execution failed!')
|
|
107
|
-
else
|
|
108
|
-
system("docker-compose -f docker-compose.lcm.yml run --rm appstore bundle exec rake -f lcm.rake test:#{t}") ||
|
|
109
|
-
fail('Test execution failed!')
|
|
110
|
-
end
|
|
111
|
-
|
|
104
|
+
system("docker-compose -f docker-compose.lcm.yml run --rm appstore bundle exec rake -f lcm.rake test:#{t}") ||
|
|
105
|
+
fail('Test execution failed!')
|
|
112
106
|
end
|
|
113
107
|
end
|
|
114
108
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# encoding: UTF-8
|
|
2
2
|
#
|
|
3
|
-
# Copyright (c) 2010-
|
|
3
|
+
# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
|
|
4
4
|
# This source code is licensed under the BSD-style license found in the
|
|
5
5
|
# LICENSE file in the root directory of this source tree.
|
|
6
6
|
|
|
@@ -12,21 +12,47 @@ module GoodData
|
|
|
12
12
|
class AWSMiddleware < Bricks::Middleware
|
|
13
13
|
def call(params)
|
|
14
14
|
params = params.to_hash
|
|
15
|
-
|
|
15
|
+
s3_config = get_s3_config(params)
|
|
16
|
+
|
|
17
|
+
unless s3_config.empty?
|
|
16
18
|
GoodData.logger.info('Setting up AWS-S3 connection')
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
19
|
+
if params.key?('aws_client')
|
|
20
|
+
params['s3_client'] = {}
|
|
21
|
+
elsif params.key?('s3_client')
|
|
22
|
+
params['input_source'] = {} unless params.key?('input_source')
|
|
23
|
+
params['input_source']['bucket'] = params['s3_client']['bucket']
|
|
24
|
+
end
|
|
25
|
+
s3_config = rewrite_for_aws_sdk_v2(s3_config)
|
|
26
|
+
symbolized_config = GoodData::Helpers.symbolize_keys(s3_config)
|
|
27
|
+
params['s3_client']['client'] = Aws::S3::Resource.new(symbolized_config)
|
|
24
28
|
end
|
|
25
29
|
@app.call(params)
|
|
26
30
|
end
|
|
27
31
|
|
|
28
32
|
private
|
|
29
33
|
|
|
34
|
+
def get_s3_config(params)
|
|
35
|
+
s3_config = {}
|
|
36
|
+
if params.key?('aws_client')
|
|
37
|
+
GoodData.logger.warn('Found two configuration aws_client and s3_client for S3 input source, use aws_client configuration') if params.key?('s3_client')
|
|
38
|
+
raise 'Unable to connect to AWS. Parameter "aws_client" seems to be empty' unless params['aws_client']
|
|
39
|
+
raise 'Unable to connect to AWS. Parameter "access_key_id" is missing' if params['aws_client']['access_key_id'].blank?
|
|
40
|
+
raise 'Unable to connect to AWS. Parameter "secret_access_key" is missing' if params['aws_client']['secret_access_key'].blank?
|
|
41
|
+
|
|
42
|
+
s3_config = params['aws_client']
|
|
43
|
+
elsif params.key?('s3_client')
|
|
44
|
+
raise 'Unable to connect to AWS. Parameter "s3_client" seems to be empty' unless params['s3_client']
|
|
45
|
+
raise 'Unable to connect to AWS. Parameter "accessKey" is missing' if params['s3_client']['accessKey'].blank?
|
|
46
|
+
raise 'Unable to connect to AWS. Parameter "secretKey" is missing' if params['s3_client']['secretKey'].blank?
|
|
47
|
+
raise 'Unable to connect to AWS. Parameter "bucket" is missing' if params['s3_client']['bucket'].blank?
|
|
48
|
+
|
|
49
|
+
s3_config['access_key_id'] = params['s3_client']['accessKey']
|
|
50
|
+
s3_config['secret_access_key'] = params['s3_client']['secretKey']
|
|
51
|
+
s3_config['region'] = params['s3_client']['region']
|
|
52
|
+
end
|
|
53
|
+
s3_config
|
|
54
|
+
end
|
|
55
|
+
|
|
30
56
|
def rewrite_for_aws_sdk_v2(config)
|
|
31
57
|
config['region'] = 'us-west-2' unless config['region']
|
|
32
58
|
if config['use_ssl']
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# encoding: UTF-8
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
#
|
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
|
7
|
+
|
|
8
|
+
require 'securerandom'
|
|
9
|
+
require 'pathname'
|
|
10
|
+
require "azure/storage/blob"
|
|
11
|
+
|
|
12
|
+
module GoodData
|
|
13
|
+
class BlobStorageClient
|
|
14
|
+
SAS_URL_PATTERN = %r{(^https?:\/\/[^\/]*)\/.*\?(.*)}
|
|
15
|
+
INVALID_BLOB_GENERAL_MESSAGE = "The connection string is not valid."
|
|
16
|
+
INVALID_BLOB_SIG_WELL_FORMED_MESSAGE = "The signature format is not valid."
|
|
17
|
+
INVALID_BLOB_CONTAINER_MESSAGE = "ContainerNotFound"
|
|
18
|
+
INVALID_BLOB_CONTAINER_FORMED_MESSAGE = "The container with the specified name is not found."
|
|
19
|
+
INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE = "Signature not valid in the specified time frame"
|
|
20
|
+
INVALID_BLOB_EXPIRED_MESSAGE = "The signature expired."
|
|
21
|
+
INVALID_BLOB_INVALID_CONNECTION_STRING_MESSAGE = "The connection string is not valid."
|
|
22
|
+
INVALID_BLOB_PATH_MESSAGE = "BlobNotFound"
|
|
23
|
+
INVALID_BLOB_INVALID_PATH_MESSAGE = "The path to the data is not found."
|
|
24
|
+
|
|
25
|
+
attr_reader :use_sas
|
|
26
|
+
|
|
27
|
+
def initialize(options = {})
|
|
28
|
+
raise("Data Source needs a client to Blob Storage to be able to get blob file but 'blobStorage_client' is empty.") unless options['blobStorage_client']
|
|
29
|
+
|
|
30
|
+
if options['blobStorage_client']['connectionString'] && options['blobStorage_client']['container']
|
|
31
|
+
@connection_string = options['blobStorage_client']['connectionString']
|
|
32
|
+
@container = options['blobStorage_client']['container']
|
|
33
|
+
@path = options['blobStorage_client']['path']
|
|
34
|
+
@use_sas = false
|
|
35
|
+
build_sas(@connection_string)
|
|
36
|
+
else
|
|
37
|
+
raise('Missing connection info for Blob Storage client')
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def realize_blob(file, _params)
|
|
42
|
+
GoodData.gd_logger.info("Realizing download from Blob Storage. Container #{@container}.")
|
|
43
|
+
filename = ''
|
|
44
|
+
begin
|
|
45
|
+
connect
|
|
46
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
|
47
|
+
blob_name = get_blob_name(@path, file)
|
|
48
|
+
|
|
49
|
+
measure = Benchmark.measure do
|
|
50
|
+
_blob, content = @client.get_blob(@container, blob_name)
|
|
51
|
+
File.open(filename, "wb") { |f| f.write(content) }
|
|
52
|
+
end
|
|
53
|
+
rescue => e
|
|
54
|
+
raise_error(e)
|
|
55
|
+
end
|
|
56
|
+
GoodData.gd_logger.info("Done downloading file type=blobStorage status=finished duration=#{measure.real}")
|
|
57
|
+
filename
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def connect
|
|
61
|
+
GoodData.logger.info "Setting up connection to Blob Storage"
|
|
62
|
+
if use_sas
|
|
63
|
+
@client = Azure::Storage::Blob::BlobService.create(:storage_blob_host => @host, :storage_sas_token => @sas_token)
|
|
64
|
+
else
|
|
65
|
+
@client = Azure::Storage::Blob::BlobService.create_from_connection_string(@connection_string)
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def build_sas(url)
|
|
70
|
+
matches = url.scan(SAS_URL_PATTERN)
|
|
71
|
+
return unless matches && matches[0]
|
|
72
|
+
|
|
73
|
+
@use_sas = true
|
|
74
|
+
@host = matches[0][0]
|
|
75
|
+
@sas_token = matches[0][1]
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def raise_error(e)
|
|
79
|
+
if e.message && e.message.include?(INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE)
|
|
80
|
+
raise INVALID_BLOB_EXPIRED_MESSAGE
|
|
81
|
+
elsif e.message && e.message.include?(INVALID_BLOB_SIG_WELL_FORMED_MESSAGE)
|
|
82
|
+
raise INVALID_BLOB_SIG_WELL_FORMED_MESSAGE
|
|
83
|
+
elsif e.message && e.message.include?(INVALID_BLOB_CONTAINER_MESSAGE)
|
|
84
|
+
raise INVALID_BLOB_CONTAINER_FORMED_MESSAGE
|
|
85
|
+
elsif e.message && e.message.include?(INVALID_BLOB_PATH_MESSAGE)
|
|
86
|
+
raise INVALID_BLOB_INVALID_PATH_MESSAGE
|
|
87
|
+
else
|
|
88
|
+
raise INVALID_BLOB_GENERAL_MESSAGE
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def get_blob_name(path, file)
|
|
93
|
+
return file unless path
|
|
94
|
+
|
|
95
|
+
path.rindex('/') == path.length - 1 ? "#{path}#{file}" : "#{path}/#{file}"
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
File without changes
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
# encoding: UTF-8
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
#
|
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
|
7
|
+
|
|
8
|
+
require 'securerandom'
|
|
9
|
+
require 'java'
|
|
10
|
+
require 'pathname'
|
|
11
|
+
require_relative '../cloud_resource_client'
|
|
12
|
+
|
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
|
15
|
+
require file unless file.start_with?('lcm-mssql-driver')
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
module GoodData
|
|
19
|
+
module CloudResources
|
|
20
|
+
class MSSQLClient < CloudResourceClient
|
|
21
|
+
MSSQL_SEPARATOR_PARAM = ";"
|
|
22
|
+
MSSQL_URL_PATTERN = %r{jdbc:sqlserver://([^:]+)(:([0-9]+))?(/)?}
|
|
23
|
+
MSSQL_DEFAULT_PORT = 1433
|
|
24
|
+
LOGIN_TIME_OUT = 30
|
|
25
|
+
MSSQL_FETCH_SIZE = 10_000
|
|
26
|
+
VERIFY_FULL = 'verify-full'
|
|
27
|
+
PREFER = 'prefer'
|
|
28
|
+
REQUIRE = 'require'
|
|
29
|
+
|
|
30
|
+
class << self
|
|
31
|
+
def accept?(type)
|
|
32
|
+
type == 'mssql'
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def initialize(options = {})
|
|
37
|
+
raise("Data Source needs a client to MSSQL to be able to query the storage but 'mssql_client' is empty.") unless options['mssql_client']
|
|
38
|
+
|
|
39
|
+
connection = options['mssql_client']['connection']
|
|
40
|
+
if connection.is_a?(Hash)
|
|
41
|
+
@database = connection['database']
|
|
42
|
+
@schema = connection['schema']
|
|
43
|
+
@authentication = connection['authentication']
|
|
44
|
+
@ssl_mode = connection['sslMode']
|
|
45
|
+
@url = connection['url']
|
|
46
|
+
|
|
47
|
+
validate
|
|
48
|
+
else
|
|
49
|
+
raise('Missing connection info for MSSQL client')
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
Java.com.microsoft.sqlserver.jdbc.SQLServerDriver
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def realize_query(query, _params)
|
|
56
|
+
GoodData.gd_logger.info("Realize SQL query: type=mssql status=started")
|
|
57
|
+
|
|
58
|
+
connect
|
|
59
|
+
|
|
60
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
|
61
|
+
measure = Benchmark.measure do
|
|
62
|
+
statement = @connection.create_statement
|
|
63
|
+
statement.set_fetch_size(MSSQL_FETCH_SIZE)
|
|
64
|
+
has_result = statement.execute(query)
|
|
65
|
+
if has_result
|
|
66
|
+
result = statement.get_result_set
|
|
67
|
+
metadata = result.get_meta_data
|
|
68
|
+
col_count = metadata.column_count
|
|
69
|
+
CSV.open(filename, 'wb') do |csv|
|
|
70
|
+
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
|
71
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
GoodData.gd_logger.info("Realize SQL query: type=mssql status=finished duration=#{measure.real}")
|
|
77
|
+
filename
|
|
78
|
+
ensure
|
|
79
|
+
@connection&.close
|
|
80
|
+
@connection = nil
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def connect
|
|
84
|
+
connection_string = build_connection_string
|
|
85
|
+
GoodData.logger.info "Setting up connection to MSSQL #{connection_string}"
|
|
86
|
+
|
|
87
|
+
authentication = @authentication['basic'] || @authentication['activeDirectoryPassword']
|
|
88
|
+
|
|
89
|
+
prop = java.util.Properties.new
|
|
90
|
+
prop.setProperty('userName', authentication['userName'])
|
|
91
|
+
prop.setProperty('password', authentication['password'])
|
|
92
|
+
|
|
93
|
+
@connection = java.sql.DriverManager.getConnection(connection_string, prop)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def validate
|
|
97
|
+
raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
|
|
98
|
+
|
|
99
|
+
raise "Instance name is not supported" if @url !~ /^[^\\]*$/
|
|
100
|
+
|
|
101
|
+
raise "The connection url is invalid. Parameter is not supported." if @url.include? MSSQL_SEPARATOR_PARAM
|
|
102
|
+
|
|
103
|
+
url_matches = @url.scan(MSSQL_URL_PATTERN)
|
|
104
|
+
raise "Cannot reach the url" if url_matches.nil? || url_matches.length.zero?
|
|
105
|
+
|
|
106
|
+
raise "The authentication method is not supported." unless @authentication['basic'] || @authentication['activeDirectoryPassword']
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def build_connection_string
|
|
110
|
+
encrypt = @ssl_mode != PREFER
|
|
111
|
+
trust_server_certificate = @ssl_mode == REQUIRE
|
|
112
|
+
|
|
113
|
+
"#{@url};" \
|
|
114
|
+
"database=#{@database};" \
|
|
115
|
+
"encrypt=#{encrypt};" \
|
|
116
|
+
"trustServerCertificate=#{trust_server_certificate};" \
|
|
117
|
+
"loginTimeout=#{LOGIN_TIME_OUT};" \
|
|
118
|
+
"#{'authentication=ActiveDirectoryPassword;' if @authentication['activeDirectoryPassword']}"
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
File without changes
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# encoding: UTF-8
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
#
|
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
|
7
|
+
|
|
8
|
+
require 'securerandom'
|
|
9
|
+
require 'java'
|
|
10
|
+
require 'pathname'
|
|
11
|
+
require_relative '../cloud_resource_client'
|
|
12
|
+
|
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
|
15
|
+
require file unless file.start_with?('lcm-mysql-driver')
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
module GoodData
|
|
19
|
+
module CloudResources
|
|
20
|
+
class MysqlClient < CloudResourceClient
|
|
21
|
+
JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
|
|
22
|
+
MYSQL_DEFAULT_PORT = 3306
|
|
23
|
+
JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://'
|
|
24
|
+
VERIFY_FULL = 'VERIFY_IDENTITY'
|
|
25
|
+
PREFER = 'PREFERRED'
|
|
26
|
+
REQUIRE = 'REQUIRED'
|
|
27
|
+
MYSQL_FETCH_SIZE = 1000
|
|
28
|
+
|
|
29
|
+
class << self
|
|
30
|
+
def accept?(type)
|
|
31
|
+
type == 'mysql'
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def initialize(options = {})
|
|
36
|
+
raise("Data Source needs a client to Mysql to be able to query the storage but 'mysql_client' is empty.") unless options['mysql_client']
|
|
37
|
+
|
|
38
|
+
if options['mysql_client']['connection'].is_a?(Hash)
|
|
39
|
+
@database = options['mysql_client']['connection']['database']
|
|
40
|
+
@authentication = options['mysql_client']['connection']['authentication']
|
|
41
|
+
@ssl_mode = options['mysql_client']['connection']['sslMode']
|
|
42
|
+
raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
|
|
43
|
+
|
|
44
|
+
@url = build_url(options['mysql_client']['connection']['url'])
|
|
45
|
+
else
|
|
46
|
+
raise('Missing connection info for Mysql client')
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
Java.com.mysql.cj.jdbc.Driver
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def realize_query(query, _params)
|
|
53
|
+
GoodData.gd_logger.info("Realize SQL query: type=mysql status=started")
|
|
54
|
+
|
|
55
|
+
connect
|
|
56
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
|
57
|
+
measure = Benchmark.measure do
|
|
58
|
+
statement = @connection.create_statement
|
|
59
|
+
statement.set_fetch_size(MYSQL_FETCH_SIZE)
|
|
60
|
+
has_result = statement.execute(query)
|
|
61
|
+
if has_result
|
|
62
|
+
result = statement.get_result_set
|
|
63
|
+
metadata = result.get_meta_data
|
|
64
|
+
col_count = metadata.column_count
|
|
65
|
+
CSV.open(filename, 'wb') do |csv|
|
|
66
|
+
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
|
67
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
GoodData.gd_logger.info("Realize SQL query: type=mysql status=finished duration=#{measure.real}")
|
|
72
|
+
filename
|
|
73
|
+
ensure
|
|
74
|
+
@connection&.close
|
|
75
|
+
@connection = nil
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def connect
|
|
79
|
+
GoodData.logger.info "Setting up connection to Mysql #{@url}"
|
|
80
|
+
|
|
81
|
+
prop = java.util.Properties.new
|
|
82
|
+
prop.setProperty('user', @authentication['basic']['userName'])
|
|
83
|
+
prop.setProperty('password', @authentication['basic']['password'])
|
|
84
|
+
|
|
85
|
+
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
|
86
|
+
@connection.set_auto_commit(false)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def build_url(url)
|
|
90
|
+
matches = url.scan(JDBC_MYSQL_PATTERN)
|
|
91
|
+
raise 'Cannot reach the url' unless matches
|
|
92
|
+
|
|
93
|
+
host = matches[0][0]
|
|
94
|
+
port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT
|
|
95
|
+
|
|
96
|
+
"#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{get_ssl_mode(@ssl_mode)}&useCursorFetch=true&enabledTLSProtocols=TLSv1.2"
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def get_ssl_mode(ssl_mode)
|
|
100
|
+
mode = PREFER
|
|
101
|
+
if ssl_mode == 'verify-full'
|
|
102
|
+
mode = VERIFY_FULL
|
|
103
|
+
elsif ssl_mode == 'require'
|
|
104
|
+
mode = REQUIRE
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
mode
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
@@ -98,7 +98,6 @@ module GoodData
|
|
|
98
98
|
|
|
99
99
|
host = matches[0][0]
|
|
100
100
|
port = matches[0][2]&.to_i || POSTGRES_DEFAULT_PORT
|
|
101
|
-
raise "Custom port #{port} is not supported. Remove it or use the default port '5432'" if POSTGRES_DEFAULT_PORT != port
|
|
102
101
|
|
|
103
102
|
"#{JDBC_POSTGRES_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{@ssl_mode}#{VERIFY_FULL == @ssl_mode ? SSL_JAVA_FACTORY : ''}"
|
|
104
103
|
end
|
|
@@ -18,6 +18,9 @@ end
|
|
|
18
18
|
module GoodData
|
|
19
19
|
module CloudResources
|
|
20
20
|
class SnowflakeClient < CloudResourceClient
|
|
21
|
+
SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=GoodData_Platform'
|
|
22
|
+
SNOWFLAKE_SEPARATOR_PARAM = '?'
|
|
23
|
+
|
|
21
24
|
class << self
|
|
22
25
|
def accept?(type)
|
|
23
26
|
type == 'snowflake'
|
|
@@ -31,7 +34,7 @@ module GoodData
|
|
|
31
34
|
@database = options['snowflake_client']['connection']['database']
|
|
32
35
|
@schema = options['snowflake_client']['connection']['schema'] || 'public'
|
|
33
36
|
@warehouse = options['snowflake_client']['connection']['warehouse']
|
|
34
|
-
@url = options['snowflake_client']['connection']['url']
|
|
37
|
+
@url = build_url(options['snowflake_client']['connection']['url'])
|
|
35
38
|
@authentication = options['snowflake_client']['connection']['authentication']
|
|
36
39
|
else
|
|
37
40
|
raise('Missing connection info for Snowflake client')
|
|
@@ -79,6 +82,20 @@ module GoodData
|
|
|
79
82
|
|
|
80
83
|
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
|
81
84
|
end
|
|
85
|
+
|
|
86
|
+
def build_url(url)
|
|
87
|
+
is_contain = url.include?(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
|
|
88
|
+
unless is_contain
|
|
89
|
+
if url.include?(SNOWFLAKE_SEPARATOR_PARAM)
|
|
90
|
+
url.concat("&")
|
|
91
|
+
else
|
|
92
|
+
url.concat(SNOWFLAKE_SEPARATOR_PARAM)
|
|
93
|
+
end
|
|
94
|
+
url.concat(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
url
|
|
98
|
+
end
|
|
82
99
|
end
|
|
83
100
|
end
|
|
84
101
|
end
|