gooddata 2.1.14-java → 2.2.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gdc-ii-config.yaml +1 -1
- data/.github/workflows/build.yml +66 -0
- data/.github/workflows/pre-merge.yml +72 -0
- data/.sonar.settings +4 -0
- data/.travis.yml +78 -12
- data/CHANGELOG.md +62 -0
- data/Dockerfile +25 -14
- data/LICENSE +4418 -17
- data/LICENSE.rb +1 -1
- data/README.md +3 -3
- data/Rakefile +8 -1
- data/SDK_VERSION +1 -1
- data/VERSION +1 -1
- data/bin/test_projects_cleanup.rb +45 -3
- data/ci/mssql/pom.xml +62 -0
- data/ci/mysql/pom.xml +57 -0
- data/ci/postgresql/pom.xml +57 -0
- data/ci/redshift/pom.xml +1 -1
- data/dev-gooddata-sso.pub.encrypted +40 -40
- data/gdc_fossa_ruby_sdk.yaml +1 -0
- data/gooddata.gemspec +10 -6
- data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
- data/k8s/charts/lcm-bricks/templates/prometheus/alertingRules.yaml +11 -1
- data/lcm.rake +2 -2
- data/lib/gooddata/bricks/middleware/aws_middleware.rb +35 -9
- data/lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb +98 -0
- data/lib/gooddata/cloud_resources/{cloud_resouce_factory.rb → cloud_resource_factory.rb} +8 -0
- data/lib/gooddata/cloud_resources/cloud_resources.rb +1 -1
- data/lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mssql/mssql_client.rb +122 -0
- data/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +111 -0
- data/lib/gooddata/cloud_resources/postgresql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +106 -0
- data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +18 -1
- data/lib/gooddata/commands/scaffold.rb +9 -10
- data/lib/gooddata/core/nil_logger.rb +3 -1
- data/lib/gooddata/helpers/data_helper.rb +9 -5
- data/lib/gooddata/helpers/global_helpers.rb +6 -5
- data/lib/gooddata/lcm/actions/associate_clients.rb +8 -2
- data/lib/gooddata/lcm/actions/base_action.rb +0 -2
- data/lib/gooddata/lcm/actions/collect_meta.rb +3 -1
- data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +3 -2
- data/lib/gooddata/lcm/actions/provision_clients.rb +31 -10
- data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -7
- data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +64 -0
- data/lib/gooddata/lcm/actions/synchronize_ldm.rb +19 -8
- data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +16 -9
- data/lib/gooddata/lcm/actions/synchronize_users.rb +7 -6
- data/lib/gooddata/lcm/actions/update_metric_formats.rb +185 -0
- data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb +5 -0
- data/lib/gooddata/lcm/helpers/release_table_helper.rb +42 -8
- data/lib/gooddata/lcm/lcm2.rb +5 -2
- data/lib/gooddata/lcm/types/base_type.rb +0 -2
- data/lib/gooddata/mixins/md_object_query.rb +9 -6
- data/lib/gooddata/models/blueprint/project_blueprint.rb +0 -2
- data/lib/gooddata/models/client.rb +14 -12
- data/lib/gooddata/models/data_source.rb +668 -0
- data/lib/gooddata/models/dataset_mapping.rb +36 -0
- data/lib/gooddata/models/domain.rb +3 -2
- data/lib/gooddata/models/metadata/analytical_dashboard.rb +49 -0
- data/lib/gooddata/models/metadata/analytical_visualization_object.rb +30 -0
- data/lib/gooddata/models/metadata/label.rb +26 -27
- data/lib/gooddata/models/metadata/visualization_object.rb +50 -0
- data/lib/gooddata/models/project.rb +66 -19
- data/lib/gooddata/models/schedule.rb +13 -1
- data/lib/gooddata/models/user_filters/user_filter_builder.rb +58 -54
- data/lib/gooddata/models/user_group.rb +0 -1
- data/lib/gooddata/rest/connection.rb +6 -4
- data/lib/gooddata/rest/phmap.rb +2 -1
- data/lib/gooddata.rb +2 -0
- data/rubydev_public.gpg.encrypted +51 -51
- data/rubydev_secret_keys.gpg.encrypted +109 -109
- metadata +51 -27
- data/DEPENDENCIES.md +0 -880
- data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb +0 -37
- data/lib/gooddata/helpers/data_source_helpers.rb +0 -47
@@ -0,0 +1,98 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'pathname'
|
10
|
+
require "azure/storage/blob"
|
11
|
+
|
12
|
+
module GoodData
|
13
|
+
class BlobStorageClient
|
14
|
+
SAS_URL_PATTERN = %r{(^https?:\/\/[^\/]*)\/.*\?(.*)}
|
15
|
+
INVALID_BLOB_GENERAL_MESSAGE = "The connection string is not valid."
|
16
|
+
INVALID_BLOB_SIG_WELL_FORMED_MESSAGE = "The signature format is not valid."
|
17
|
+
INVALID_BLOB_CONTAINER_MESSAGE = "ContainerNotFound"
|
18
|
+
INVALID_BLOB_CONTAINER_FORMED_MESSAGE = "The container with the specified name is not found."
|
19
|
+
INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE = "Signature not valid in the specified time frame"
|
20
|
+
INVALID_BLOB_EXPIRED_MESSAGE = "The signature expired."
|
21
|
+
INVALID_BLOB_INVALID_CONNECTION_STRING_MESSAGE = "The connection string is not valid."
|
22
|
+
INVALID_BLOB_PATH_MESSAGE = "BlobNotFound"
|
23
|
+
INVALID_BLOB_INVALID_PATH_MESSAGE = "The path to the data is not found."
|
24
|
+
|
25
|
+
attr_reader :use_sas
|
26
|
+
|
27
|
+
def initialize(options = {})
|
28
|
+
raise("Data Source needs a client to Blob Storage to be able to get blob file but 'blobStorage_client' is empty.") unless options['blobStorage_client']
|
29
|
+
|
30
|
+
if options['blobStorage_client']['connectionString'] && options['blobStorage_client']['container']
|
31
|
+
@connection_string = options['blobStorage_client']['connectionString']
|
32
|
+
@container = options['blobStorage_client']['container']
|
33
|
+
@path = options['blobStorage_client']['path']
|
34
|
+
@use_sas = false
|
35
|
+
build_sas(@connection_string)
|
36
|
+
else
|
37
|
+
raise('Missing connection info for Blob Storage client')
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def realize_blob(file, _params)
|
42
|
+
GoodData.gd_logger.info("Realizing download from Blob Storage. Container #{@container}.")
|
43
|
+
filename = ''
|
44
|
+
begin
|
45
|
+
connect
|
46
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
47
|
+
blob_name = get_blob_name(@path, file)
|
48
|
+
|
49
|
+
measure = Benchmark.measure do
|
50
|
+
_blob, content = @client.get_blob(@container, blob_name)
|
51
|
+
File.open(filename, "wb") { |f| f.write(content) }
|
52
|
+
end
|
53
|
+
rescue => e
|
54
|
+
raise_error(e)
|
55
|
+
end
|
56
|
+
GoodData.gd_logger.info("Done downloading file type=blobStorage status=finished duration=#{measure.real}")
|
57
|
+
filename
|
58
|
+
end
|
59
|
+
|
60
|
+
def connect
|
61
|
+
GoodData.logger.info "Setting up connection to Blob Storage"
|
62
|
+
if use_sas
|
63
|
+
@client = Azure::Storage::Blob::BlobService.create(:storage_blob_host => @host, :storage_sas_token => @sas_token)
|
64
|
+
else
|
65
|
+
@client = Azure::Storage::Blob::BlobService.create_from_connection_string(@connection_string)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def build_sas(url)
|
70
|
+
matches = url.scan(SAS_URL_PATTERN)
|
71
|
+
return unless matches && matches[0]
|
72
|
+
|
73
|
+
@use_sas = true
|
74
|
+
@host = matches[0][0]
|
75
|
+
@sas_token = matches[0][1]
|
76
|
+
end
|
77
|
+
|
78
|
+
def raise_error(e)
|
79
|
+
if e.message && e.message.include?(INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE)
|
80
|
+
raise INVALID_BLOB_EXPIRED_MESSAGE
|
81
|
+
elsif e.message && e.message.include?(INVALID_BLOB_SIG_WELL_FORMED_MESSAGE)
|
82
|
+
raise INVALID_BLOB_SIG_WELL_FORMED_MESSAGE
|
83
|
+
elsif e.message && e.message.include?(INVALID_BLOB_CONTAINER_MESSAGE)
|
84
|
+
raise INVALID_BLOB_CONTAINER_FORMED_MESSAGE
|
85
|
+
elsif e.message && e.message.include?(INVALID_BLOB_PATH_MESSAGE)
|
86
|
+
raise INVALID_BLOB_INVALID_PATH_MESSAGE
|
87
|
+
else
|
88
|
+
raise INVALID_BLOB_GENERAL_MESSAGE
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def get_blob_name(path, file)
|
93
|
+
return file unless path
|
94
|
+
|
95
|
+
path.rindex('/') == path.length - 1 ? "#{path}#{file}" : "#{path}/#{file}"
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
@@ -11,7 +11,15 @@ module GoodData
|
|
11
11
|
module CloudResources
|
12
12
|
class CloudResourceFactory
|
13
13
|
class << self
|
14
|
+
def load_cloud_resource(type)
|
15
|
+
base = "#{Pathname(__FILE__).dirname.expand_path}#{File::SEPARATOR}#{type}#{File::SEPARATOR}"
|
16
|
+
Dir.glob(base + '**/*.rb').each do |file|
|
17
|
+
require file
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
14
21
|
def create(type, data = {}, opts = {})
|
22
|
+
load_cloud_resource(type)
|
15
23
|
clients = CloudResourceClient.descendants.select { |c| c.respond_to?("accept?") && c.send("accept?", type) }
|
16
24
|
raise "DataSource does not support type \"#{type}\"" if clients.empty?
|
17
25
|
|
File without changes
|
@@ -0,0 +1,122 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'java'
|
10
|
+
require 'pathname'
|
11
|
+
require_relative '../cloud_resource_client'
|
12
|
+
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
15
|
+
require file unless file.start_with?('lcm-mssql-driver')
|
16
|
+
end
|
17
|
+
|
18
|
+
module GoodData
|
19
|
+
module CloudResources
|
20
|
+
class MSSQLClient < CloudResourceClient
|
21
|
+
MSSQL_SEPARATOR_PARAM = ";"
|
22
|
+
MSSQL_URL_PATTERN = %r{jdbc:sqlserver://([^:]+)(:([0-9]+))?(/)?}
|
23
|
+
MSSQL_DEFAULT_PORT = 1433
|
24
|
+
LOGIN_TIME_OUT = 30
|
25
|
+
MSSQL_FETCH_SIZE = 10_000
|
26
|
+
VERIFY_FULL = 'verify-full'
|
27
|
+
PREFER = 'prefer'
|
28
|
+
REQUIRE = 'require'
|
29
|
+
|
30
|
+
class << self
|
31
|
+
def accept?(type)
|
32
|
+
type == 'mssql'
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def initialize(options = {})
|
37
|
+
raise("Data Source needs a client to MSSQL to be able to query the storage but 'mssql_client' is empty.") unless options['mssql_client']
|
38
|
+
|
39
|
+
connection = options['mssql_client']['connection']
|
40
|
+
if connection.is_a?(Hash)
|
41
|
+
@database = connection['database']
|
42
|
+
@schema = connection['schema']
|
43
|
+
@authentication = connection['authentication']
|
44
|
+
@ssl_mode = connection['sslMode']
|
45
|
+
@url = connection['url']
|
46
|
+
|
47
|
+
validate
|
48
|
+
else
|
49
|
+
raise('Missing connection info for MSSQL client')
|
50
|
+
end
|
51
|
+
|
52
|
+
Java.com.microsoft.sqlserver.jdbc.SQLServerDriver
|
53
|
+
end
|
54
|
+
|
55
|
+
def realize_query(query, _params)
|
56
|
+
GoodData.gd_logger.info("Realize SQL query: type=mssql status=started")
|
57
|
+
|
58
|
+
connect
|
59
|
+
|
60
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
61
|
+
measure = Benchmark.measure do
|
62
|
+
statement = @connection.create_statement
|
63
|
+
statement.set_fetch_size(MSSQL_FETCH_SIZE)
|
64
|
+
has_result = statement.execute(query)
|
65
|
+
if has_result
|
66
|
+
result = statement.get_result_set
|
67
|
+
metadata = result.get_meta_data
|
68
|
+
col_count = metadata.column_count
|
69
|
+
CSV.open(filename, 'wb') do |csv|
|
70
|
+
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
71
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
GoodData.gd_logger.info("Realize SQL query: type=mssql status=finished duration=#{measure.real}")
|
77
|
+
filename
|
78
|
+
ensure
|
79
|
+
@connection&.close
|
80
|
+
@connection = nil
|
81
|
+
end
|
82
|
+
|
83
|
+
def connect
|
84
|
+
connection_string = build_connection_string
|
85
|
+
GoodData.logger.info "Setting up connection to MSSQL #{connection_string}"
|
86
|
+
|
87
|
+
authentication = @authentication['basic'] || @authentication['activeDirectoryPassword']
|
88
|
+
|
89
|
+
prop = java.util.Properties.new
|
90
|
+
prop.setProperty('userName', authentication['userName'])
|
91
|
+
prop.setProperty('password', authentication['password'])
|
92
|
+
|
93
|
+
@connection = java.sql.DriverManager.getConnection(connection_string, prop)
|
94
|
+
end
|
95
|
+
|
96
|
+
def validate
|
97
|
+
raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
|
98
|
+
|
99
|
+
raise "Instance name is not supported" if @url !~ /^[^\\]*$/
|
100
|
+
|
101
|
+
raise "The connection url is invalid. Parameter is not supported." if @url.include? MSSQL_SEPARATOR_PARAM
|
102
|
+
|
103
|
+
url_matches = @url.scan(MSSQL_URL_PATTERN)
|
104
|
+
raise "Cannot reach the url" if url_matches.nil? || url_matches.length.zero?
|
105
|
+
|
106
|
+
raise "The authentication method is not supported." unless @authentication['basic'] || @authentication['activeDirectoryPassword']
|
107
|
+
end
|
108
|
+
|
109
|
+
def build_connection_string
|
110
|
+
encrypt = @ssl_mode != PREFER
|
111
|
+
trust_server_certificate = @ssl_mode == REQUIRE
|
112
|
+
|
113
|
+
"#{@url};" \
|
114
|
+
"database=#{@database};" \
|
115
|
+
"encrypt=#{encrypt};" \
|
116
|
+
"trustServerCertificate=#{trust_server_certificate};" \
|
117
|
+
"loginTimeout=#{LOGIN_TIME_OUT};" \
|
118
|
+
"#{'authentication=ActiveDirectoryPassword;' if @authentication['activeDirectoryPassword']}"
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
File without changes
|
@@ -0,0 +1,111 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'java'
|
10
|
+
require 'pathname'
|
11
|
+
require_relative '../cloud_resource_client'
|
12
|
+
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
15
|
+
require file unless file.start_with?('lcm-mysql-driver')
|
16
|
+
end
|
17
|
+
|
18
|
+
module GoodData
|
19
|
+
module CloudResources
|
20
|
+
class MysqlClient < CloudResourceClient
|
21
|
+
JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
|
22
|
+
MYSQL_DEFAULT_PORT = 3306
|
23
|
+
JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://'
|
24
|
+
VERIFY_FULL = 'VERIFY_IDENTITY'
|
25
|
+
PREFER = 'PREFERRED'
|
26
|
+
REQUIRE = 'REQUIRED'
|
27
|
+
MYSQL_FETCH_SIZE = 1000
|
28
|
+
|
29
|
+
class << self
|
30
|
+
def accept?(type)
|
31
|
+
type == 'mysql'
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def initialize(options = {})
|
36
|
+
raise("Data Source needs a client to Mysql to be able to query the storage but 'mysql_client' is empty.") unless options['mysql_client']
|
37
|
+
|
38
|
+
if options['mysql_client']['connection'].is_a?(Hash)
|
39
|
+
@database = options['mysql_client']['connection']['database']
|
40
|
+
@authentication = options['mysql_client']['connection']['authentication']
|
41
|
+
@ssl_mode = options['mysql_client']['connection']['sslMode']
|
42
|
+
raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
|
43
|
+
|
44
|
+
@url = build_url(options['mysql_client']['connection']['url'])
|
45
|
+
else
|
46
|
+
raise('Missing connection info for Mysql client')
|
47
|
+
end
|
48
|
+
|
49
|
+
Java.com.mysql.cj.jdbc.Driver
|
50
|
+
end
|
51
|
+
|
52
|
+
def realize_query(query, _params)
|
53
|
+
GoodData.gd_logger.info("Realize SQL query: type=mysql status=started")
|
54
|
+
|
55
|
+
connect
|
56
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
57
|
+
measure = Benchmark.measure do
|
58
|
+
statement = @connection.create_statement
|
59
|
+
statement.set_fetch_size(MYSQL_FETCH_SIZE)
|
60
|
+
has_result = statement.execute(query)
|
61
|
+
if has_result
|
62
|
+
result = statement.get_result_set
|
63
|
+
metadata = result.get_meta_data
|
64
|
+
col_count = metadata.column_count
|
65
|
+
CSV.open(filename, 'wb') do |csv|
|
66
|
+
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
67
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
GoodData.gd_logger.info("Realize SQL query: type=mysql status=finished duration=#{measure.real}")
|
72
|
+
filename
|
73
|
+
ensure
|
74
|
+
@connection&.close
|
75
|
+
@connection = nil
|
76
|
+
end
|
77
|
+
|
78
|
+
def connect
|
79
|
+
GoodData.logger.info "Setting up connection to Mysql #{@url}"
|
80
|
+
|
81
|
+
prop = java.util.Properties.new
|
82
|
+
prop.setProperty('user', @authentication['basic']['userName'])
|
83
|
+
prop.setProperty('password', @authentication['basic']['password'])
|
84
|
+
|
85
|
+
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
86
|
+
@connection.set_auto_commit(false)
|
87
|
+
end
|
88
|
+
|
89
|
+
def build_url(url)
|
90
|
+
matches = url.scan(JDBC_MYSQL_PATTERN)
|
91
|
+
raise 'Cannot reach the url' unless matches
|
92
|
+
|
93
|
+
host = matches[0][0]
|
94
|
+
port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT
|
95
|
+
|
96
|
+
"#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{get_ssl_mode(@ssl_mode)}&useCursorFetch=true&enabledTLSProtocols=TLSv1.2"
|
97
|
+
end
|
98
|
+
|
99
|
+
def get_ssl_mode(ssl_mode)
|
100
|
+
mode = PREFER
|
101
|
+
if ssl_mode == 'verify-full'
|
102
|
+
mode = VERIFY_FULL
|
103
|
+
elsif ssl_mode == 'require'
|
104
|
+
mode = REQUIRE
|
105
|
+
end
|
106
|
+
|
107
|
+
mode
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
File without changes
|
@@ -0,0 +1,106 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'java'
|
10
|
+
require 'pathname'
|
11
|
+
require_relative '../cloud_resource_client'
|
12
|
+
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
15
|
+
require file unless file.start_with?('lcm-postgresql-driver')
|
16
|
+
end
|
17
|
+
|
18
|
+
module GoodData
|
19
|
+
module CloudResources
|
20
|
+
class PostgresClient < CloudResourceClient
|
21
|
+
JDBC_POSTGRES_PATTERN = %r{jdbc:postgresql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
|
22
|
+
POSTGRES_DEFAULT_PORT = 5432
|
23
|
+
JDBC_POSTGRES_PROTOCOL = 'jdbc:postgresql://'
|
24
|
+
SSL_JAVA_FACTORY = '&sslfactory=org.postgresql.ssl.DefaultJavaSSLFactory'
|
25
|
+
VERIFY_FULL = 'verify-full'
|
26
|
+
PREFER = 'prefer'
|
27
|
+
REQUIRE = 'require'
|
28
|
+
POSTGRES_SET_SCHEMA_COMMAND = "set search_path to"
|
29
|
+
POSTGRES_FETCH_SIZE = 1000
|
30
|
+
|
31
|
+
class << self
|
32
|
+
def accept?(type)
|
33
|
+
type == 'postgresql'
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def initialize(options = {})
|
38
|
+
raise("Data Source needs a client to Postgres to be able to query the storage but 'postgresql_client' is empty.") unless options['postgresql_client']
|
39
|
+
|
40
|
+
if options['postgresql_client']['connection'].is_a?(Hash)
|
41
|
+
@database = options['postgresql_client']['connection']['database']
|
42
|
+
@schema = options['postgresql_client']['connection']['schema'] || 'public'
|
43
|
+
@authentication = options['postgresql_client']['connection']['authentication']
|
44
|
+
@ssl_mode = options['postgresql_client']['connection']['sslMode']
|
45
|
+
raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
|
46
|
+
|
47
|
+
@url = build_url(options['postgresql_client']['connection']['url'])
|
48
|
+
else
|
49
|
+
raise('Missing connection info for Postgres client')
|
50
|
+
end
|
51
|
+
|
52
|
+
Java.org.postgresql.Driver
|
53
|
+
end
|
54
|
+
|
55
|
+
def realize_query(query, _params)
|
56
|
+
GoodData.gd_logger.info("Realize SQL query: type=postgresql status=started")
|
57
|
+
|
58
|
+
connect
|
59
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
60
|
+
measure = Benchmark.measure do
|
61
|
+
statement = @connection.create_statement
|
62
|
+
statement.set_fetch_size(POSTGRES_FETCH_SIZE)
|
63
|
+
has_result = statement.execute(query)
|
64
|
+
if has_result
|
65
|
+
result = statement.get_result_set
|
66
|
+
metadata = result.get_meta_data
|
67
|
+
col_count = metadata.column_count
|
68
|
+
CSV.open(filename, 'wb') do |csv|
|
69
|
+
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
70
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
GoodData.gd_logger.info("Realize SQL query: type=postgresql status=finished duration=#{measure.real}")
|
75
|
+
filename
|
76
|
+
ensure
|
77
|
+
@connection&.close
|
78
|
+
@connection = nil
|
79
|
+
end
|
80
|
+
|
81
|
+
def connect
|
82
|
+
GoodData.logger.info "Setting up connection to Postgresql #{@url}"
|
83
|
+
|
84
|
+
prop = java.util.Properties.new
|
85
|
+
prop.setProperty('user', @authentication['basic']['userName'])
|
86
|
+
prop.setProperty('password', @authentication['basic']['password'])
|
87
|
+
prop.setProperty('schema', @schema)
|
88
|
+
|
89
|
+
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
90
|
+
statement = @connection.create_statement
|
91
|
+
statement.execute("#{POSTGRES_SET_SCHEMA_COMMAND} #{@schema}")
|
92
|
+
@connection.set_auto_commit(false)
|
93
|
+
end
|
94
|
+
|
95
|
+
def build_url(url)
|
96
|
+
matches = url.scan(JDBC_POSTGRES_PATTERN)
|
97
|
+
raise 'Cannot reach the url' unless matches
|
98
|
+
|
99
|
+
host = matches[0][0]
|
100
|
+
port = matches[0][2]&.to_i || POSTGRES_DEFAULT_PORT
|
101
|
+
|
102
|
+
"#{JDBC_POSTGRES_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{@ssl_mode}#{VERIFY_FULL == @ssl_mode ? SSL_JAVA_FACTORY : ''}"
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
@@ -18,6 +18,9 @@ end
|
|
18
18
|
module GoodData
|
19
19
|
module CloudResources
|
20
20
|
class SnowflakeClient < CloudResourceClient
|
21
|
+
SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=GoodData_Platform'
|
22
|
+
SNOWFLAKE_SEPARATOR_PARAM = '?'
|
23
|
+
|
21
24
|
class << self
|
22
25
|
def accept?(type)
|
23
26
|
type == 'snowflake'
|
@@ -31,7 +34,7 @@ module GoodData
|
|
31
34
|
@database = options['snowflake_client']['connection']['database']
|
32
35
|
@schema = options['snowflake_client']['connection']['schema'] || 'public'
|
33
36
|
@warehouse = options['snowflake_client']['connection']['warehouse']
|
34
|
-
@url = options['snowflake_client']['connection']['url']
|
37
|
+
@url = build_url(options['snowflake_client']['connection']['url'])
|
35
38
|
@authentication = options['snowflake_client']['connection']['authentication']
|
36
39
|
else
|
37
40
|
raise('Missing connection info for Snowflake client')
|
@@ -79,6 +82,20 @@ module GoodData
|
|
79
82
|
|
80
83
|
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
81
84
|
end
|
85
|
+
|
86
|
+
def build_url(url)
|
87
|
+
is_contain = url.include?(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
|
88
|
+
unless is_contain
|
89
|
+
if url.include?(SNOWFLAKE_SEPARATOR_PARAM)
|
90
|
+
url.concat("&")
|
91
|
+
else
|
92
|
+
url.concat(SNOWFLAKE_SEPARATOR_PARAM)
|
93
|
+
end
|
94
|
+
url.concat(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
|
95
|
+
end
|
96
|
+
|
97
|
+
url
|
98
|
+
end
|
82
99
|
end
|
83
100
|
end
|
84
101
|
end
|
@@ -4,7 +4,7 @@
|
|
4
4
|
# This source code is licensed under the BSD-style license found in the
|
5
5
|
# LICENSE file in the root directory of this source tree.
|
6
6
|
|
7
|
-
require '
|
7
|
+
require 'erb'
|
8
8
|
require 'fileutils'
|
9
9
|
require 'pathname'
|
10
10
|
|
@@ -18,15 +18,14 @@ module GoodData
|
|
18
18
|
# TODO: Add option for custom output dir
|
19
19
|
def project(name)
|
20
20
|
fail ArgumentError, 'No name specified' if name.nil?
|
21
|
-
|
22
21
|
FileUtils.mkdir(name)
|
23
22
|
FileUtils.cd(name) do
|
24
23
|
FileUtils.mkdir('model')
|
25
24
|
FileUtils.cd('model') do
|
26
25
|
input = File.read(TEMPLATES_PATH + 'project/model/model.rb.erb')
|
27
|
-
|
26
|
+
erb = ERB.new(input)
|
28
27
|
File.open('model.rb', 'w') do |f|
|
29
|
-
f.write(
|
28
|
+
f.write(erb.result_with_hash(:name => name))
|
30
29
|
end
|
31
30
|
end
|
32
31
|
|
@@ -36,9 +35,9 @@ module GoodData
|
|
36
35
|
end
|
37
36
|
|
38
37
|
input = File.read(TEMPLATES_PATH + 'project/Goodfile.erb')
|
39
|
-
|
38
|
+
erb = ERB.new(input)
|
40
39
|
File.open('Goodfile', 'w') do |f|
|
41
|
-
f.write(
|
40
|
+
f.write(erb.result)
|
42
41
|
end
|
43
42
|
end
|
44
43
|
end
|
@@ -51,15 +50,15 @@ module GoodData
|
|
51
50
|
FileUtils.mkdir(name)
|
52
51
|
FileUtils.cd(name) do
|
53
52
|
input = File.read(TEMPLATES_PATH + 'bricks/brick.rb.erb')
|
54
|
-
|
53
|
+
erb = ERB.new(input)
|
55
54
|
File.open('brick.rb', 'w') do |f|
|
56
|
-
f.write(
|
55
|
+
f.write(erb.result)
|
57
56
|
end
|
58
57
|
|
59
58
|
input = File.read(TEMPLATES_PATH + 'bricks/main.rb.erb')
|
60
|
-
|
59
|
+
erb = ERB.new(input)
|
61
60
|
File.open('main.rb', 'w') do |f|
|
62
|
-
f.write(
|
61
|
+
f.write(erb.result)
|
63
62
|
end
|
64
63
|
end
|
65
64
|
end
|
@@ -1,4 +1,5 @@
|
|
1
|
-
#
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
|
2
3
|
# This source code is licensed under the BSD-style license found in the
|
3
4
|
# LICENSE file in the root directory of this source tree.
|
4
5
|
|
@@ -11,6 +12,7 @@ module GoodData
|
|
11
12
|
@level = nil
|
12
13
|
end
|
13
14
|
|
15
|
+
# No body define need for dummy logger
|
14
16
|
def debug(*_args)
|
15
17
|
end
|
16
18
|
|
@@ -44,11 +44,14 @@ module GoodData
|
|
44
44
|
realize_link
|
45
45
|
when 's3'
|
46
46
|
realize_s3(params)
|
47
|
-
when 'redshift', 'snowflake', 'bigquery'
|
47
|
+
when 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql'
|
48
48
|
raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
|
49
|
-
|
50
49
|
require_relative '../cloud_resources/cloud_resources'
|
51
50
|
realize_cloud_resource(source, params)
|
51
|
+
when 'blobStorage'
|
52
|
+
require_relative '../cloud_resources/blobstorage/blobstorage_client'
|
53
|
+
blob_storage_client = GoodData::BlobStorageClient.new(params)
|
54
|
+
blob_storage_client.realize_blob(@options[:file], params)
|
52
55
|
else
|
53
56
|
raise "DataSource does not support type \"#{source}\""
|
54
57
|
end
|
@@ -112,12 +115,13 @@ module GoodData
|
|
112
115
|
end
|
113
116
|
|
114
117
|
def realize_s3(params)
|
115
|
-
s3_client = params['
|
118
|
+
s3_client = params['s3_client'] && params['s3_client']['client']
|
116
119
|
raise 'AWS client not present. Perhaps S3Middleware is missing in the brick definition?' if !s3_client || !s3_client.respond_to?(:bucket)
|
117
120
|
bucket_name = @options[:bucket]
|
118
|
-
key = @options[:key]
|
121
|
+
key = @options[:key].present? ? @options[:key] : @options[:file]
|
119
122
|
raise 'Key "bucket" is missing in S3 datasource' if bucket_name.blank?
|
120
|
-
raise 'Key "key" is missing in S3 datasource' if key.blank?
|
123
|
+
raise 'Key "key" or "file" is missing in S3 datasource' if key.blank?
|
124
|
+
|
121
125
|
GoodData.logger.info("Realizing download from S3. Bucket #{bucket_name}, object with key #{key}.")
|
122
126
|
filename = Digest::SHA256.new.hexdigest(@options.to_json)
|
123
127
|
bucket = s3_client.bucket(bucket_name)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
#
|
3
|
-
# Copyright (c) 2010-
|
3
|
+
# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
|
4
4
|
# This source code is licensed under the BSD-style license found in the
|
5
5
|
# LICENSE file in the root directory of this source tree.
|
6
6
|
|
@@ -34,6 +34,7 @@ module GoodData
|
|
34
34
|
end
|
35
35
|
|
36
36
|
set_const :GD_MAX_RETRY, (ENV['GD_MAX_RETRY'] && ENV['GD_MAX_RETRY'].to_i) || 12
|
37
|
+
AES_256_CBC_CIPHER = 'aes-256-cbc'
|
37
38
|
|
38
39
|
class << self
|
39
40
|
def error(msg)
|
@@ -222,7 +223,7 @@ module GoodData
|
|
222
223
|
# encrypts data with the given key. returns a binary data with the
|
223
224
|
# unhashed random iv in the first 16 bytes
|
224
225
|
def encrypt(data, key)
|
225
|
-
cipher = OpenSSL::Cipher::Cipher.new(
|
226
|
+
cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
|
226
227
|
cipher.encrypt
|
227
228
|
cipher.key = key = Digest::SHA256.digest(key)
|
228
229
|
random_iv = cipher.random_iv
|
@@ -236,7 +237,7 @@ module GoodData
|
|
236
237
|
|
237
238
|
# Simple encrypt data with given key
|
238
239
|
def simple_encrypt(data, key)
|
239
|
-
cipher = OpenSSL::Cipher::Cipher.new(
|
240
|
+
cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
|
240
241
|
cipher.encrypt
|
241
242
|
cipher.key = key
|
242
243
|
encrypted = cipher.update(data)
|
@@ -253,7 +254,7 @@ module GoodData
|
|
253
254
|
|
254
255
|
data = Base64.decode64(database64)
|
255
256
|
|
256
|
-
cipher = OpenSSL::Cipher::Cipher.new(
|
257
|
+
cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
|
257
258
|
cipher.decrypt
|
258
259
|
cipher.key = cipher_key = Digest::SHA256.digest(key)
|
259
260
|
random_iv = data[0..15] # extract iv from first 16 bytes
|
@@ -273,7 +274,7 @@ module GoodData
|
|
273
274
|
|
274
275
|
data = Base64.decode64(database64)
|
275
276
|
|
276
|
-
cipher = OpenSSL::Cipher::Cipher.new(
|
277
|
+
cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
|
277
278
|
cipher.decrypt
|
278
279
|
cipher.key = key
|
279
280
|
decrypted = cipher.update(data)
|