gooddata 2.1.14-java → 2.2.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gdc-ii-config.yaml +1 -1
- data/.github/workflows/build.yml +66 -0
- data/.github/workflows/pre-merge.yml +72 -0
- data/.sonar.settings +4 -0
- data/.travis.yml +78 -12
- data/CHANGELOG.md +62 -0
- data/Dockerfile +25 -14
- data/LICENSE +4418 -17
- data/LICENSE.rb +1 -1
- data/README.md +3 -3
- data/Rakefile +8 -1
- data/SDK_VERSION +1 -1
- data/VERSION +1 -1
- data/bin/test_projects_cleanup.rb +45 -3
- data/ci/mssql/pom.xml +62 -0
- data/ci/mysql/pom.xml +57 -0
- data/ci/postgresql/pom.xml +57 -0
- data/ci/redshift/pom.xml +1 -1
- data/dev-gooddata-sso.pub.encrypted +40 -40
- data/gdc_fossa_ruby_sdk.yaml +1 -0
- data/gooddata.gemspec +10 -6
- data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
- data/k8s/charts/lcm-bricks/templates/prometheus/alertingRules.yaml +11 -1
- data/lcm.rake +2 -2
- data/lib/gooddata/bricks/middleware/aws_middleware.rb +35 -9
- data/lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb +98 -0
- data/lib/gooddata/cloud_resources/{cloud_resouce_factory.rb → cloud_resource_factory.rb} +8 -0
- data/lib/gooddata/cloud_resources/cloud_resources.rb +1 -1
- data/lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mssql/mssql_client.rb +122 -0
- data/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +111 -0
- data/lib/gooddata/cloud_resources/postgresql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +106 -0
- data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +18 -1
- data/lib/gooddata/commands/scaffold.rb +9 -10
- data/lib/gooddata/core/nil_logger.rb +3 -1
- data/lib/gooddata/helpers/data_helper.rb +9 -5
- data/lib/gooddata/helpers/global_helpers.rb +6 -5
- data/lib/gooddata/lcm/actions/associate_clients.rb +8 -2
- data/lib/gooddata/lcm/actions/base_action.rb +0 -2
- data/lib/gooddata/lcm/actions/collect_meta.rb +3 -1
- data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +3 -2
- data/lib/gooddata/lcm/actions/provision_clients.rb +31 -10
- data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -7
- data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +64 -0
- data/lib/gooddata/lcm/actions/synchronize_ldm.rb +19 -8
- data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +16 -9
- data/lib/gooddata/lcm/actions/synchronize_users.rb +7 -6
- data/lib/gooddata/lcm/actions/update_metric_formats.rb +185 -0
- data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb +5 -0
- data/lib/gooddata/lcm/helpers/release_table_helper.rb +42 -8
- data/lib/gooddata/lcm/lcm2.rb +5 -2
- data/lib/gooddata/lcm/types/base_type.rb +0 -2
- data/lib/gooddata/mixins/md_object_query.rb +9 -6
- data/lib/gooddata/models/blueprint/project_blueprint.rb +0 -2
- data/lib/gooddata/models/client.rb +14 -12
- data/lib/gooddata/models/data_source.rb +668 -0
- data/lib/gooddata/models/dataset_mapping.rb +36 -0
- data/lib/gooddata/models/domain.rb +3 -2
- data/lib/gooddata/models/metadata/analytical_dashboard.rb +49 -0
- data/lib/gooddata/models/metadata/analytical_visualization_object.rb +30 -0
- data/lib/gooddata/models/metadata/label.rb +26 -27
- data/lib/gooddata/models/metadata/visualization_object.rb +50 -0
- data/lib/gooddata/models/project.rb +66 -19
- data/lib/gooddata/models/schedule.rb +13 -1
- data/lib/gooddata/models/user_filters/user_filter_builder.rb +58 -54
- data/lib/gooddata/models/user_group.rb +0 -1
- data/lib/gooddata/rest/connection.rb +6 -4
- data/lib/gooddata/rest/phmap.rb +2 -1
- data/lib/gooddata.rb +2 -0
- data/rubydev_public.gpg.encrypted +51 -51
- data/rubydev_secret_keys.gpg.encrypted +109 -109
- metadata +51 -27
- data/DEPENDENCIES.md +0 -880
- data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb +0 -37
- data/lib/gooddata/helpers/data_source_helpers.rb +0 -47
@@ -0,0 +1,98 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'pathname'
|
10
|
+
require "azure/storage/blob"
|
11
|
+
|
12
|
+
module GoodData
|
13
|
+
class BlobStorageClient
|
14
|
+
SAS_URL_PATTERN = %r{(^https?:\/\/[^\/]*)\/.*\?(.*)}
|
15
|
+
INVALID_BLOB_GENERAL_MESSAGE = "The connection string is not valid."
|
16
|
+
INVALID_BLOB_SIG_WELL_FORMED_MESSAGE = "The signature format is not valid."
|
17
|
+
INVALID_BLOB_CONTAINER_MESSAGE = "ContainerNotFound"
|
18
|
+
INVALID_BLOB_CONTAINER_FORMED_MESSAGE = "The container with the specified name is not found."
|
19
|
+
INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE = "Signature not valid in the specified time frame"
|
20
|
+
INVALID_BLOB_EXPIRED_MESSAGE = "The signature expired."
|
21
|
+
INVALID_BLOB_INVALID_CONNECTION_STRING_MESSAGE = "The connection string is not valid."
|
22
|
+
INVALID_BLOB_PATH_MESSAGE = "BlobNotFound"
|
23
|
+
INVALID_BLOB_INVALID_PATH_MESSAGE = "The path to the data is not found."
|
24
|
+
|
25
|
+
attr_reader :use_sas
|
26
|
+
|
27
|
+
def initialize(options = {})
|
28
|
+
raise("Data Source needs a client to Blob Storage to be able to get blob file but 'blobStorage_client' is empty.") unless options['blobStorage_client']
|
29
|
+
|
30
|
+
if options['blobStorage_client']['connectionString'] && options['blobStorage_client']['container']
|
31
|
+
@connection_string = options['blobStorage_client']['connectionString']
|
32
|
+
@container = options['blobStorage_client']['container']
|
33
|
+
@path = options['blobStorage_client']['path']
|
34
|
+
@use_sas = false
|
35
|
+
build_sas(@connection_string)
|
36
|
+
else
|
37
|
+
raise('Missing connection info for Blob Storage client')
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def realize_blob(file, _params)
|
42
|
+
GoodData.gd_logger.info("Realizing download from Blob Storage. Container #{@container}.")
|
43
|
+
filename = ''
|
44
|
+
begin
|
45
|
+
connect
|
46
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
47
|
+
blob_name = get_blob_name(@path, file)
|
48
|
+
|
49
|
+
measure = Benchmark.measure do
|
50
|
+
_blob, content = @client.get_blob(@container, blob_name)
|
51
|
+
File.open(filename, "wb") { |f| f.write(content) }
|
52
|
+
end
|
53
|
+
rescue => e
|
54
|
+
raise_error(e)
|
55
|
+
end
|
56
|
+
GoodData.gd_logger.info("Done downloading file type=blobStorage status=finished duration=#{measure.real}")
|
57
|
+
filename
|
58
|
+
end
|
59
|
+
|
60
|
+
def connect
|
61
|
+
GoodData.logger.info "Setting up connection to Blob Storage"
|
62
|
+
if use_sas
|
63
|
+
@client = Azure::Storage::Blob::BlobService.create(:storage_blob_host => @host, :storage_sas_token => @sas_token)
|
64
|
+
else
|
65
|
+
@client = Azure::Storage::Blob::BlobService.create_from_connection_string(@connection_string)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def build_sas(url)
|
70
|
+
matches = url.scan(SAS_URL_PATTERN)
|
71
|
+
return unless matches && matches[0]
|
72
|
+
|
73
|
+
@use_sas = true
|
74
|
+
@host = matches[0][0]
|
75
|
+
@sas_token = matches[0][1]
|
76
|
+
end
|
77
|
+
|
78
|
+
def raise_error(e)
|
79
|
+
if e.message && e.message.include?(INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE)
|
80
|
+
raise INVALID_BLOB_EXPIRED_MESSAGE
|
81
|
+
elsif e.message && e.message.include?(INVALID_BLOB_SIG_WELL_FORMED_MESSAGE)
|
82
|
+
raise INVALID_BLOB_SIG_WELL_FORMED_MESSAGE
|
83
|
+
elsif e.message && e.message.include?(INVALID_BLOB_CONTAINER_MESSAGE)
|
84
|
+
raise INVALID_BLOB_CONTAINER_FORMED_MESSAGE
|
85
|
+
elsif e.message && e.message.include?(INVALID_BLOB_PATH_MESSAGE)
|
86
|
+
raise INVALID_BLOB_INVALID_PATH_MESSAGE
|
87
|
+
else
|
88
|
+
raise INVALID_BLOB_GENERAL_MESSAGE
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def get_blob_name(path, file)
|
93
|
+
return file unless path
|
94
|
+
|
95
|
+
path.rindex('/') == path.length - 1 ? "#{path}#{file}" : "#{path}/#{file}"
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
@@ -11,7 +11,15 @@ module GoodData
|
|
11
11
|
module CloudResources
|
12
12
|
class CloudResourceFactory
|
13
13
|
class << self
|
14
|
+
def load_cloud_resource(type)
|
15
|
+
base = "#{Pathname(__FILE__).dirname.expand_path}#{File::SEPARATOR}#{type}#{File::SEPARATOR}"
|
16
|
+
Dir.glob(base + '**/*.rb').each do |file|
|
17
|
+
require file
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
14
21
|
def create(type, data = {}, opts = {})
|
22
|
+
load_cloud_resource(type)
|
15
23
|
clients = CloudResourceClient.descendants.select { |c| c.respond_to?("accept?") && c.send("accept?", type) }
|
16
24
|
raise "DataSource does not support type \"#{type}\"" if clients.empty?
|
17
25
|
|
File without changes
|
@@ -0,0 +1,122 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'java'
|
10
|
+
require 'pathname'
|
11
|
+
require_relative '../cloud_resource_client'
|
12
|
+
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
15
|
+
require file unless file.start_with?('lcm-mssql-driver')
|
16
|
+
end
|
17
|
+
|
18
|
+
module GoodData
|
19
|
+
module CloudResources
|
20
|
+
class MSSQLClient < CloudResourceClient
|
21
|
+
MSSQL_SEPARATOR_PARAM = ";"
|
22
|
+
MSSQL_URL_PATTERN = %r{jdbc:sqlserver://([^:]+)(:([0-9]+))?(/)?}
|
23
|
+
MSSQL_DEFAULT_PORT = 1433
|
24
|
+
LOGIN_TIME_OUT = 30
|
25
|
+
MSSQL_FETCH_SIZE = 10_000
|
26
|
+
VERIFY_FULL = 'verify-full'
|
27
|
+
PREFER = 'prefer'
|
28
|
+
REQUIRE = 'require'
|
29
|
+
|
30
|
+
class << self
|
31
|
+
def accept?(type)
|
32
|
+
type == 'mssql'
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def initialize(options = {})
|
37
|
+
raise("Data Source needs a client to MSSQL to be able to query the storage but 'mssql_client' is empty.") unless options['mssql_client']
|
38
|
+
|
39
|
+
connection = options['mssql_client']['connection']
|
40
|
+
if connection.is_a?(Hash)
|
41
|
+
@database = connection['database']
|
42
|
+
@schema = connection['schema']
|
43
|
+
@authentication = connection['authentication']
|
44
|
+
@ssl_mode = connection['sslMode']
|
45
|
+
@url = connection['url']
|
46
|
+
|
47
|
+
validate
|
48
|
+
else
|
49
|
+
raise('Missing connection info for MSSQL client')
|
50
|
+
end
|
51
|
+
|
52
|
+
Java.com.microsoft.sqlserver.jdbc.SQLServerDriver
|
53
|
+
end
|
54
|
+
|
55
|
+
def realize_query(query, _params)
|
56
|
+
GoodData.gd_logger.info("Realize SQL query: type=mssql status=started")
|
57
|
+
|
58
|
+
connect
|
59
|
+
|
60
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
61
|
+
measure = Benchmark.measure do
|
62
|
+
statement = @connection.create_statement
|
63
|
+
statement.set_fetch_size(MSSQL_FETCH_SIZE)
|
64
|
+
has_result = statement.execute(query)
|
65
|
+
if has_result
|
66
|
+
result = statement.get_result_set
|
67
|
+
metadata = result.get_meta_data
|
68
|
+
col_count = metadata.column_count
|
69
|
+
CSV.open(filename, 'wb') do |csv|
|
70
|
+
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
71
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
GoodData.gd_logger.info("Realize SQL query: type=mssql status=finished duration=#{measure.real}")
|
77
|
+
filename
|
78
|
+
ensure
|
79
|
+
@connection&.close
|
80
|
+
@connection = nil
|
81
|
+
end
|
82
|
+
|
83
|
+
def connect
|
84
|
+
connection_string = build_connection_string
|
85
|
+
GoodData.logger.info "Setting up connection to MSSQL #{connection_string}"
|
86
|
+
|
87
|
+
authentication = @authentication['basic'] || @authentication['activeDirectoryPassword']
|
88
|
+
|
89
|
+
prop = java.util.Properties.new
|
90
|
+
prop.setProperty('userName', authentication['userName'])
|
91
|
+
prop.setProperty('password', authentication['password'])
|
92
|
+
|
93
|
+
@connection = java.sql.DriverManager.getConnection(connection_string, prop)
|
94
|
+
end
|
95
|
+
|
96
|
+
def validate
|
97
|
+
raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
|
98
|
+
|
99
|
+
raise "Instance name is not supported" if @url !~ /^[^\\]*$/
|
100
|
+
|
101
|
+
raise "The connection url is invalid. Parameter is not supported." if @url.include? MSSQL_SEPARATOR_PARAM
|
102
|
+
|
103
|
+
url_matches = @url.scan(MSSQL_URL_PATTERN)
|
104
|
+
raise "Cannot reach the url" if url_matches.nil? || url_matches.length.zero?
|
105
|
+
|
106
|
+
raise "The authentication method is not supported." unless @authentication['basic'] || @authentication['activeDirectoryPassword']
|
107
|
+
end
|
108
|
+
|
109
|
+
def build_connection_string
|
110
|
+
encrypt = @ssl_mode != PREFER
|
111
|
+
trust_server_certificate = @ssl_mode == REQUIRE
|
112
|
+
|
113
|
+
"#{@url};" \
|
114
|
+
"database=#{@database};" \
|
115
|
+
"encrypt=#{encrypt};" \
|
116
|
+
"trustServerCertificate=#{trust_server_certificate};" \
|
117
|
+
"loginTimeout=#{LOGIN_TIME_OUT};" \
|
118
|
+
"#{'authentication=ActiveDirectoryPassword;' if @authentication['activeDirectoryPassword']}"
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
File without changes
|
@@ -0,0 +1,111 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'java'
|
10
|
+
require 'pathname'
|
11
|
+
require_relative '../cloud_resource_client'
|
12
|
+
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
15
|
+
require file unless file.start_with?('lcm-mysql-driver')
|
16
|
+
end
|
17
|
+
|
18
|
+
module GoodData
|
19
|
+
module CloudResources
|
20
|
+
class MysqlClient < CloudResourceClient
|
21
|
+
JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
|
22
|
+
MYSQL_DEFAULT_PORT = 3306
|
23
|
+
JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://'
|
24
|
+
VERIFY_FULL = 'VERIFY_IDENTITY'
|
25
|
+
PREFER = 'PREFERRED'
|
26
|
+
REQUIRE = 'REQUIRED'
|
27
|
+
MYSQL_FETCH_SIZE = 1000
|
28
|
+
|
29
|
+
class << self
|
30
|
+
def accept?(type)
|
31
|
+
type == 'mysql'
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def initialize(options = {})
|
36
|
+
raise("Data Source needs a client to Mysql to be able to query the storage but 'mysql_client' is empty.") unless options['mysql_client']
|
37
|
+
|
38
|
+
if options['mysql_client']['connection'].is_a?(Hash)
|
39
|
+
@database = options['mysql_client']['connection']['database']
|
40
|
+
@authentication = options['mysql_client']['connection']['authentication']
|
41
|
+
@ssl_mode = options['mysql_client']['connection']['sslMode']
|
42
|
+
raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
|
43
|
+
|
44
|
+
@url = build_url(options['mysql_client']['connection']['url'])
|
45
|
+
else
|
46
|
+
raise('Missing connection info for Mysql client')
|
47
|
+
end
|
48
|
+
|
49
|
+
Java.com.mysql.cj.jdbc.Driver
|
50
|
+
end
|
51
|
+
|
52
|
+
def realize_query(query, _params)
|
53
|
+
GoodData.gd_logger.info("Realize SQL query: type=mysql status=started")
|
54
|
+
|
55
|
+
connect
|
56
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
57
|
+
measure = Benchmark.measure do
|
58
|
+
statement = @connection.create_statement
|
59
|
+
statement.set_fetch_size(MYSQL_FETCH_SIZE)
|
60
|
+
has_result = statement.execute(query)
|
61
|
+
if has_result
|
62
|
+
result = statement.get_result_set
|
63
|
+
metadata = result.get_meta_data
|
64
|
+
col_count = metadata.column_count
|
65
|
+
CSV.open(filename, 'wb') do |csv|
|
66
|
+
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
67
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
GoodData.gd_logger.info("Realize SQL query: type=mysql status=finished duration=#{measure.real}")
|
72
|
+
filename
|
73
|
+
ensure
|
74
|
+
@connection&.close
|
75
|
+
@connection = nil
|
76
|
+
end
|
77
|
+
|
78
|
+
def connect
|
79
|
+
GoodData.logger.info "Setting up connection to Mysql #{@url}"
|
80
|
+
|
81
|
+
prop = java.util.Properties.new
|
82
|
+
prop.setProperty('user', @authentication['basic']['userName'])
|
83
|
+
prop.setProperty('password', @authentication['basic']['password'])
|
84
|
+
|
85
|
+
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
86
|
+
@connection.set_auto_commit(false)
|
87
|
+
end
|
88
|
+
|
89
|
+
def build_url(url)
|
90
|
+
matches = url.scan(JDBC_MYSQL_PATTERN)
|
91
|
+
raise 'Cannot reach the url' unless matches
|
92
|
+
|
93
|
+
host = matches[0][0]
|
94
|
+
port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT
|
95
|
+
|
96
|
+
"#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{get_ssl_mode(@ssl_mode)}&useCursorFetch=true&enabledTLSProtocols=TLSv1.2"
|
97
|
+
end
|
98
|
+
|
99
|
+
def get_ssl_mode(ssl_mode)
|
100
|
+
mode = PREFER
|
101
|
+
if ssl_mode == 'verify-full'
|
102
|
+
mode = VERIFY_FULL
|
103
|
+
elsif ssl_mode == 'require'
|
104
|
+
mode = REQUIRE
|
105
|
+
end
|
106
|
+
|
107
|
+
mode
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
File without changes
|
@@ -0,0 +1,106 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'java'
|
10
|
+
require 'pathname'
|
11
|
+
require_relative '../cloud_resource_client'
|
12
|
+
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
15
|
+
require file unless file.start_with?('lcm-postgresql-driver')
|
16
|
+
end
|
17
|
+
|
18
|
+
module GoodData
|
19
|
+
module CloudResources
|
20
|
+
class PostgresClient < CloudResourceClient
|
21
|
+
JDBC_POSTGRES_PATTERN = %r{jdbc:postgresql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
|
22
|
+
POSTGRES_DEFAULT_PORT = 5432
|
23
|
+
JDBC_POSTGRES_PROTOCOL = 'jdbc:postgresql://'
|
24
|
+
SSL_JAVA_FACTORY = '&sslfactory=org.postgresql.ssl.DefaultJavaSSLFactory'
|
25
|
+
VERIFY_FULL = 'verify-full'
|
26
|
+
PREFER = 'prefer'
|
27
|
+
REQUIRE = 'require'
|
28
|
+
POSTGRES_SET_SCHEMA_COMMAND = "set search_path to"
|
29
|
+
POSTGRES_FETCH_SIZE = 1000
|
30
|
+
|
31
|
+
class << self
|
32
|
+
def accept?(type)
|
33
|
+
type == 'postgresql'
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def initialize(options = {})
|
38
|
+
raise("Data Source needs a client to Postgres to be able to query the storage but 'postgresql_client' is empty.") unless options['postgresql_client']
|
39
|
+
|
40
|
+
if options['postgresql_client']['connection'].is_a?(Hash)
|
41
|
+
@database = options['postgresql_client']['connection']['database']
|
42
|
+
@schema = options['postgresql_client']['connection']['schema'] || 'public'
|
43
|
+
@authentication = options['postgresql_client']['connection']['authentication']
|
44
|
+
@ssl_mode = options['postgresql_client']['connection']['sslMode']
|
45
|
+
raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
|
46
|
+
|
47
|
+
@url = build_url(options['postgresql_client']['connection']['url'])
|
48
|
+
else
|
49
|
+
raise('Missing connection info for Postgres client')
|
50
|
+
end
|
51
|
+
|
52
|
+
Java.org.postgresql.Driver
|
53
|
+
end
|
54
|
+
|
55
|
+
def realize_query(query, _params)
|
56
|
+
GoodData.gd_logger.info("Realize SQL query: type=postgresql status=started")
|
57
|
+
|
58
|
+
connect
|
59
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
60
|
+
measure = Benchmark.measure do
|
61
|
+
statement = @connection.create_statement
|
62
|
+
statement.set_fetch_size(POSTGRES_FETCH_SIZE)
|
63
|
+
has_result = statement.execute(query)
|
64
|
+
if has_result
|
65
|
+
result = statement.get_result_set
|
66
|
+
metadata = result.get_meta_data
|
67
|
+
col_count = metadata.column_count
|
68
|
+
CSV.open(filename, 'wb') do |csv|
|
69
|
+
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
70
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
GoodData.gd_logger.info("Realize SQL query: type=postgresql status=finished duration=#{measure.real}")
|
75
|
+
filename
|
76
|
+
ensure
|
77
|
+
@connection&.close
|
78
|
+
@connection = nil
|
79
|
+
end
|
80
|
+
|
81
|
+
def connect
|
82
|
+
GoodData.logger.info "Setting up connection to Postgresql #{@url}"
|
83
|
+
|
84
|
+
prop = java.util.Properties.new
|
85
|
+
prop.setProperty('user', @authentication['basic']['userName'])
|
86
|
+
prop.setProperty('password', @authentication['basic']['password'])
|
87
|
+
prop.setProperty('schema', @schema)
|
88
|
+
|
89
|
+
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
90
|
+
statement = @connection.create_statement
|
91
|
+
statement.execute("#{POSTGRES_SET_SCHEMA_COMMAND} #{@schema}")
|
92
|
+
@connection.set_auto_commit(false)
|
93
|
+
end
|
94
|
+
|
95
|
+
def build_url(url)
|
96
|
+
matches = url.scan(JDBC_POSTGRES_PATTERN)
|
97
|
+
raise 'Cannot reach the url' unless matches
|
98
|
+
|
99
|
+
host = matches[0][0]
|
100
|
+
port = matches[0][2]&.to_i || POSTGRES_DEFAULT_PORT
|
101
|
+
|
102
|
+
"#{JDBC_POSTGRES_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{@ssl_mode}#{VERIFY_FULL == @ssl_mode ? SSL_JAVA_FACTORY : ''}"
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
@@ -18,6 +18,9 @@ end
|
|
18
18
|
module GoodData
|
19
19
|
module CloudResources
|
20
20
|
class SnowflakeClient < CloudResourceClient
|
21
|
+
SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=GoodData_Platform'
|
22
|
+
SNOWFLAKE_SEPARATOR_PARAM = '?'
|
23
|
+
|
21
24
|
class << self
|
22
25
|
def accept?(type)
|
23
26
|
type == 'snowflake'
|
@@ -31,7 +34,7 @@ module GoodData
|
|
31
34
|
@database = options['snowflake_client']['connection']['database']
|
32
35
|
@schema = options['snowflake_client']['connection']['schema'] || 'public'
|
33
36
|
@warehouse = options['snowflake_client']['connection']['warehouse']
|
34
|
-
@url = options['snowflake_client']['connection']['url']
|
37
|
+
@url = build_url(options['snowflake_client']['connection']['url'])
|
35
38
|
@authentication = options['snowflake_client']['connection']['authentication']
|
36
39
|
else
|
37
40
|
raise('Missing connection info for Snowflake client')
|
@@ -79,6 +82,20 @@ module GoodData
|
|
79
82
|
|
80
83
|
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
81
84
|
end
|
85
|
+
|
86
|
+
def build_url(url)
|
87
|
+
is_contain = url.include?(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
|
88
|
+
unless is_contain
|
89
|
+
if url.include?(SNOWFLAKE_SEPARATOR_PARAM)
|
90
|
+
url.concat("&")
|
91
|
+
else
|
92
|
+
url.concat(SNOWFLAKE_SEPARATOR_PARAM)
|
93
|
+
end
|
94
|
+
url.concat(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
|
95
|
+
end
|
96
|
+
|
97
|
+
url
|
98
|
+
end
|
82
99
|
end
|
83
100
|
end
|
84
101
|
end
|
@@ -4,7 +4,7 @@
|
|
4
4
|
# This source code is licensed under the BSD-style license found in the
|
5
5
|
# LICENSE file in the root directory of this source tree.
|
6
6
|
|
7
|
-
require '
|
7
|
+
require 'erb'
|
8
8
|
require 'fileutils'
|
9
9
|
require 'pathname'
|
10
10
|
|
@@ -18,15 +18,14 @@ module GoodData
|
|
18
18
|
# TODO: Add option for custom output dir
|
19
19
|
def project(name)
|
20
20
|
fail ArgumentError, 'No name specified' if name.nil?
|
21
|
-
|
22
21
|
FileUtils.mkdir(name)
|
23
22
|
FileUtils.cd(name) do
|
24
23
|
FileUtils.mkdir('model')
|
25
24
|
FileUtils.cd('model') do
|
26
25
|
input = File.read(TEMPLATES_PATH + 'project/model/model.rb.erb')
|
27
|
-
|
26
|
+
erb = ERB.new(input)
|
28
27
|
File.open('model.rb', 'w') do |f|
|
29
|
-
f.write(
|
28
|
+
f.write(erb.result_with_hash(:name => name))
|
30
29
|
end
|
31
30
|
end
|
32
31
|
|
@@ -36,9 +35,9 @@ module GoodData
|
|
36
35
|
end
|
37
36
|
|
38
37
|
input = File.read(TEMPLATES_PATH + 'project/Goodfile.erb')
|
39
|
-
|
38
|
+
erb = ERB.new(input)
|
40
39
|
File.open('Goodfile', 'w') do |f|
|
41
|
-
f.write(
|
40
|
+
f.write(erb.result)
|
42
41
|
end
|
43
42
|
end
|
44
43
|
end
|
@@ -51,15 +50,15 @@ module GoodData
|
|
51
50
|
FileUtils.mkdir(name)
|
52
51
|
FileUtils.cd(name) do
|
53
52
|
input = File.read(TEMPLATES_PATH + 'bricks/brick.rb.erb')
|
54
|
-
|
53
|
+
erb = ERB.new(input)
|
55
54
|
File.open('brick.rb', 'w') do |f|
|
56
|
-
f.write(
|
55
|
+
f.write(erb.result)
|
57
56
|
end
|
58
57
|
|
59
58
|
input = File.read(TEMPLATES_PATH + 'bricks/main.rb.erb')
|
60
|
-
|
59
|
+
erb = ERB.new(input)
|
61
60
|
File.open('main.rb', 'w') do |f|
|
62
|
-
f.write(
|
61
|
+
f.write(erb.result)
|
63
62
|
end
|
64
63
|
end
|
65
64
|
end
|
@@ -1,4 +1,5 @@
|
|
1
|
-
#
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
|
2
3
|
# This source code is licensed under the BSD-style license found in the
|
3
4
|
# LICENSE file in the root directory of this source tree.
|
4
5
|
|
@@ -11,6 +12,7 @@ module GoodData
|
|
11
12
|
@level = nil
|
12
13
|
end
|
13
14
|
|
15
|
+
# No body define need for dummy logger
|
14
16
|
def debug(*_args)
|
15
17
|
end
|
16
18
|
|
@@ -44,11 +44,14 @@ module GoodData
|
|
44
44
|
realize_link
|
45
45
|
when 's3'
|
46
46
|
realize_s3(params)
|
47
|
-
when 'redshift', 'snowflake', 'bigquery'
|
47
|
+
when 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql'
|
48
48
|
raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
|
49
|
-
|
50
49
|
require_relative '../cloud_resources/cloud_resources'
|
51
50
|
realize_cloud_resource(source, params)
|
51
|
+
when 'blobStorage'
|
52
|
+
require_relative '../cloud_resources/blobstorage/blobstorage_client'
|
53
|
+
blob_storage_client = GoodData::BlobStorageClient.new(params)
|
54
|
+
blob_storage_client.realize_blob(@options[:file], params)
|
52
55
|
else
|
53
56
|
raise "DataSource does not support type \"#{source}\""
|
54
57
|
end
|
@@ -112,12 +115,13 @@ module GoodData
|
|
112
115
|
end
|
113
116
|
|
114
117
|
def realize_s3(params)
|
115
|
-
s3_client = params['
|
118
|
+
s3_client = params['s3_client'] && params['s3_client']['client']
|
116
119
|
raise 'AWS client not present. Perhaps S3Middleware is missing in the brick definition?' if !s3_client || !s3_client.respond_to?(:bucket)
|
117
120
|
bucket_name = @options[:bucket]
|
118
|
-
key = @options[:key]
|
121
|
+
key = @options[:key].present? ? @options[:key] : @options[:file]
|
119
122
|
raise 'Key "bucket" is missing in S3 datasource' if bucket_name.blank?
|
120
|
-
raise 'Key "key" is missing in S3 datasource' if key.blank?
|
123
|
+
raise 'Key "key" or "file" is missing in S3 datasource' if key.blank?
|
124
|
+
|
121
125
|
GoodData.logger.info("Realizing download from S3. Bucket #{bucket_name}, object with key #{key}.")
|
122
126
|
filename = Digest::SHA256.new.hexdigest(@options.to_json)
|
123
127
|
bucket = s3_client.bucket(bucket_name)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
#
|
3
|
-
# Copyright (c) 2010-
|
3
|
+
# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
|
4
4
|
# This source code is licensed under the BSD-style license found in the
|
5
5
|
# LICENSE file in the root directory of this source tree.
|
6
6
|
|
@@ -34,6 +34,7 @@ module GoodData
|
|
34
34
|
end
|
35
35
|
|
36
36
|
set_const :GD_MAX_RETRY, (ENV['GD_MAX_RETRY'] && ENV['GD_MAX_RETRY'].to_i) || 12
|
37
|
+
AES_256_CBC_CIPHER = 'aes-256-cbc'
|
37
38
|
|
38
39
|
class << self
|
39
40
|
def error(msg)
|
@@ -222,7 +223,7 @@ module GoodData
|
|
222
223
|
# encrypts data with the given key. returns a binary data with the
|
223
224
|
# unhashed random iv in the first 16 bytes
|
224
225
|
def encrypt(data, key)
|
225
|
-
cipher = OpenSSL::Cipher::Cipher.new(
|
226
|
+
cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
|
226
227
|
cipher.encrypt
|
227
228
|
cipher.key = key = Digest::SHA256.digest(key)
|
228
229
|
random_iv = cipher.random_iv
|
@@ -236,7 +237,7 @@ module GoodData
|
|
236
237
|
|
237
238
|
# Simple encrypt data with given key
|
238
239
|
def simple_encrypt(data, key)
|
239
|
-
cipher = OpenSSL::Cipher::Cipher.new(
|
240
|
+
cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
|
240
241
|
cipher.encrypt
|
241
242
|
cipher.key = key
|
242
243
|
encrypted = cipher.update(data)
|
@@ -253,7 +254,7 @@ module GoodData
|
|
253
254
|
|
254
255
|
data = Base64.decode64(database64)
|
255
256
|
|
256
|
-
cipher = OpenSSL::Cipher::Cipher.new(
|
257
|
+
cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
|
257
258
|
cipher.decrypt
|
258
259
|
cipher.key = cipher_key = Digest::SHA256.digest(key)
|
259
260
|
random_iv = data[0..15] # extract iv from first 16 bytes
|
@@ -273,7 +274,7 @@ module GoodData
|
|
273
274
|
|
274
275
|
data = Base64.decode64(database64)
|
275
276
|
|
276
|
-
cipher = OpenSSL::Cipher::Cipher.new(
|
277
|
+
cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
|
277
278
|
cipher.decrypt
|
278
279
|
cipher.key = key
|
279
280
|
decrypted = cipher.update(data)
|