gooddata 2.1.19 → 2.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gdc-ii-config.yaml +42 -1
- data/.github/workflows/build.yml +67 -0
- data/.github/workflows/pre-merge.yml +72 -0
- data/.pronto.yml +1 -0
- data/.rubocop.yml +2 -14
- data/CHANGELOG.md +47 -0
- data/Dockerfile +27 -14
- data/Dockerfile.jruby +5 -15
- data/Dockerfile.ruby +5 -7
- data/Gemfile +4 -2
- data/LICENSE +4409 -16
- data/README.md +6 -6
- data/Rakefile +1 -1
- data/SDK_VERSION +1 -1
- data/VERSION +1 -1
- data/bin/run_brick.rb +7 -0
- data/ci/mssql/pom.xml +62 -0
- data/ci/mysql/pom.xml +62 -0
- data/ci/redshift/pom.xml +4 -5
- data/docker-compose.lcm.yml +42 -4
- data/docker-compose.yml +42 -0
- data/gooddata.gemspec +21 -21
- data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
- data/lcm.rake +11 -8
- data/lib/gooddata/bricks/base_pipeline.rb +26 -0
- data/lib/gooddata/bricks/brick.rb +0 -1
- data/lib/gooddata/bricks/middleware/aws_middleware.rb +35 -9
- data/lib/gooddata/bricks/middleware/execution_result_middleware.rb +3 -3
- data/lib/gooddata/bricks/pipeline.rb +2 -14
- data/lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb +98 -0
- data/lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mssql/mssql_client.rb +122 -0
- data/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +121 -0
- data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +0 -1
- data/lib/gooddata/cloud_resources/redshift/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/redshift/redshift_client.rb +0 -2
- data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +18 -1
- data/lib/gooddata/helpers/data_helper.rb +9 -4
- data/lib/gooddata/lcm/actions/base_action.rb +157 -0
- data/lib/gooddata/lcm/actions/collect_data_product.rb +2 -1
- data/lib/gooddata/lcm/actions/collect_meta.rb +3 -1
- data/lib/gooddata/lcm/actions/collect_projects_warning_status.rb +53 -0
- data/lib/gooddata/lcm/actions/collect_segment_clients.rb +14 -0
- data/lib/gooddata/lcm/actions/initialize_continue_on_error_option.rb +87 -0
- data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +31 -4
- data/lib/gooddata/lcm/actions/provision_clients.rb +34 -5
- data/lib/gooddata/lcm/actions/synchronize_cas.rb +24 -4
- data/lib/gooddata/lcm/actions/synchronize_clients.rb +112 -11
- data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +89 -0
- data/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb +48 -11
- data/lib/gooddata/lcm/actions/synchronize_kd_dashboard_permission.rb +103 -0
- data/lib/gooddata/lcm/actions/synchronize_ldm.rb +79 -23
- data/lib/gooddata/lcm/actions/synchronize_ldm_layout.rb +98 -0
- data/lib/gooddata/lcm/actions/synchronize_pp_dashboard_permission.rb +108 -0
- data/lib/gooddata/lcm/actions/synchronize_schedules.rb +31 -1
- data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +26 -18
- data/lib/gooddata/lcm/actions/synchronize_user_groups.rb +30 -4
- data/lib/gooddata/lcm/actions/synchronize_users.rb +11 -10
- data/lib/gooddata/lcm/actions/update_metric_formats.rb +202 -0
- data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb +5 -0
- data/lib/gooddata/lcm/exceptions/lcm_execution_warning.rb +15 -0
- data/lib/gooddata/lcm/helpers/check_helper.rb +19 -0
- data/lib/gooddata/lcm/helpers/release_table_helper.rb +42 -8
- data/lib/gooddata/lcm/lcm2.rb +50 -4
- data/lib/gooddata/lcm/user_bricks_helper.rb +9 -0
- data/lib/gooddata/mixins/inspector.rb +1 -1
- data/lib/gooddata/mixins/md_object_query.rb +1 -0
- data/lib/gooddata/models/data_source.rb +5 -1
- data/lib/gooddata/models/dataset_mapping.rb +36 -0
- data/lib/gooddata/models/ldm_layout.rb +38 -0
- data/lib/gooddata/models/metadata/label.rb +26 -27
- data/lib/gooddata/models/project.rb +230 -30
- data/lib/gooddata/models/project_creator.rb +83 -6
- data/lib/gooddata/models/schedule.rb +13 -1
- data/lib/gooddata/models/segment.rb +2 -1
- data/lib/gooddata/models/user_filters/user_filter_builder.rb +162 -68
- data/lib/gooddata/rest/connection.rb +5 -3
- data/lib/gooddata/rest/phmap.rb +2 -0
- data/lib/gooddata.rb +1 -0
- data/lib/gooddata_brick_base.rb +35 -0
- data/sonar-project.properties +6 -0
- metadata +100 -68
- data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb +0 -37
- data/lib/gooddata/cloud_resources/redshift/drivers/log4j.properties +0 -15
@@ -0,0 +1,98 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'pathname'
|
10
|
+
require "azure/storage/blob"
|
11
|
+
|
12
|
+
module GoodData
|
13
|
+
class BlobStorageClient
|
14
|
+
SAS_URL_PATTERN = %r{(^https?:\/\/[^\/]*)\/.*\?(.*)}
|
15
|
+
INVALID_BLOB_GENERAL_MESSAGE = "The connection string is not valid."
|
16
|
+
INVALID_BLOB_SIG_WELL_FORMED_MESSAGE = "The signature format is not valid."
|
17
|
+
INVALID_BLOB_CONTAINER_MESSAGE = "ContainerNotFound"
|
18
|
+
INVALID_BLOB_CONTAINER_FORMED_MESSAGE = "The container with the specified name is not found."
|
19
|
+
INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE = "Signature not valid in the specified time frame"
|
20
|
+
INVALID_BLOB_EXPIRED_MESSAGE = "The signature expired."
|
21
|
+
INVALID_BLOB_INVALID_CONNECTION_STRING_MESSAGE = "The connection string is not valid."
|
22
|
+
INVALID_BLOB_PATH_MESSAGE = "BlobNotFound"
|
23
|
+
INVALID_BLOB_INVALID_PATH_MESSAGE = "The path to the data is not found."
|
24
|
+
|
25
|
+
attr_reader :use_sas
|
26
|
+
|
27
|
+
def initialize(options = {})
|
28
|
+
raise("Data Source needs a client to Blob Storage to be able to get blob file but 'blobStorage_client' is empty.") unless options['blobStorage_client']
|
29
|
+
|
30
|
+
if options['blobStorage_client']['connectionString'] && options['blobStorage_client']['container']
|
31
|
+
@connection_string = options['blobStorage_client']['connectionString']
|
32
|
+
@container = options['blobStorage_client']['container']
|
33
|
+
@path = options['blobStorage_client']['path']
|
34
|
+
@use_sas = false
|
35
|
+
build_sas(@connection_string)
|
36
|
+
else
|
37
|
+
raise('Missing connection info for Blob Storage client')
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def realize_blob(file, _params)
|
42
|
+
GoodData.gd_logger.info("Realizing download from Blob Storage. Container #{@container}.")
|
43
|
+
filename = ''
|
44
|
+
begin
|
45
|
+
connect
|
46
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
47
|
+
blob_name = get_blob_name(@path, file)
|
48
|
+
|
49
|
+
measure = Benchmark.measure do
|
50
|
+
_blob, content = @client.get_blob(@container, blob_name)
|
51
|
+
File.open(filename, "wb") { |f| f.write(content) }
|
52
|
+
end
|
53
|
+
rescue => e
|
54
|
+
raise_error(e)
|
55
|
+
end
|
56
|
+
GoodData.gd_logger.info("Done downloading file type=blobStorage status=finished duration=#{measure.real}")
|
57
|
+
filename
|
58
|
+
end
|
59
|
+
|
60
|
+
def connect
|
61
|
+
GoodData.logger.info "Setting up connection to Blob Storage"
|
62
|
+
if use_sas
|
63
|
+
@client = Azure::Storage::Blob::BlobService.create(:storage_blob_host => @host, :storage_sas_token => @sas_token)
|
64
|
+
else
|
65
|
+
@client = Azure::Storage::Blob::BlobService.create_from_connection_string(@connection_string)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def build_sas(url)
|
70
|
+
matches = url.scan(SAS_URL_PATTERN)
|
71
|
+
return unless matches && matches[0]
|
72
|
+
|
73
|
+
@use_sas = true
|
74
|
+
@host = matches[0][0]
|
75
|
+
@sas_token = matches[0][1]
|
76
|
+
end
|
77
|
+
|
78
|
+
def raise_error(e)
|
79
|
+
if e.message && e.message.include?(INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE)
|
80
|
+
raise INVALID_BLOB_EXPIRED_MESSAGE
|
81
|
+
elsif e.message && e.message.include?(INVALID_BLOB_SIG_WELL_FORMED_MESSAGE)
|
82
|
+
raise INVALID_BLOB_SIG_WELL_FORMED_MESSAGE
|
83
|
+
elsif e.message && e.message.include?(INVALID_BLOB_CONTAINER_MESSAGE)
|
84
|
+
raise INVALID_BLOB_CONTAINER_FORMED_MESSAGE
|
85
|
+
elsif e.message && e.message.include?(INVALID_BLOB_PATH_MESSAGE)
|
86
|
+
raise INVALID_BLOB_INVALID_PATH_MESSAGE
|
87
|
+
else
|
88
|
+
raise INVALID_BLOB_GENERAL_MESSAGE
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def get_blob_name(path, file)
|
93
|
+
return file unless path
|
94
|
+
|
95
|
+
path.rindex('/') == path.length - 1 ? "#{path}#{file}" : "#{path}/#{file}"
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
File without changes
|
@@ -0,0 +1,122 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'java'
|
10
|
+
require 'pathname'
|
11
|
+
require_relative '../cloud_resource_client'
|
12
|
+
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
15
|
+
require file unless file.start_with?('lcm-mssql-driver')
|
16
|
+
end
|
17
|
+
|
18
|
+
module GoodData
|
19
|
+
module CloudResources
|
20
|
+
class MSSQLClient < CloudResourceClient
|
21
|
+
MSSQL_SEPARATOR_PARAM = ";"
|
22
|
+
MSSQL_URL_PATTERN = %r{jdbc:sqlserver://([^:]+)(:([0-9]+))?(/)?}
|
23
|
+
MSSQL_DEFAULT_PORT = 1433
|
24
|
+
LOGIN_TIME_OUT = 30
|
25
|
+
MSSQL_FETCH_SIZE = 10_000
|
26
|
+
VERIFY_FULL = 'verify-full'
|
27
|
+
PREFER = 'prefer'
|
28
|
+
REQUIRE = 'require'
|
29
|
+
|
30
|
+
class << self
|
31
|
+
def accept?(type)
|
32
|
+
type == 'mssql'
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def initialize(options = {})
|
37
|
+
raise("Data Source needs a client to MSSQL to be able to query the storage but 'mssql_client' is empty.") unless options['mssql_client']
|
38
|
+
|
39
|
+
connection = options['mssql_client']['connection']
|
40
|
+
if connection.is_a?(Hash)
|
41
|
+
@database = connection['database']
|
42
|
+
@schema = connection['schema']
|
43
|
+
@authentication = connection['authentication']
|
44
|
+
@ssl_mode = connection['sslMode']
|
45
|
+
@url = connection['url']
|
46
|
+
|
47
|
+
validate
|
48
|
+
else
|
49
|
+
raise('Missing connection info for MSSQL client')
|
50
|
+
end
|
51
|
+
|
52
|
+
Java.com.microsoft.sqlserver.jdbc.SQLServerDriver
|
53
|
+
end
|
54
|
+
|
55
|
+
def realize_query(query, _params)
|
56
|
+
GoodData.gd_logger.info("Realize SQL query: type=mssql status=started")
|
57
|
+
|
58
|
+
connect
|
59
|
+
|
60
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
61
|
+
measure = Benchmark.measure do
|
62
|
+
statement = @connection.create_statement
|
63
|
+
statement.set_fetch_size(MSSQL_FETCH_SIZE)
|
64
|
+
has_result = statement.execute(query)
|
65
|
+
if has_result
|
66
|
+
result = statement.get_result_set
|
67
|
+
metadata = result.get_meta_data
|
68
|
+
col_count = metadata.column_count
|
69
|
+
CSV.open(filename, 'wb') do |csv|
|
70
|
+
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
71
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
GoodData.gd_logger.info("Realize SQL query: type=mssql status=finished duration=#{measure.real}")
|
77
|
+
filename
|
78
|
+
ensure
|
79
|
+
@connection&.close
|
80
|
+
@connection = nil
|
81
|
+
end
|
82
|
+
|
83
|
+
def connect
|
84
|
+
connection_string = build_connection_string
|
85
|
+
GoodData.logger.info "Setting up connection to MSSQL #{connection_string}"
|
86
|
+
|
87
|
+
authentication = @authentication['basic'] || @authentication['activeDirectoryPassword']
|
88
|
+
|
89
|
+
prop = java.util.Properties.new
|
90
|
+
prop.setProperty('userName', authentication['userName'])
|
91
|
+
prop.setProperty('password', authentication['password'])
|
92
|
+
|
93
|
+
@connection = java.sql.DriverManager.getConnection(connection_string, prop)
|
94
|
+
end
|
95
|
+
|
96
|
+
def validate
|
97
|
+
raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
|
98
|
+
|
99
|
+
raise "Instance name is not supported" if @url !~ /^[^\\]*$/
|
100
|
+
|
101
|
+
raise "The connection url is invalid. Parameter is not supported." if @url.include? MSSQL_SEPARATOR_PARAM
|
102
|
+
|
103
|
+
url_matches = @url.scan(MSSQL_URL_PATTERN)
|
104
|
+
raise "Cannot reach the url" if url_matches.nil? || url_matches.length.zero?
|
105
|
+
|
106
|
+
raise "The authentication method is not supported." unless @authentication['basic'] || @authentication['activeDirectoryPassword']
|
107
|
+
end
|
108
|
+
|
109
|
+
def build_connection_string
|
110
|
+
encrypt = @ssl_mode != PREFER
|
111
|
+
trust_server_certificate = @ssl_mode == REQUIRE
|
112
|
+
|
113
|
+
"#{@url};" \
|
114
|
+
"database=#{@database};" \
|
115
|
+
"encrypt=#{encrypt};" \
|
116
|
+
"trustServerCertificate=#{trust_server_certificate};" \
|
117
|
+
"loginTimeout=#{LOGIN_TIME_OUT};" \
|
118
|
+
"#{'authentication=ActiveDirectoryPassword;' if @authentication['activeDirectoryPassword']}"
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
File without changes
|
@@ -0,0 +1,121 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'java'
|
10
|
+
require 'pathname'
|
11
|
+
require_relative '../cloud_resource_client'
|
12
|
+
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
15
|
+
require file unless file.start_with?('lcm-mysql-driver')
|
16
|
+
end
|
17
|
+
|
18
|
+
module GoodData
|
19
|
+
module CloudResources
|
20
|
+
class MysqlClient < CloudResourceClient
|
21
|
+
JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
|
22
|
+
MYSQL_DEFAULT_PORT = 3306
|
23
|
+
JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://'
|
24
|
+
VERIFY_FULL = '&useSSL=true&verifyServerCertificate=true'
|
25
|
+
PREFER = '&useSSL=true&requireSSL=false&verifyServerCertificate=false'
|
26
|
+
REQUIRE = '&useSSL=true&requireSSL=true&verifyServerCertificate=false'
|
27
|
+
MYSQL_FETCH_SIZE = 1000
|
28
|
+
MONGO_BI_FETCH_SIZE = 1
|
29
|
+
MONGO_BI_TYPE = 'MongoDBConnector'
|
30
|
+
|
31
|
+
class << self
|
32
|
+
def accept?(type)
|
33
|
+
type == 'mysql'
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def initialize(options = {})
|
38
|
+
raise("Data Source needs a client to Mysql to be able to query the storage but 'mysql_client' is empty.") unless options['mysql_client']
|
39
|
+
|
40
|
+
if options['mysql_client']['connection'].is_a?(Hash)
|
41
|
+
@database = options['mysql_client']['connection']['database']
|
42
|
+
@authentication = options['mysql_client']['connection']['authentication']
|
43
|
+
@ssl_mode = options['mysql_client']['connection']['sslMode']
|
44
|
+
@database_type = options['mysql_client']['connection']['databaseType']
|
45
|
+
raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
|
46
|
+
|
47
|
+
@url = build_url(options['mysql_client']['connection']['url'])
|
48
|
+
else
|
49
|
+
raise('Missing connection info for Mysql client')
|
50
|
+
end
|
51
|
+
|
52
|
+
Java.com.mysql.jdbc.Driver
|
53
|
+
end
|
54
|
+
|
55
|
+
def realize_query(query, _params)
|
56
|
+
GoodData.gd_logger.info("Realize SQL query: type=mysql status=started")
|
57
|
+
|
58
|
+
connect
|
59
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
60
|
+
measure = Benchmark.measure do
|
61
|
+
statement = @connection.create_statement
|
62
|
+
statement.set_fetch_size(fetch_size)
|
63
|
+
has_result = statement.execute(query)
|
64
|
+
if has_result
|
65
|
+
result = statement.get_result_set
|
66
|
+
metadata = result.get_meta_data
|
67
|
+
col_count = metadata.column_count
|
68
|
+
CSV.open(filename, 'wb') do |csv|
|
69
|
+
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
70
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
GoodData.gd_logger.info("Realize SQL query: type=mysql status=finished duration=#{measure.real}")
|
75
|
+
filename
|
76
|
+
ensure
|
77
|
+
@connection&.close
|
78
|
+
@connection = nil
|
79
|
+
end
|
80
|
+
|
81
|
+
def connect
|
82
|
+
GoodData.logger.info "Setting up connection to Mysql #{@database_type} #{@url} "
|
83
|
+
|
84
|
+
prop = java.util.Properties.new
|
85
|
+
prop.setProperty('user', @authentication['basic']['userName'])
|
86
|
+
prop.setProperty('password', @authentication['basic']['password'])
|
87
|
+
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
88
|
+
@connection.set_auto_commit(false)
|
89
|
+
end
|
90
|
+
|
91
|
+
def build_url(url)
|
92
|
+
matches = url.scan(JDBC_MYSQL_PATTERN)
|
93
|
+
raise 'Cannot reach the url' unless matches
|
94
|
+
|
95
|
+
host = matches[0][0]
|
96
|
+
port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT
|
97
|
+
|
98
|
+
"#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?#{get_ssl_mode(@ssl_mode)}#{add_extended}&useCursorFetch=true&enabledTLSProtocols=TLSv1.2"
|
99
|
+
end
|
100
|
+
|
101
|
+
def fetch_size
|
102
|
+
@database_type == MONGO_BI_TYPE ? MONGO_BI_FETCH_SIZE : MYSQL_FETCH_SIZE
|
103
|
+
end
|
104
|
+
|
105
|
+
def get_ssl_mode(ssl_mode)
|
106
|
+
mode = PREFER
|
107
|
+
if ssl_mode == 'verify-full'
|
108
|
+
mode = VERIFY_FULL
|
109
|
+
elsif ssl_mode == 'require'
|
110
|
+
mode = REQUIRE
|
111
|
+
end
|
112
|
+
|
113
|
+
mode
|
114
|
+
end
|
115
|
+
|
116
|
+
def add_extended
|
117
|
+
@database_type == MONGO_BI_TYPE ? '&authenticationPlugins=org.mongodb.mongosql.auth.plugin.MongoSqlAuthenticationPlugin&useLocalTransactionState=true' : ''
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
@@ -98,7 +98,6 @@ module GoodData
|
|
98
98
|
|
99
99
|
host = matches[0][0]
|
100
100
|
port = matches[0][2]&.to_i || POSTGRES_DEFAULT_PORT
|
101
|
-
raise "Custom port #{port} is not supported. Remove it or use the default port '5432'" if POSTGRES_DEFAULT_PORT != port
|
102
101
|
|
103
102
|
"#{JDBC_POSTGRES_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{@ssl_mode}#{VERIFY_FULL == @ssl_mode ? SSL_JAVA_FACTORY : ''}"
|
104
103
|
end
|
File without changes
|
@@ -39,8 +39,6 @@ module GoodData
|
|
39
39
|
@debug = options['debug'] == true || options['debug'] == 'true'
|
40
40
|
|
41
41
|
Java.com.amazon.redshift.jdbc42.Driver
|
42
|
-
base = Pathname(__FILE__).dirname
|
43
|
-
org.apache.log4j.PropertyConfigurator.configure("#{base}/drivers/log4j.properties")
|
44
42
|
end
|
45
43
|
|
46
44
|
def realize_query(query, _params)
|
@@ -18,6 +18,9 @@ end
|
|
18
18
|
module GoodData
|
19
19
|
module CloudResources
|
20
20
|
class SnowflakeClient < CloudResourceClient
|
21
|
+
SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=gooddata_platform'
|
22
|
+
SNOWFLAKE_SEPARATOR_PARAM = '?'
|
23
|
+
|
21
24
|
class << self
|
22
25
|
def accept?(type)
|
23
26
|
type == 'snowflake'
|
@@ -31,7 +34,7 @@ module GoodData
|
|
31
34
|
@database = options['snowflake_client']['connection']['database']
|
32
35
|
@schema = options['snowflake_client']['connection']['schema'] || 'public'
|
33
36
|
@warehouse = options['snowflake_client']['connection']['warehouse']
|
34
|
-
@url = options['snowflake_client']['connection']['url']
|
37
|
+
@url = build_url(options['snowflake_client']['connection']['url'])
|
35
38
|
@authentication = options['snowflake_client']['connection']['authentication']
|
36
39
|
else
|
37
40
|
raise('Missing connection info for Snowflake client')
|
@@ -79,6 +82,20 @@ module GoodData
|
|
79
82
|
|
80
83
|
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
81
84
|
end
|
85
|
+
|
86
|
+
def build_url(url)
|
87
|
+
is_contain = url.include?(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
|
88
|
+
unless is_contain
|
89
|
+
if url.include?(SNOWFLAKE_SEPARATOR_PARAM)
|
90
|
+
url.concat("&")
|
91
|
+
else
|
92
|
+
url.concat(SNOWFLAKE_SEPARATOR_PARAM)
|
93
|
+
end
|
94
|
+
url.concat(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
|
95
|
+
end
|
96
|
+
|
97
|
+
url
|
98
|
+
end
|
82
99
|
end
|
83
100
|
end
|
84
101
|
end
|
@@ -44,10 +44,14 @@ module GoodData
|
|
44
44
|
realize_link
|
45
45
|
when 's3'
|
46
46
|
realize_s3(params)
|
47
|
-
when 'redshift', 'snowflake', 'bigquery', 'postgresql'
|
47
|
+
when 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql'
|
48
48
|
raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
|
49
49
|
require_relative '../cloud_resources/cloud_resources'
|
50
50
|
realize_cloud_resource(source, params)
|
51
|
+
when 'blobStorage'
|
52
|
+
require_relative '../cloud_resources/blobstorage/blobstorage_client'
|
53
|
+
blob_storage_client = GoodData::BlobStorageClient.new(params)
|
54
|
+
blob_storage_client.realize_blob(@options[:file], params)
|
51
55
|
else
|
52
56
|
raise "DataSource does not support type \"#{source}\""
|
53
57
|
end
|
@@ -111,12 +115,13 @@ module GoodData
|
|
111
115
|
end
|
112
116
|
|
113
117
|
def realize_s3(params)
|
114
|
-
s3_client = params['
|
118
|
+
s3_client = params['s3_client'] && params['s3_client']['client']
|
115
119
|
raise 'AWS client not present. Perhaps S3Middleware is missing in the brick definition?' if !s3_client || !s3_client.respond_to?(:bucket)
|
116
120
|
bucket_name = @options[:bucket]
|
117
|
-
key = @options[:key]
|
121
|
+
key = @options[:key].present? ? @options[:key] : @options[:file]
|
118
122
|
raise 'Key "bucket" is missing in S3 datasource' if bucket_name.blank?
|
119
|
-
raise 'Key "key" is missing in S3 datasource' if key.blank?
|
123
|
+
raise 'Key "key" or "file" is missing in S3 datasource' if key.blank?
|
124
|
+
|
120
125
|
GoodData.logger.info("Realizing download from S3. Bucket #{bucket_name}, object with key #{key}.")
|
121
126
|
filename = Digest::SHA256.new.hexdigest(@options.to_json)
|
122
127
|
bucket = s3_client.bucket(bucket_name)
|
@@ -18,6 +18,11 @@ module GoodData
|
|
18
18
|
class << self
|
19
19
|
include Dsl::Dsl
|
20
20
|
|
21
|
+
SYNC_FAILED_LIST = 'sync_failed_list'.to_sym
|
22
|
+
FAILED_PROJECTS = 'failed_projects'.to_sym
|
23
|
+
FAILED_CLIENTS = 'failed_clients'.to_sym
|
24
|
+
FAILED_SEGMENTS = 'failed_segments'.to_sym
|
25
|
+
|
21
26
|
def check_params(specification, params)
|
22
27
|
Helpers.check_params(specification, params)
|
23
28
|
end
|
@@ -31,6 +36,158 @@ module GoodData
|
|
31
36
|
params.setup_filters(specification) # enables params validation
|
32
37
|
result
|
33
38
|
end
|
39
|
+
|
40
|
+
def print_result(_params)
|
41
|
+
true
|
42
|
+
end
|
43
|
+
|
44
|
+
def continue_on_error(params)
|
45
|
+
Helpers.continue_on_error(params)
|
46
|
+
end
|
47
|
+
|
48
|
+
def collect_synced_status(params)
|
49
|
+
Helpers.collect_synced_status(params)
|
50
|
+
end
|
51
|
+
|
52
|
+
def add_failed_project(project_id, message, failed_action, params)
|
53
|
+
if collect_synced_status(params) && !sync_failed_project(project_id, params)
|
54
|
+
sync_failed_list = sync_failed_list(params)
|
55
|
+
project_client_mappings = sync_failed_list[:project_client_mappings]
|
56
|
+
project_client_mapping = project_client_mappings ? project_client_mappings[project_id.to_sym] : nil
|
57
|
+
client_id = project_client_mapping ? project_client_mapping[:client_id] : nil
|
58
|
+
segment_id = project_client_mapping ? project_client_mapping[:segment_id] : nil
|
59
|
+
|
60
|
+
failed_detailed_project = {
|
61
|
+
project_id: project_id,
|
62
|
+
client_id: client_id,
|
63
|
+
segment: segment_id,
|
64
|
+
message: message,
|
65
|
+
action: failed_action
|
66
|
+
}
|
67
|
+
add_failed_detail(params, failed_detailed_project, sync_failed_list)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def add_failed_client(client_id, message, error_action, params)
|
72
|
+
if collect_synced_status(params) && !sync_failed_client(client_id, params)
|
73
|
+
sync_failed_list = sync_failed_list(params)
|
74
|
+
client_project_mappings = sync_failed_list[:client_project_mappings]
|
75
|
+
client_project_mapping = client_project_mappings ? client_project_mappings[client_id.to_sym] : nil
|
76
|
+
project_id = client_project_mapping ? client_project_mapping[:project_id] : nil
|
77
|
+
segment_id = client_project_mapping ? client_project_mapping[:segment_id] : nil
|
78
|
+
|
79
|
+
failed_detailed_client = {
|
80
|
+
project_id: project_id,
|
81
|
+
client_id: client_id,
|
82
|
+
segment: segment_id,
|
83
|
+
message: message,
|
84
|
+
action: error_action
|
85
|
+
}
|
86
|
+
add_failed_detail(params, failed_detailed_client, sync_failed_list)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def add_failed_segment(segment_id, message, error_action, params)
|
91
|
+
if collect_synced_status(params) && !sync_failed_segment(segment_id, params)
|
92
|
+
sync_failed_list = sync_failed_list(params)
|
93
|
+
failed_detailed_segment = {
|
94
|
+
project_id: nil,
|
95
|
+
client_id: nil,
|
96
|
+
segment: segment_id,
|
97
|
+
message: message,
|
98
|
+
action: error_action
|
99
|
+
}
|
100
|
+
add_failed_detail(params, failed_detailed_segment, sync_failed_list, true)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
# Add new clients to project client mapping
|
105
|
+
#
|
106
|
+
# @param [String] project_id project identify will be added to mapping
|
107
|
+
# @param [String] client_id client identify will be added to mapping
|
108
|
+
# @param [String] segment_id segment identify will be added to mapping
|
109
|
+
# @param [Hash] params the hash contains list of parameters and values
|
110
|
+
def add_new_clients_to_project_client_mapping(project_id, client_id, segment_id, params)
|
111
|
+
if collect_synced_status(params)
|
112
|
+
sync_failed_list = sync_failed_list(params)
|
113
|
+
client_project_mappings = sync_failed_list[:client_project_mappings]
|
114
|
+
project_client_mappings = sync_failed_list[:project_client_mappings]
|
115
|
+
client_project_mappings[client_id.to_sym] = {
|
116
|
+
project_id: project_id,
|
117
|
+
segment_id: segment_id
|
118
|
+
} if client_project_mappings
|
119
|
+
|
120
|
+
project_client_mappings[project_id.to_sym] = {
|
121
|
+
client_id: client_id,
|
122
|
+
segment_id: segment_id
|
123
|
+
} if project_client_mappings
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def process_failed_project(project_id, failed_message, failed_projects, continue_on_error)
|
128
|
+
fail(failed_message) unless continue_on_error
|
129
|
+
|
130
|
+
failed_projects << {
|
131
|
+
project_id: project_id,
|
132
|
+
message: failed_message
|
133
|
+
}
|
134
|
+
end
|
135
|
+
|
136
|
+
def process_failed_projects(failed_projects, failed_action, params)
|
137
|
+
failed_projects.each do |failed_project|
|
138
|
+
add_failed_project(failed_project[:project_id], failed_project[:message], failed_action, params)
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def sync_failed_project(project_id, params)
|
143
|
+
collect_synced_status(params) && params[SYNC_FAILED_LIST][FAILED_PROJECTS].include?(project_id)
|
144
|
+
end
|
145
|
+
|
146
|
+
def sync_failed_client(client_id, params)
|
147
|
+
collect_synced_status(params) && params[SYNC_FAILED_LIST][FAILED_CLIENTS].include?(client_id)
|
148
|
+
end
|
149
|
+
|
150
|
+
def sync_failed_segment(segment_id, params)
|
151
|
+
collect_synced_status(params) && params[SYNC_FAILED_LIST][FAILED_SEGMENTS].include?(segment_id)
|
152
|
+
end
|
153
|
+
|
154
|
+
private
|
155
|
+
|
156
|
+
def add_failed_detail(params, failed_detailed_project, sync_failed_list, ignore_segment = false)
|
157
|
+
params.gdc_logger&.warn failed_detailed_project[:message]
|
158
|
+
sync_failed_list[:failed_detailed_projects] << failed_detailed_project
|
159
|
+
|
160
|
+
if ignore_segment
|
161
|
+
add_failed_detail_segment(failed_detailed_project[:segment_id], sync_failed_list)
|
162
|
+
else
|
163
|
+
add_failed_detail_client(failed_detailed_project[:client_id], failed_detailed_project[:project_id], sync_failed_list)
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
def sync_failed_list(params)
|
168
|
+
if params.include?(SYNC_FAILED_LIST)
|
169
|
+
params[SYNC_FAILED_LIST]
|
170
|
+
else
|
171
|
+
nil
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
def add_failed_detail_client(client_id, project_id, sync_failed_list)
|
176
|
+
sync_failed_list[FAILED_CLIENTS] << client_id if client_id
|
177
|
+
|
178
|
+
sync_failed_list[FAILED_PROJECTS] << project_id if project_id
|
179
|
+
end
|
180
|
+
|
181
|
+
def add_failed_detail_segment(segment_id, sync_failed_list)
|
182
|
+
if segment_id
|
183
|
+
sync_failed_list[FAILED_SEGMENTS] << segment_id
|
184
|
+
|
185
|
+
client_project_mappings = sync_failed_list[:client_project_mappings]
|
186
|
+
client_project_mappings.each do |client_id, client_project_mapping|
|
187
|
+
add_failed_detail_client(client_id, client_project_mapping[:project_id], sync_failed_list) if client_project_mapping[:segment_id] == segment_id
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
34
191
|
end
|
35
192
|
end
|
36
193
|
end
|
@@ -47,8 +47,9 @@ module GoodData
|
|
47
47
|
end
|
48
48
|
|
49
49
|
begin
|
50
|
+
params.gdc_logger.info "Starting to find DataProduct ID: #{data_product_id}"
|
50
51
|
data_product = domain.data_products(data_product_id)
|
51
|
-
rescue RestClient::BadRequest
|
52
|
+
rescue RestClient::BadRequest, RestClient::NotFound
|
52
53
|
params.gdc_logger.info "Can not find DataProduct #{params.data_product}, creating it instead"
|
53
54
|
data_product = domain.create_data_product(id: params.data_product)
|
54
55
|
end
|
@@ -53,7 +53,9 @@ module GoodData
|
|
53
53
|
client: development_client
|
54
54
|
)
|
55
55
|
kpi_dashboards = MdObject.query('analyticalDashboard', MdObject, client: development_client, project: from_project)
|
56
|
-
|
56
|
+
kpi_dashboard_plugin = MdObject.query('dashboardPlugin', MdObject, client: development_client, project: from_project)
|
57
|
+
kpi_date_filter_config = MdObject.query('dateFilterConfig', MdObject, client: development_client, project: from_project)
|
58
|
+
objects = old_dashboards.to_a + kpi_dashboards.to_a + kpi_dashboard_plugin.to_a + kpi_date_filter_config.to_a
|
57
59
|
else
|
58
60
|
objects = GoodData::Dashboard.find_by_tag(
|
59
61
|
production_tags,
|