gooddata 2.1.19-java → 2.2.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gdc-ii-config.yaml +1 -1
- data/.github/workflows/build.yml +66 -0
- data/.github/workflows/pre-merge.yml +72 -0
- data/CHANGELOG.md +38 -0
- data/Dockerfile +21 -14
- data/Dockerfile.jruby +1 -11
- data/README.md +1 -2
- data/SDK_VERSION +1 -1
- data/VERSION +1 -1
- data/ci/mssql/pom.xml +62 -0
- data/ci/mysql/pom.xml +57 -0
- data/ci/redshift/pom.xml +1 -1
- data/docker-compose.lcm.yml +0 -3
- data/gooddata.gemspec +2 -1
- data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
- data/lcm.rake +2 -8
- data/lib/gooddata/bricks/middleware/aws_middleware.rb +35 -9
- data/lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb +98 -0
- data/lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mssql/mssql_client.rb +122 -0
- data/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +111 -0
- data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +0 -1
- data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +18 -1
- data/lib/gooddata/helpers/data_helper.rb +9 -4
- data/lib/gooddata/lcm/actions/collect_meta.rb +3 -1
- data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +3 -2
- data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -7
- data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +64 -0
- data/lib/gooddata/lcm/actions/synchronize_ldm.rb +19 -8
- data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +12 -9
- data/lib/gooddata/lcm/actions/update_metric_formats.rb +185 -0
- data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb +5 -0
- data/lib/gooddata/lcm/helpers/release_table_helper.rb +42 -8
- data/lib/gooddata/lcm/lcm2.rb +5 -0
- data/lib/gooddata/mixins/md_object_query.rb +1 -0
- data/lib/gooddata/models/data_source.rb +5 -1
- data/lib/gooddata/models/dataset_mapping.rb +36 -0
- data/lib/gooddata/models/metadata/label.rb +26 -27
- data/lib/gooddata/models/project.rb +34 -9
- data/lib/gooddata/models/schedule.rb +13 -1
- data/lib/gooddata/models/user_filters/user_filter_builder.rb +58 -53
- data/lib/gooddata/rest/phmap.rb +1 -0
- metadata +44 -18
- data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb +0 -37
@@ -0,0 +1,98 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'pathname'
|
10
|
+
require "azure/storage/blob"
|
11
|
+
|
12
|
+
module GoodData
|
13
|
+
class BlobStorageClient
|
14
|
+
SAS_URL_PATTERN = %r{(^https?:\/\/[^\/]*)\/.*\?(.*)}
|
15
|
+
INVALID_BLOB_GENERAL_MESSAGE = "The connection string is not valid."
|
16
|
+
INVALID_BLOB_SIG_WELL_FORMED_MESSAGE = "The signature format is not valid."
|
17
|
+
INVALID_BLOB_CONTAINER_MESSAGE = "ContainerNotFound"
|
18
|
+
INVALID_BLOB_CONTAINER_FORMED_MESSAGE = "The container with the specified name is not found."
|
19
|
+
INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE = "Signature not valid in the specified time frame"
|
20
|
+
INVALID_BLOB_EXPIRED_MESSAGE = "The signature expired."
|
21
|
+
INVALID_BLOB_INVALID_CONNECTION_STRING_MESSAGE = "The connection string is not valid."
|
22
|
+
INVALID_BLOB_PATH_MESSAGE = "BlobNotFound"
|
23
|
+
INVALID_BLOB_INVALID_PATH_MESSAGE = "The path to the data is not found."
|
24
|
+
|
25
|
+
attr_reader :use_sas
|
26
|
+
|
27
|
+
def initialize(options = {})
|
28
|
+
raise("Data Source needs a client to Blob Storage to be able to get blob file but 'blobStorage_client' is empty.") unless options['blobStorage_client']
|
29
|
+
|
30
|
+
if options['blobStorage_client']['connectionString'] && options['blobStorage_client']['container']
|
31
|
+
@connection_string = options['blobStorage_client']['connectionString']
|
32
|
+
@container = options['blobStorage_client']['container']
|
33
|
+
@path = options['blobStorage_client']['path']
|
34
|
+
@use_sas = false
|
35
|
+
build_sas(@connection_string)
|
36
|
+
else
|
37
|
+
raise('Missing connection info for Blob Storage client')
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def realize_blob(file, _params)
|
42
|
+
GoodData.gd_logger.info("Realizing download from Blob Storage. Container #{@container}.")
|
43
|
+
filename = ''
|
44
|
+
begin
|
45
|
+
connect
|
46
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
47
|
+
blob_name = get_blob_name(@path, file)
|
48
|
+
|
49
|
+
measure = Benchmark.measure do
|
50
|
+
_blob, content = @client.get_blob(@container, blob_name)
|
51
|
+
File.open(filename, "wb") { |f| f.write(content) }
|
52
|
+
end
|
53
|
+
rescue => e
|
54
|
+
raise_error(e)
|
55
|
+
end
|
56
|
+
GoodData.gd_logger.info("Done downloading file type=blobStorage status=finished duration=#{measure.real}")
|
57
|
+
filename
|
58
|
+
end
|
59
|
+
|
60
|
+
def connect
|
61
|
+
GoodData.logger.info "Setting up connection to Blob Storage"
|
62
|
+
if use_sas
|
63
|
+
@client = Azure::Storage::Blob::BlobService.create(:storage_blob_host => @host, :storage_sas_token => @sas_token)
|
64
|
+
else
|
65
|
+
@client = Azure::Storage::Blob::BlobService.create_from_connection_string(@connection_string)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def build_sas(url)
|
70
|
+
matches = url.scan(SAS_URL_PATTERN)
|
71
|
+
return unless matches && matches[0]
|
72
|
+
|
73
|
+
@use_sas = true
|
74
|
+
@host = matches[0][0]
|
75
|
+
@sas_token = matches[0][1]
|
76
|
+
end
|
77
|
+
|
78
|
+
def raise_error(e)
|
79
|
+
if e.message && e.message.include?(INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE)
|
80
|
+
raise INVALID_BLOB_EXPIRED_MESSAGE
|
81
|
+
elsif e.message && e.message.include?(INVALID_BLOB_SIG_WELL_FORMED_MESSAGE)
|
82
|
+
raise INVALID_BLOB_SIG_WELL_FORMED_MESSAGE
|
83
|
+
elsif e.message && e.message.include?(INVALID_BLOB_CONTAINER_MESSAGE)
|
84
|
+
raise INVALID_BLOB_CONTAINER_FORMED_MESSAGE
|
85
|
+
elsif e.message && e.message.include?(INVALID_BLOB_PATH_MESSAGE)
|
86
|
+
raise INVALID_BLOB_INVALID_PATH_MESSAGE
|
87
|
+
else
|
88
|
+
raise INVALID_BLOB_GENERAL_MESSAGE
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def get_blob_name(path, file)
|
93
|
+
return file unless path
|
94
|
+
|
95
|
+
path.rindex('/') == path.length - 1 ? "#{path}#{file}" : "#{path}/#{file}"
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
File without changes
|
@@ -0,0 +1,122 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'java'
|
10
|
+
require 'pathname'
|
11
|
+
require_relative '../cloud_resource_client'
|
12
|
+
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
15
|
+
require file unless file.start_with?('lcm-mssql-driver')
|
16
|
+
end
|
17
|
+
|
18
|
+
module GoodData
|
19
|
+
module CloudResources
|
20
|
+
class MSSQLClient < CloudResourceClient
|
21
|
+
MSSQL_SEPARATOR_PARAM = ";"
|
22
|
+
MSSQL_URL_PATTERN = %r{jdbc:sqlserver://([^:]+)(:([0-9]+))?(/)?}
|
23
|
+
MSSQL_DEFAULT_PORT = 1433
|
24
|
+
LOGIN_TIME_OUT = 30
|
25
|
+
MSSQL_FETCH_SIZE = 10_000
|
26
|
+
VERIFY_FULL = 'verify-full'
|
27
|
+
PREFER = 'prefer'
|
28
|
+
REQUIRE = 'require'
|
29
|
+
|
30
|
+
class << self
|
31
|
+
def accept?(type)
|
32
|
+
type == 'mssql'
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def initialize(options = {})
|
37
|
+
raise("Data Source needs a client to MSSQL to be able to query the storage but 'mssql_client' is empty.") unless options['mssql_client']
|
38
|
+
|
39
|
+
connection = options['mssql_client']['connection']
|
40
|
+
if connection.is_a?(Hash)
|
41
|
+
@database = connection['database']
|
42
|
+
@schema = connection['schema']
|
43
|
+
@authentication = connection['authentication']
|
44
|
+
@ssl_mode = connection['sslMode']
|
45
|
+
@url = connection['url']
|
46
|
+
|
47
|
+
validate
|
48
|
+
else
|
49
|
+
raise('Missing connection info for MSSQL client')
|
50
|
+
end
|
51
|
+
|
52
|
+
Java.com.microsoft.sqlserver.jdbc.SQLServerDriver
|
53
|
+
end
|
54
|
+
|
55
|
+
def realize_query(query, _params)
|
56
|
+
GoodData.gd_logger.info("Realize SQL query: type=mssql status=started")
|
57
|
+
|
58
|
+
connect
|
59
|
+
|
60
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
61
|
+
measure = Benchmark.measure do
|
62
|
+
statement = @connection.create_statement
|
63
|
+
statement.set_fetch_size(MSSQL_FETCH_SIZE)
|
64
|
+
has_result = statement.execute(query)
|
65
|
+
if has_result
|
66
|
+
result = statement.get_result_set
|
67
|
+
metadata = result.get_meta_data
|
68
|
+
col_count = metadata.column_count
|
69
|
+
CSV.open(filename, 'wb') do |csv|
|
70
|
+
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
71
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
GoodData.gd_logger.info("Realize SQL query: type=mssql status=finished duration=#{measure.real}")
|
77
|
+
filename
|
78
|
+
ensure
|
79
|
+
@connection&.close
|
80
|
+
@connection = nil
|
81
|
+
end
|
82
|
+
|
83
|
+
def connect
|
84
|
+
connection_string = build_connection_string
|
85
|
+
GoodData.logger.info "Setting up connection to MSSQL #{connection_string}"
|
86
|
+
|
87
|
+
authentication = @authentication['basic'] || @authentication['activeDirectoryPassword']
|
88
|
+
|
89
|
+
prop = java.util.Properties.new
|
90
|
+
prop.setProperty('userName', authentication['userName'])
|
91
|
+
prop.setProperty('password', authentication['password'])
|
92
|
+
|
93
|
+
@connection = java.sql.DriverManager.getConnection(connection_string, prop)
|
94
|
+
end
|
95
|
+
|
96
|
+
def validate
|
97
|
+
raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
|
98
|
+
|
99
|
+
raise "Instance name is not supported" if @url !~ /^[^\\]*$/
|
100
|
+
|
101
|
+
raise "The connection url is invalid. Parameter is not supported." if @url.include? MSSQL_SEPARATOR_PARAM
|
102
|
+
|
103
|
+
url_matches = @url.scan(MSSQL_URL_PATTERN)
|
104
|
+
raise "Cannot reach the url" if url_matches.nil? || url_matches.length.zero?
|
105
|
+
|
106
|
+
raise "The authentication method is not supported." unless @authentication['basic'] || @authentication['activeDirectoryPassword']
|
107
|
+
end
|
108
|
+
|
109
|
+
def build_connection_string
|
110
|
+
encrypt = @ssl_mode != PREFER
|
111
|
+
trust_server_certificate = @ssl_mode == REQUIRE
|
112
|
+
|
113
|
+
"#{@url};" \
|
114
|
+
"database=#{@database};" \
|
115
|
+
"encrypt=#{encrypt};" \
|
116
|
+
"trustServerCertificate=#{trust_server_certificate};" \
|
117
|
+
"loginTimeout=#{LOGIN_TIME_OUT};" \
|
118
|
+
"#{'authentication=ActiveDirectoryPassword;' if @authentication['activeDirectoryPassword']}"
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
File without changes
|
@@ -0,0 +1,111 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'java'
|
10
|
+
require 'pathname'
|
11
|
+
require_relative '../cloud_resource_client'
|
12
|
+
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
15
|
+
require file unless file.start_with?('lcm-mysql-driver')
|
16
|
+
end
|
17
|
+
|
18
|
+
module GoodData
|
19
|
+
module CloudResources
|
20
|
+
class MysqlClient < CloudResourceClient
|
21
|
+
JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
|
22
|
+
MYSQL_DEFAULT_PORT = 3306
|
23
|
+
JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://'
|
24
|
+
VERIFY_FULL = 'VERIFY_IDENTITY'
|
25
|
+
PREFER = 'PREFERRED'
|
26
|
+
REQUIRE = 'REQUIRED'
|
27
|
+
MYSQL_FETCH_SIZE = 1000
|
28
|
+
|
29
|
+
class << self
|
30
|
+
def accept?(type)
|
31
|
+
type == 'mysql'
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def initialize(options = {})
|
36
|
+
raise("Data Source needs a client to Mysql to be able to query the storage but 'mysql_client' is empty.") unless options['mysql_client']
|
37
|
+
|
38
|
+
if options['mysql_client']['connection'].is_a?(Hash)
|
39
|
+
@database = options['mysql_client']['connection']['database']
|
40
|
+
@authentication = options['mysql_client']['connection']['authentication']
|
41
|
+
@ssl_mode = options['mysql_client']['connection']['sslMode']
|
42
|
+
raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
|
43
|
+
|
44
|
+
@url = build_url(options['mysql_client']['connection']['url'])
|
45
|
+
else
|
46
|
+
raise('Missing connection info for Mysql client')
|
47
|
+
end
|
48
|
+
|
49
|
+
Java.com.mysql.cj.jdbc.Driver
|
50
|
+
end
|
51
|
+
|
52
|
+
def realize_query(query, _params)
|
53
|
+
GoodData.gd_logger.info("Realize SQL query: type=mysql status=started")
|
54
|
+
|
55
|
+
connect
|
56
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
57
|
+
measure = Benchmark.measure do
|
58
|
+
statement = @connection.create_statement
|
59
|
+
statement.set_fetch_size(MYSQL_FETCH_SIZE)
|
60
|
+
has_result = statement.execute(query)
|
61
|
+
if has_result
|
62
|
+
result = statement.get_result_set
|
63
|
+
metadata = result.get_meta_data
|
64
|
+
col_count = metadata.column_count
|
65
|
+
CSV.open(filename, 'wb') do |csv|
|
66
|
+
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
67
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
GoodData.gd_logger.info("Realize SQL query: type=mysql status=finished duration=#{measure.real}")
|
72
|
+
filename
|
73
|
+
ensure
|
74
|
+
@connection&.close
|
75
|
+
@connection = nil
|
76
|
+
end
|
77
|
+
|
78
|
+
def connect
|
79
|
+
GoodData.logger.info "Setting up connection to Mysql #{@url}"
|
80
|
+
|
81
|
+
prop = java.util.Properties.new
|
82
|
+
prop.setProperty('user', @authentication['basic']['userName'])
|
83
|
+
prop.setProperty('password', @authentication['basic']['password'])
|
84
|
+
|
85
|
+
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
86
|
+
@connection.set_auto_commit(false)
|
87
|
+
end
|
88
|
+
|
89
|
+
def build_url(url)
|
90
|
+
matches = url.scan(JDBC_MYSQL_PATTERN)
|
91
|
+
raise 'Cannot reach the url' unless matches
|
92
|
+
|
93
|
+
host = matches[0][0]
|
94
|
+
port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT
|
95
|
+
|
96
|
+
"#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{get_ssl_mode(@ssl_mode)}&useCursorFetch=true&enabledTLSProtocols=TLSv1.2"
|
97
|
+
end
|
98
|
+
|
99
|
+
def get_ssl_mode(ssl_mode)
|
100
|
+
mode = PREFER
|
101
|
+
if ssl_mode == 'verify-full'
|
102
|
+
mode = VERIFY_FULL
|
103
|
+
elsif ssl_mode == 'require'
|
104
|
+
mode = REQUIRE
|
105
|
+
end
|
106
|
+
|
107
|
+
mode
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
@@ -98,7 +98,6 @@ module GoodData
|
|
98
98
|
|
99
99
|
host = matches[0][0]
|
100
100
|
port = matches[0][2]&.to_i || POSTGRES_DEFAULT_PORT
|
101
|
-
raise "Custom port #{port} is not supported. Remove it or use the default port '5432'" if POSTGRES_DEFAULT_PORT != port
|
102
101
|
|
103
102
|
"#{JDBC_POSTGRES_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{@ssl_mode}#{VERIFY_FULL == @ssl_mode ? SSL_JAVA_FACTORY : ''}"
|
104
103
|
end
|
@@ -18,6 +18,9 @@ end
|
|
18
18
|
module GoodData
|
19
19
|
module CloudResources
|
20
20
|
class SnowflakeClient < CloudResourceClient
|
21
|
+
SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=GoodData_Platform'
|
22
|
+
SNOWFLAKE_SEPARATOR_PARAM = '?'
|
23
|
+
|
21
24
|
class << self
|
22
25
|
def accept?(type)
|
23
26
|
type == 'snowflake'
|
@@ -31,7 +34,7 @@ module GoodData
|
|
31
34
|
@database = options['snowflake_client']['connection']['database']
|
32
35
|
@schema = options['snowflake_client']['connection']['schema'] || 'public'
|
33
36
|
@warehouse = options['snowflake_client']['connection']['warehouse']
|
34
|
-
@url = options['snowflake_client']['connection']['url']
|
37
|
+
@url = build_url(options['snowflake_client']['connection']['url'])
|
35
38
|
@authentication = options['snowflake_client']['connection']['authentication']
|
36
39
|
else
|
37
40
|
raise('Missing connection info for Snowflake client')
|
@@ -79,6 +82,20 @@ module GoodData
|
|
79
82
|
|
80
83
|
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
81
84
|
end
|
85
|
+
|
86
|
+
def build_url(url)
|
87
|
+
is_contain = url.include?(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
|
88
|
+
unless is_contain
|
89
|
+
if url.include?(SNOWFLAKE_SEPARATOR_PARAM)
|
90
|
+
url.concat("&")
|
91
|
+
else
|
92
|
+
url.concat(SNOWFLAKE_SEPARATOR_PARAM)
|
93
|
+
end
|
94
|
+
url.concat(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
|
95
|
+
end
|
96
|
+
|
97
|
+
url
|
98
|
+
end
|
82
99
|
end
|
83
100
|
end
|
84
101
|
end
|
@@ -44,10 +44,14 @@ module GoodData
|
|
44
44
|
realize_link
|
45
45
|
when 's3'
|
46
46
|
realize_s3(params)
|
47
|
-
when 'redshift', 'snowflake', 'bigquery', 'postgresql'
|
47
|
+
when 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql'
|
48
48
|
raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
|
49
49
|
require_relative '../cloud_resources/cloud_resources'
|
50
50
|
realize_cloud_resource(source, params)
|
51
|
+
when 'blobStorage'
|
52
|
+
require_relative '../cloud_resources/blobstorage/blobstorage_client'
|
53
|
+
blob_storage_client = GoodData::BlobStorageClient.new(params)
|
54
|
+
blob_storage_client.realize_blob(@options[:file], params)
|
51
55
|
else
|
52
56
|
raise "DataSource does not support type \"#{source}\""
|
53
57
|
end
|
@@ -111,12 +115,13 @@ module GoodData
|
|
111
115
|
end
|
112
116
|
|
113
117
|
def realize_s3(params)
|
114
|
-
s3_client = params['
|
118
|
+
s3_client = params['s3_client'] && params['s3_client']['client']
|
115
119
|
raise 'AWS client not present. Perhaps S3Middleware is missing in the brick definition?' if !s3_client || !s3_client.respond_to?(:bucket)
|
116
120
|
bucket_name = @options[:bucket]
|
117
|
-
key = @options[:key]
|
121
|
+
key = @options[:key].present? ? @options[:key] : @options[:file]
|
118
122
|
raise 'Key "bucket" is missing in S3 datasource' if bucket_name.blank?
|
119
|
-
raise 'Key "key" is missing in S3 datasource' if key.blank?
|
123
|
+
raise 'Key "key" or "file" is missing in S3 datasource' if key.blank?
|
124
|
+
|
120
125
|
GoodData.logger.info("Realizing download from S3. Bucket #{bucket_name}, object with key #{key}.")
|
121
126
|
filename = Digest::SHA256.new.hexdigest(@options.to_json)
|
122
127
|
bucket = s3_client.bucket(bucket_name)
|
@@ -53,7 +53,9 @@ module GoodData
|
|
53
53
|
client: development_client
|
54
54
|
)
|
55
55
|
kpi_dashboards = MdObject.query('analyticalDashboard', MdObject, client: development_client, project: from_project)
|
56
|
-
|
56
|
+
kpi_dashboard_plugin = MdObject.query('dashboardPlugin', MdObject, client: development_client, project: from_project)
|
57
|
+
kpi_date_filter_config = MdObject.query('dateFilterConfig', MdObject, client: development_client, project: from_project)
|
58
|
+
objects = old_dashboards.to_a + kpi_dashboards.to_a + kpi_dashboard_plugin.to_a + kpi_date_filter_config.to_a
|
57
59
|
else
|
58
60
|
objects = GoodData::Dashboard.find_by_tag(
|
59
61
|
production_tags,
|
@@ -50,6 +50,7 @@ module GoodData
|
|
50
50
|
segment_info[:to].pmap do |entry|
|
51
51
|
pid = entry[:pid]
|
52
52
|
to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
|
53
|
+
GoodData.logger.info "Migrating date dimension, project: '#{to_project.title}', PID: #{pid}"
|
53
54
|
to_blueprint = to_project.blueprint
|
54
55
|
upgrade_datasets = get_upgrade_dates(latest_blueprint, to_blueprint)
|
55
56
|
next if upgrade_datasets.empty?
|
@@ -71,9 +72,9 @@ module GoodData
|
|
71
72
|
dest_dates = get_date_dimensions(dest_blueprint) if dest_blueprint
|
72
73
|
src_dates = get_date_dimensions(src_blueprint) if src_blueprint
|
73
74
|
|
74
|
-
return false if dest_dates.empty? || src_dates.empty?
|
75
|
-
|
76
75
|
upgrade_datasets = []
|
76
|
+
return upgrade_datasets if dest_dates.empty? || src_dates.empty?
|
77
|
+
|
77
78
|
dest_dates.each do |dest|
|
78
79
|
src_dim = get_date_dimension(src_blueprint, dest[:id])
|
79
80
|
next unless src_dim
|
@@ -33,6 +33,9 @@ module GoodData
|
|
33
33
|
description 'ADS Client'
|
34
34
|
param :ads_client, instance_of(Type::AdsClientType), required: false
|
35
35
|
|
36
|
+
description 'Keep number of old master workspace excluding the latest one'
|
37
|
+
param :keep_only_previous_masters_count, instance_of(Type::StringType), required: false, default: '-1'
|
38
|
+
|
36
39
|
description 'Additional Hidden Parameters'
|
37
40
|
param :additional_hidden_params, instance_of(Type::HashType), required: false
|
38
41
|
end
|
@@ -53,6 +56,7 @@ module GoodData
|
|
53
56
|
domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
|
54
57
|
data_product = params.data_product
|
55
58
|
domain_segments = domain.segments(:all, data_product)
|
59
|
+
keep_only_previous_masters_count = Integer(params.keep_only_previous_masters_count || "-1")
|
56
60
|
|
57
61
|
segments = params.segments.map do |seg|
|
58
62
|
domain_segments.find do |s|
|
@@ -62,18 +66,14 @@ module GoodData
|
|
62
66
|
|
63
67
|
results = segments.map do |segment|
|
64
68
|
if params.ads_client
|
65
|
-
|
66
|
-
params.release_table_name,
|
67
|
-
params.ads_client,
|
68
|
-
segment.segment_id
|
69
|
-
)
|
69
|
+
master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_ads(params.release_table_name, params.ads_client, segment.segment_id)
|
70
70
|
else
|
71
|
-
|
71
|
+
master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
|
72
72
|
end
|
73
73
|
|
74
|
+
current_master = master_projects.last
|
74
75
|
# TODO: Check res.first.nil? || res.first[:master_project_id].nil?
|
75
76
|
master = client.projects(current_master[:master_project_id])
|
76
|
-
|
77
77
|
segment.master_project = master
|
78
78
|
segment.save
|
79
79
|
|
@@ -87,6 +87,19 @@ module GoodData
|
|
87
87
|
"Details: #{sync_result['links']['details']}")
|
88
88
|
end
|
89
89
|
|
90
|
+
if keep_only_previous_masters_count >= 0
|
91
|
+
number_of_deleted_projects = master_projects.count - (keep_only_previous_masters_count + 1)
|
92
|
+
|
93
|
+
if number_of_deleted_projects.positive?
|
94
|
+
begin
|
95
|
+
removal_master_project_ids = remove_multiple_workspace(params, segment.segment_id, master_projects, number_of_deleted_projects)
|
96
|
+
remove_old_workspaces_from_release_table(params, domain_name, data_product.data_product_id, segment.segment_id, master_projects, removal_master_project_ids)
|
97
|
+
rescue Exception => e # rubocop:disable RescueException
|
98
|
+
GoodData.logger.error "Problem occurs when removing old master workspace, reason: #{e.message}"
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
90
103
|
{
|
91
104
|
segment: segment.id,
|
92
105
|
master_pid: master.pid,
|
@@ -98,6 +111,42 @@ module GoodData
|
|
98
111
|
# Return results
|
99
112
|
results
|
100
113
|
end
|
114
|
+
|
115
|
+
def remove_multiple_workspace(params, segment_id, master_projects, number_of_deleted_projects)
|
116
|
+
removal_master_project_ids = []
|
117
|
+
need_to_delete_projects = master_projects.take(number_of_deleted_projects)
|
118
|
+
|
119
|
+
need_to_delete_projects.each do |project_wrapper|
|
120
|
+
master_project_id = project_wrapper[:master_project_id]
|
121
|
+
next if master_project_id.to_s.empty?
|
122
|
+
|
123
|
+
begin
|
124
|
+
project = params.gdc_gd_client.projects(master_project_id)
|
125
|
+
if project && !%w[deleted archived].include?(project.state.to_s)
|
126
|
+
GoodData.logger.info "Segment #{segment_id}: Deleting old master workspace, project: '#{project.title}', PID: (#{project.pid})."
|
127
|
+
project.delete
|
128
|
+
end
|
129
|
+
removal_master_project_ids << master_project_id
|
130
|
+
master_projects.delete_if { |p| p[:master_project_id] == master_project_id }
|
131
|
+
rescue Exception => ex # rubocop:disable RescueException
|
132
|
+
GoodData.logger.error "Unable to remove master workspace: '#{master_project_id}', Error: #{ex.message}"
|
133
|
+
end
|
134
|
+
end
|
135
|
+
removal_master_project_ids
|
136
|
+
end
|
137
|
+
|
138
|
+
# rubocop:disable Metrics/ParameterLists
|
139
|
+
def remove_old_workspaces_from_release_table(params, domain_id, data_product_id, segment_id, master_projects, removal_master_project_ids)
|
140
|
+
unless removal_master_project_ids.empty?
|
141
|
+
if params.ads_client
|
142
|
+
GoodData::LCM2::Helpers.delete_master_project_from_ads(params.release_table_name, params.ads_client, segment_id, removal_master_project_ids)
|
143
|
+
else
|
144
|
+
data = master_projects.sort_by { |master| master[:version] }
|
145
|
+
GoodData::LCM2::Helpers.update_master_project_to_nfs(domain_id, data_product_id, segment_id, data)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
# rubocop:enable Metrics/ParameterLists
|
101
150
|
end
|
102
151
|
end
|
103
152
|
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require_relative 'base_action'
|
9
|
+
|
10
|
+
module GoodData
|
11
|
+
module LCM2
|
12
|
+
class SynchronizeDataSetMapping < BaseAction
|
13
|
+
DESCRIPTION = 'Synchronize Dataset Mappings'
|
14
|
+
|
15
|
+
PARAMS = define_params(self) do
|
16
|
+
description 'Client Used for Connecting to GD'
|
17
|
+
param :gdc_gd_client, instance_of(Type::GdClientType), required: true
|
18
|
+
|
19
|
+
description 'Client used to connecting to development domain'
|
20
|
+
param :development_client, instance_of(Type::GdClientType), required: true
|
21
|
+
|
22
|
+
description 'Synchronization Info'
|
23
|
+
param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
|
24
|
+
|
25
|
+
description 'Logger'
|
26
|
+
param :gdc_logger, instance_of(Type::GdLogger), required: true
|
27
|
+
end
|
28
|
+
|
29
|
+
RESULT_HEADER = %i[from to count status]
|
30
|
+
|
31
|
+
class << self
|
32
|
+
def call(params)
|
33
|
+
results = []
|
34
|
+
|
35
|
+
client = params.gdc_gd_client
|
36
|
+
development_client = params.development_client
|
37
|
+
|
38
|
+
params.synchronize.peach do |info|
|
39
|
+
from_project = info.from
|
40
|
+
to_projects = info.to
|
41
|
+
|
42
|
+
from = development_client.projects(from_project) || fail("Invalid 'from' project specified - '#{from_project}'")
|
43
|
+
dataset_mapping = from.dataset_mapping
|
44
|
+
if dataset_mapping&.dig('datasetMappings', 'items').nil? || dataset_mapping['datasetMappings']['items'].empty?
|
45
|
+
params.gdc_logger.info "Project: '#{from.title}', PID: '#{from.pid}' has no model mapping, skip synchronizing model mapping."
|
46
|
+
else
|
47
|
+
to_projects.peach do |to|
|
48
|
+
pid = to[:pid]
|
49
|
+
to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
|
50
|
+
|
51
|
+
params.gdc_logger.info "Transferring model mapping, from project: '#{from.title}', PID: '#{from.pid}', to project: '#{to_project.title}', PID: '#{to_project.pid}'"
|
52
|
+
res = to_project.update_dataset_mapping(dataset_mapping)
|
53
|
+
res[:from] = from.pid
|
54
|
+
results << res
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
# Return results
|
59
|
+
results.flatten
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
@@ -83,18 +83,29 @@ module GoodData
|
|
83
83
|
segment_info[:from_blueprint] = blueprint
|
84
84
|
maql_diff = nil
|
85
85
|
previous_master = segment_info[:previous_master]
|
86
|
+
synchronize_ldm_mode = params[:synchronize_ldm].downcase
|
86
87
|
diff_against_master = %w(diff_against_master_with_fallback diff_against_master)
|
87
|
-
.include?(
|
88
|
-
GoodData.logger.info "Synchronize LDM mode: '#{
|
89
|
-
if previous_master && diff_against_master
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
88
|
+
.include?(synchronize_ldm_mode)
|
89
|
+
GoodData.logger.info "Synchronize LDM mode: '#{synchronize_ldm_mode}'"
|
90
|
+
if segment_info.key?(:previous_master) && diff_against_master
|
91
|
+
if previous_master
|
92
|
+
maql_diff_params = [:includeGrain]
|
93
|
+
maql_diff_params << :excludeFactRule if exclude_fact_rule
|
94
|
+
maql_diff_params << :includeDeprecated if include_deprecated
|
95
|
+
maql_diff = previous_master.maql_diff(blueprint: blueprint, params: maql_diff_params)
|
96
|
+
else
|
97
|
+
maql_diff = {
|
98
|
+
"projectModelDiff" =>
|
99
|
+
{
|
100
|
+
"updateOperations" => [],
|
101
|
+
"updateScripts" => []
|
102
|
+
}
|
103
|
+
}
|
104
|
+
end
|
94
105
|
chunks = maql_diff['projectModelDiff']['updateScripts']
|
95
106
|
if chunks.empty?
|
96
107
|
GoodData.logger.info "Synchronize LDM to clients will not proceed in mode \
|
97
|
-
'#{
|
108
|
+
'#{synchronize_ldm_mode}' due to no LDM changes in the segment master project. \
|
98
109
|
If you had changed LDM of clients manually, please use mode 'diff_against_clients' \
|
99
110
|
to force synchronize LDM to clients"
|
100
111
|
end
|