gooddata 2.1.19-java → 2.2.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gdc-ii-config.yaml +1 -1
- data/.github/workflows/build.yml +66 -0
- data/.github/workflows/pre-merge.yml +72 -0
- data/CHANGELOG.md +38 -0
- data/Dockerfile +21 -14
- data/Dockerfile.jruby +1 -11
- data/README.md +1 -2
- data/SDK_VERSION +1 -1
- data/VERSION +1 -1
- data/ci/mssql/pom.xml +62 -0
- data/ci/mysql/pom.xml +57 -0
- data/ci/redshift/pom.xml +1 -1
- data/docker-compose.lcm.yml +0 -3
- data/gooddata.gemspec +2 -1
- data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
- data/lcm.rake +2 -8
- data/lib/gooddata/bricks/middleware/aws_middleware.rb +35 -9
- data/lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb +98 -0
- data/lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mssql/mssql_client.rb +122 -0
- data/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +111 -0
- data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +0 -1
- data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +18 -1
- data/lib/gooddata/helpers/data_helper.rb +9 -4
- data/lib/gooddata/lcm/actions/collect_meta.rb +3 -1
- data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +3 -2
- data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -7
- data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +64 -0
- data/lib/gooddata/lcm/actions/synchronize_ldm.rb +19 -8
- data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +12 -9
- data/lib/gooddata/lcm/actions/update_metric_formats.rb +185 -0
- data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb +5 -0
- data/lib/gooddata/lcm/helpers/release_table_helper.rb +42 -8
- data/lib/gooddata/lcm/lcm2.rb +5 -0
- data/lib/gooddata/mixins/md_object_query.rb +1 -0
- data/lib/gooddata/models/data_source.rb +5 -1
- data/lib/gooddata/models/dataset_mapping.rb +36 -0
- data/lib/gooddata/models/metadata/label.rb +26 -27
- data/lib/gooddata/models/project.rb +34 -9
- data/lib/gooddata/models/schedule.rb +13 -1
- data/lib/gooddata/models/user_filters/user_filter_builder.rb +58 -53
- data/lib/gooddata/rest/phmap.rb +1 -0
- metadata +44 -18
- data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb +0 -37
@@ -0,0 +1,98 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'pathname'
|
10
|
+
require "azure/storage/blob"
|
11
|
+
|
12
|
+
module GoodData
|
13
|
+
class BlobStorageClient
|
14
|
+
SAS_URL_PATTERN = %r{(^https?:\/\/[^\/]*)\/.*\?(.*)}
|
15
|
+
INVALID_BLOB_GENERAL_MESSAGE = "The connection string is not valid."
|
16
|
+
INVALID_BLOB_SIG_WELL_FORMED_MESSAGE = "The signature format is not valid."
|
17
|
+
INVALID_BLOB_CONTAINER_MESSAGE = "ContainerNotFound"
|
18
|
+
INVALID_BLOB_CONTAINER_FORMED_MESSAGE = "The container with the specified name is not found."
|
19
|
+
INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE = "Signature not valid in the specified time frame"
|
20
|
+
INVALID_BLOB_EXPIRED_MESSAGE = "The signature expired."
|
21
|
+
INVALID_BLOB_INVALID_CONNECTION_STRING_MESSAGE = "The connection string is not valid."
|
22
|
+
INVALID_BLOB_PATH_MESSAGE = "BlobNotFound"
|
23
|
+
INVALID_BLOB_INVALID_PATH_MESSAGE = "The path to the data is not found."
|
24
|
+
|
25
|
+
attr_reader :use_sas
|
26
|
+
|
27
|
+
def initialize(options = {})
|
28
|
+
raise("Data Source needs a client to Blob Storage to be able to get blob file but 'blobStorage_client' is empty.") unless options['blobStorage_client']
|
29
|
+
|
30
|
+
if options['blobStorage_client']['connectionString'] && options['blobStorage_client']['container']
|
31
|
+
@connection_string = options['blobStorage_client']['connectionString']
|
32
|
+
@container = options['blobStorage_client']['container']
|
33
|
+
@path = options['blobStorage_client']['path']
|
34
|
+
@use_sas = false
|
35
|
+
build_sas(@connection_string)
|
36
|
+
else
|
37
|
+
raise('Missing connection info for Blob Storage client')
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def realize_blob(file, _params)
|
42
|
+
GoodData.gd_logger.info("Realizing download from Blob Storage. Container #{@container}.")
|
43
|
+
filename = ''
|
44
|
+
begin
|
45
|
+
connect
|
46
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
47
|
+
blob_name = get_blob_name(@path, file)
|
48
|
+
|
49
|
+
measure = Benchmark.measure do
|
50
|
+
_blob, content = @client.get_blob(@container, blob_name)
|
51
|
+
File.open(filename, "wb") { |f| f.write(content) }
|
52
|
+
end
|
53
|
+
rescue => e
|
54
|
+
raise_error(e)
|
55
|
+
end
|
56
|
+
GoodData.gd_logger.info("Done downloading file type=blobStorage status=finished duration=#{measure.real}")
|
57
|
+
filename
|
58
|
+
end
|
59
|
+
|
60
|
+
def connect
|
61
|
+
GoodData.logger.info "Setting up connection to Blob Storage"
|
62
|
+
if use_sas
|
63
|
+
@client = Azure::Storage::Blob::BlobService.create(:storage_blob_host => @host, :storage_sas_token => @sas_token)
|
64
|
+
else
|
65
|
+
@client = Azure::Storage::Blob::BlobService.create_from_connection_string(@connection_string)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def build_sas(url)
|
70
|
+
matches = url.scan(SAS_URL_PATTERN)
|
71
|
+
return unless matches && matches[0]
|
72
|
+
|
73
|
+
@use_sas = true
|
74
|
+
@host = matches[0][0]
|
75
|
+
@sas_token = matches[0][1]
|
76
|
+
end
|
77
|
+
|
78
|
+
def raise_error(e)
|
79
|
+
if e.message && e.message.include?(INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE)
|
80
|
+
raise INVALID_BLOB_EXPIRED_MESSAGE
|
81
|
+
elsif e.message && e.message.include?(INVALID_BLOB_SIG_WELL_FORMED_MESSAGE)
|
82
|
+
raise INVALID_BLOB_SIG_WELL_FORMED_MESSAGE
|
83
|
+
elsif e.message && e.message.include?(INVALID_BLOB_CONTAINER_MESSAGE)
|
84
|
+
raise INVALID_BLOB_CONTAINER_FORMED_MESSAGE
|
85
|
+
elsif e.message && e.message.include?(INVALID_BLOB_PATH_MESSAGE)
|
86
|
+
raise INVALID_BLOB_INVALID_PATH_MESSAGE
|
87
|
+
else
|
88
|
+
raise INVALID_BLOB_GENERAL_MESSAGE
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def get_blob_name(path, file)
|
93
|
+
return file unless path
|
94
|
+
|
95
|
+
path.rindex('/') == path.length - 1 ? "#{path}#{file}" : "#{path}/#{file}"
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
File without changes
|
@@ -0,0 +1,122 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'java'
|
10
|
+
require 'pathname'
|
11
|
+
require_relative '../cloud_resource_client'
|
12
|
+
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
15
|
+
require file unless file.start_with?('lcm-mssql-driver')
|
16
|
+
end
|
17
|
+
|
18
|
+
module GoodData
|
19
|
+
module CloudResources
|
20
|
+
class MSSQLClient < CloudResourceClient
|
21
|
+
MSSQL_SEPARATOR_PARAM = ";"
|
22
|
+
MSSQL_URL_PATTERN = %r{jdbc:sqlserver://([^:]+)(:([0-9]+))?(/)?}
|
23
|
+
MSSQL_DEFAULT_PORT = 1433
|
24
|
+
LOGIN_TIME_OUT = 30
|
25
|
+
MSSQL_FETCH_SIZE = 10_000
|
26
|
+
VERIFY_FULL = 'verify-full'
|
27
|
+
PREFER = 'prefer'
|
28
|
+
REQUIRE = 'require'
|
29
|
+
|
30
|
+
class << self
|
31
|
+
def accept?(type)
|
32
|
+
type == 'mssql'
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def initialize(options = {})
|
37
|
+
raise("Data Source needs a client to MSSQL to be able to query the storage but 'mssql_client' is empty.") unless options['mssql_client']
|
38
|
+
|
39
|
+
connection = options['mssql_client']['connection']
|
40
|
+
if connection.is_a?(Hash)
|
41
|
+
@database = connection['database']
|
42
|
+
@schema = connection['schema']
|
43
|
+
@authentication = connection['authentication']
|
44
|
+
@ssl_mode = connection['sslMode']
|
45
|
+
@url = connection['url']
|
46
|
+
|
47
|
+
validate
|
48
|
+
else
|
49
|
+
raise('Missing connection info for MSSQL client')
|
50
|
+
end
|
51
|
+
|
52
|
+
Java.com.microsoft.sqlserver.jdbc.SQLServerDriver
|
53
|
+
end
|
54
|
+
|
55
|
+
def realize_query(query, _params)
|
56
|
+
GoodData.gd_logger.info("Realize SQL query: type=mssql status=started")
|
57
|
+
|
58
|
+
connect
|
59
|
+
|
60
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
61
|
+
measure = Benchmark.measure do
|
62
|
+
statement = @connection.create_statement
|
63
|
+
statement.set_fetch_size(MSSQL_FETCH_SIZE)
|
64
|
+
has_result = statement.execute(query)
|
65
|
+
if has_result
|
66
|
+
result = statement.get_result_set
|
67
|
+
metadata = result.get_meta_data
|
68
|
+
col_count = metadata.column_count
|
69
|
+
CSV.open(filename, 'wb') do |csv|
|
70
|
+
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
71
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
GoodData.gd_logger.info("Realize SQL query: type=mssql status=finished duration=#{measure.real}")
|
77
|
+
filename
|
78
|
+
ensure
|
79
|
+
@connection&.close
|
80
|
+
@connection = nil
|
81
|
+
end
|
82
|
+
|
83
|
+
def connect
|
84
|
+
connection_string = build_connection_string
|
85
|
+
GoodData.logger.info "Setting up connection to MSSQL #{connection_string}"
|
86
|
+
|
87
|
+
authentication = @authentication['basic'] || @authentication['activeDirectoryPassword']
|
88
|
+
|
89
|
+
prop = java.util.Properties.new
|
90
|
+
prop.setProperty('userName', authentication['userName'])
|
91
|
+
prop.setProperty('password', authentication['password'])
|
92
|
+
|
93
|
+
@connection = java.sql.DriverManager.getConnection(connection_string, prop)
|
94
|
+
end
|
95
|
+
|
96
|
+
def validate
|
97
|
+
raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
|
98
|
+
|
99
|
+
raise "Instance name is not supported" if @url !~ /^[^\\]*$/
|
100
|
+
|
101
|
+
raise "The connection url is invalid. Parameter is not supported." if @url.include? MSSQL_SEPARATOR_PARAM
|
102
|
+
|
103
|
+
url_matches = @url.scan(MSSQL_URL_PATTERN)
|
104
|
+
raise "Cannot reach the url" if url_matches.nil? || url_matches.length.zero?
|
105
|
+
|
106
|
+
raise "The authentication method is not supported." unless @authentication['basic'] || @authentication['activeDirectoryPassword']
|
107
|
+
end
|
108
|
+
|
109
|
+
def build_connection_string
|
110
|
+
encrypt = @ssl_mode != PREFER
|
111
|
+
trust_server_certificate = @ssl_mode == REQUIRE
|
112
|
+
|
113
|
+
"#{@url};" \
|
114
|
+
"database=#{@database};" \
|
115
|
+
"encrypt=#{encrypt};" \
|
116
|
+
"trustServerCertificate=#{trust_server_certificate};" \
|
117
|
+
"loginTimeout=#{LOGIN_TIME_OUT};" \
|
118
|
+
"#{'authentication=ActiveDirectoryPassword;' if @authentication['activeDirectoryPassword']}"
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
File without changes
|
@@ -0,0 +1,111 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require 'securerandom'
|
9
|
+
require 'java'
|
10
|
+
require 'pathname'
|
11
|
+
require_relative '../cloud_resource_client'
|
12
|
+
|
13
|
+
base = Pathname(__FILE__).dirname.expand_path
|
14
|
+
Dir.glob(base + 'drivers/*.jar').each do |file|
|
15
|
+
require file unless file.start_with?('lcm-mysql-driver')
|
16
|
+
end
|
17
|
+
|
18
|
+
module GoodData
|
19
|
+
module CloudResources
|
20
|
+
class MysqlClient < CloudResourceClient
|
21
|
+
JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
|
22
|
+
MYSQL_DEFAULT_PORT = 3306
|
23
|
+
JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://'
|
24
|
+
VERIFY_FULL = 'VERIFY_IDENTITY'
|
25
|
+
PREFER = 'PREFERRED'
|
26
|
+
REQUIRE = 'REQUIRED'
|
27
|
+
MYSQL_FETCH_SIZE = 1000
|
28
|
+
|
29
|
+
class << self
|
30
|
+
def accept?(type)
|
31
|
+
type == 'mysql'
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def initialize(options = {})
|
36
|
+
raise("Data Source needs a client to Mysql to be able to query the storage but 'mysql_client' is empty.") unless options['mysql_client']
|
37
|
+
|
38
|
+
if options['mysql_client']['connection'].is_a?(Hash)
|
39
|
+
@database = options['mysql_client']['connection']['database']
|
40
|
+
@authentication = options['mysql_client']['connection']['authentication']
|
41
|
+
@ssl_mode = options['mysql_client']['connection']['sslMode']
|
42
|
+
raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
|
43
|
+
|
44
|
+
@url = build_url(options['mysql_client']['connection']['url'])
|
45
|
+
else
|
46
|
+
raise('Missing connection info for Mysql client')
|
47
|
+
end
|
48
|
+
|
49
|
+
Java.com.mysql.cj.jdbc.Driver
|
50
|
+
end
|
51
|
+
|
52
|
+
def realize_query(query, _params)
|
53
|
+
GoodData.gd_logger.info("Realize SQL query: type=mysql status=started")
|
54
|
+
|
55
|
+
connect
|
56
|
+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
57
|
+
measure = Benchmark.measure do
|
58
|
+
statement = @connection.create_statement
|
59
|
+
statement.set_fetch_size(MYSQL_FETCH_SIZE)
|
60
|
+
has_result = statement.execute(query)
|
61
|
+
if has_result
|
62
|
+
result = statement.get_result_set
|
63
|
+
metadata = result.get_meta_data
|
64
|
+
col_count = metadata.column_count
|
65
|
+
CSV.open(filename, 'wb') do |csv|
|
66
|
+
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
|
67
|
+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
GoodData.gd_logger.info("Realize SQL query: type=mysql status=finished duration=#{measure.real}")
|
72
|
+
filename
|
73
|
+
ensure
|
74
|
+
@connection&.close
|
75
|
+
@connection = nil
|
76
|
+
end
|
77
|
+
|
78
|
+
def connect
|
79
|
+
GoodData.logger.info "Setting up connection to Mysql #{@url}"
|
80
|
+
|
81
|
+
prop = java.util.Properties.new
|
82
|
+
prop.setProperty('user', @authentication['basic']['userName'])
|
83
|
+
prop.setProperty('password', @authentication['basic']['password'])
|
84
|
+
|
85
|
+
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
86
|
+
@connection.set_auto_commit(false)
|
87
|
+
end
|
88
|
+
|
89
|
+
def build_url(url)
|
90
|
+
matches = url.scan(JDBC_MYSQL_PATTERN)
|
91
|
+
raise 'Cannot reach the url' unless matches
|
92
|
+
|
93
|
+
host = matches[0][0]
|
94
|
+
port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT
|
95
|
+
|
96
|
+
"#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{get_ssl_mode(@ssl_mode)}&useCursorFetch=true&enabledTLSProtocols=TLSv1.2"
|
97
|
+
end
|
98
|
+
|
99
|
+
def get_ssl_mode(ssl_mode)
|
100
|
+
mode = PREFER
|
101
|
+
if ssl_mode == 'verify-full'
|
102
|
+
mode = VERIFY_FULL
|
103
|
+
elsif ssl_mode == 'require'
|
104
|
+
mode = REQUIRE
|
105
|
+
end
|
106
|
+
|
107
|
+
mode
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
@@ -98,7 +98,6 @@ module GoodData
|
|
98
98
|
|
99
99
|
host = matches[0][0]
|
100
100
|
port = matches[0][2]&.to_i || POSTGRES_DEFAULT_PORT
|
101
|
-
raise "Custom port #{port} is not supported. Remove it or use the default port '5432'" if POSTGRES_DEFAULT_PORT != port
|
102
101
|
|
103
102
|
"#{JDBC_POSTGRES_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{@ssl_mode}#{VERIFY_FULL == @ssl_mode ? SSL_JAVA_FACTORY : ''}"
|
104
103
|
end
|
@@ -18,6 +18,9 @@ end
|
|
18
18
|
module GoodData
|
19
19
|
module CloudResources
|
20
20
|
class SnowflakeClient < CloudResourceClient
|
21
|
+
SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=GoodData_Platform'
|
22
|
+
SNOWFLAKE_SEPARATOR_PARAM = '?'
|
23
|
+
|
21
24
|
class << self
|
22
25
|
def accept?(type)
|
23
26
|
type == 'snowflake'
|
@@ -31,7 +34,7 @@ module GoodData
|
|
31
34
|
@database = options['snowflake_client']['connection']['database']
|
32
35
|
@schema = options['snowflake_client']['connection']['schema'] || 'public'
|
33
36
|
@warehouse = options['snowflake_client']['connection']['warehouse']
|
34
|
-
@url = options['snowflake_client']['connection']['url']
|
37
|
+
@url = build_url(options['snowflake_client']['connection']['url'])
|
35
38
|
@authentication = options['snowflake_client']['connection']['authentication']
|
36
39
|
else
|
37
40
|
raise('Missing connection info for Snowflake client')
|
@@ -79,6 +82,20 @@ module GoodData
|
|
79
82
|
|
80
83
|
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
81
84
|
end
|
85
|
+
|
86
|
+
def build_url(url)
|
87
|
+
is_contain = url.include?(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
|
88
|
+
unless is_contain
|
89
|
+
if url.include?(SNOWFLAKE_SEPARATOR_PARAM)
|
90
|
+
url.concat("&")
|
91
|
+
else
|
92
|
+
url.concat(SNOWFLAKE_SEPARATOR_PARAM)
|
93
|
+
end
|
94
|
+
url.concat(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
|
95
|
+
end
|
96
|
+
|
97
|
+
url
|
98
|
+
end
|
82
99
|
end
|
83
100
|
end
|
84
101
|
end
|
@@ -44,10 +44,14 @@ module GoodData
|
|
44
44
|
realize_link
|
45
45
|
when 's3'
|
46
46
|
realize_s3(params)
|
47
|
-
when 'redshift', 'snowflake', 'bigquery', 'postgresql'
|
47
|
+
when 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql'
|
48
48
|
raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
|
49
49
|
require_relative '../cloud_resources/cloud_resources'
|
50
50
|
realize_cloud_resource(source, params)
|
51
|
+
when 'blobStorage'
|
52
|
+
require_relative '../cloud_resources/blobstorage/blobstorage_client'
|
53
|
+
blob_storage_client = GoodData::BlobStorageClient.new(params)
|
54
|
+
blob_storage_client.realize_blob(@options[:file], params)
|
51
55
|
else
|
52
56
|
raise "DataSource does not support type \"#{source}\""
|
53
57
|
end
|
@@ -111,12 +115,13 @@ module GoodData
|
|
111
115
|
end
|
112
116
|
|
113
117
|
def realize_s3(params)
|
114
|
-
s3_client = params['
|
118
|
+
s3_client = params['s3_client'] && params['s3_client']['client']
|
115
119
|
raise 'AWS client not present. Perhaps S3Middleware is missing in the brick definition?' if !s3_client || !s3_client.respond_to?(:bucket)
|
116
120
|
bucket_name = @options[:bucket]
|
117
|
-
key = @options[:key]
|
121
|
+
key = @options[:key].present? ? @options[:key] : @options[:file]
|
118
122
|
raise 'Key "bucket" is missing in S3 datasource' if bucket_name.blank?
|
119
|
-
raise 'Key "key" is missing in S3 datasource' if key.blank?
|
123
|
+
raise 'Key "key" or "file" is missing in S3 datasource' if key.blank?
|
124
|
+
|
120
125
|
GoodData.logger.info("Realizing download from S3. Bucket #{bucket_name}, object with key #{key}.")
|
121
126
|
filename = Digest::SHA256.new.hexdigest(@options.to_json)
|
122
127
|
bucket = s3_client.bucket(bucket_name)
|
@@ -53,7 +53,9 @@ module GoodData
|
|
53
53
|
client: development_client
|
54
54
|
)
|
55
55
|
kpi_dashboards = MdObject.query('analyticalDashboard', MdObject, client: development_client, project: from_project)
|
56
|
-
|
56
|
+
kpi_dashboard_plugin = MdObject.query('dashboardPlugin', MdObject, client: development_client, project: from_project)
|
57
|
+
kpi_date_filter_config = MdObject.query('dateFilterConfig', MdObject, client: development_client, project: from_project)
|
58
|
+
objects = old_dashboards.to_a + kpi_dashboards.to_a + kpi_dashboard_plugin.to_a + kpi_date_filter_config.to_a
|
57
59
|
else
|
58
60
|
objects = GoodData::Dashboard.find_by_tag(
|
59
61
|
production_tags,
|
@@ -50,6 +50,7 @@ module GoodData
|
|
50
50
|
segment_info[:to].pmap do |entry|
|
51
51
|
pid = entry[:pid]
|
52
52
|
to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
|
53
|
+
GoodData.logger.info "Migrating date dimension, project: '#{to_project.title}', PID: #{pid}"
|
53
54
|
to_blueprint = to_project.blueprint
|
54
55
|
upgrade_datasets = get_upgrade_dates(latest_blueprint, to_blueprint)
|
55
56
|
next if upgrade_datasets.empty?
|
@@ -71,9 +72,9 @@ module GoodData
|
|
71
72
|
dest_dates = get_date_dimensions(dest_blueprint) if dest_blueprint
|
72
73
|
src_dates = get_date_dimensions(src_blueprint) if src_blueprint
|
73
74
|
|
74
|
-
return false if dest_dates.empty? || src_dates.empty?
|
75
|
-
|
76
75
|
upgrade_datasets = []
|
76
|
+
return upgrade_datasets if dest_dates.empty? || src_dates.empty?
|
77
|
+
|
77
78
|
dest_dates.each do |dest|
|
78
79
|
src_dim = get_date_dimension(src_blueprint, dest[:id])
|
79
80
|
next unless src_dim
|
@@ -33,6 +33,9 @@ module GoodData
|
|
33
33
|
description 'ADS Client'
|
34
34
|
param :ads_client, instance_of(Type::AdsClientType), required: false
|
35
35
|
|
36
|
+
description 'Keep number of old master workspace excluding the latest one'
|
37
|
+
param :keep_only_previous_masters_count, instance_of(Type::StringType), required: false, default: '-1'
|
38
|
+
|
36
39
|
description 'Additional Hidden Parameters'
|
37
40
|
param :additional_hidden_params, instance_of(Type::HashType), required: false
|
38
41
|
end
|
@@ -53,6 +56,7 @@ module GoodData
|
|
53
56
|
domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
|
54
57
|
data_product = params.data_product
|
55
58
|
domain_segments = domain.segments(:all, data_product)
|
59
|
+
keep_only_previous_masters_count = Integer(params.keep_only_previous_masters_count || "-1")
|
56
60
|
|
57
61
|
segments = params.segments.map do |seg|
|
58
62
|
domain_segments.find do |s|
|
@@ -62,18 +66,14 @@ module GoodData
|
|
62
66
|
|
63
67
|
results = segments.map do |segment|
|
64
68
|
if params.ads_client
|
65
|
-
|
66
|
-
params.release_table_name,
|
67
|
-
params.ads_client,
|
68
|
-
segment.segment_id
|
69
|
-
)
|
69
|
+
master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_ads(params.release_table_name, params.ads_client, segment.segment_id)
|
70
70
|
else
|
71
|
-
|
71
|
+
master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
|
72
72
|
end
|
73
73
|
|
74
|
+
current_master = master_projects.last
|
74
75
|
# TODO: Check res.first.nil? || res.first[:master_project_id].nil?
|
75
76
|
master = client.projects(current_master[:master_project_id])
|
76
|
-
|
77
77
|
segment.master_project = master
|
78
78
|
segment.save
|
79
79
|
|
@@ -87,6 +87,19 @@ module GoodData
|
|
87
87
|
"Details: #{sync_result['links']['details']}")
|
88
88
|
end
|
89
89
|
|
90
|
+
if keep_only_previous_masters_count >= 0
|
91
|
+
number_of_deleted_projects = master_projects.count - (keep_only_previous_masters_count + 1)
|
92
|
+
|
93
|
+
if number_of_deleted_projects.positive?
|
94
|
+
begin
|
95
|
+
removal_master_project_ids = remove_multiple_workspace(params, segment.segment_id, master_projects, number_of_deleted_projects)
|
96
|
+
remove_old_workspaces_from_release_table(params, domain_name, data_product.data_product_id, segment.segment_id, master_projects, removal_master_project_ids)
|
97
|
+
rescue Exception => e # rubocop:disable RescueException
|
98
|
+
GoodData.logger.error "Problem occurs when removing old master workspace, reason: #{e.message}"
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
90
103
|
{
|
91
104
|
segment: segment.id,
|
92
105
|
master_pid: master.pid,
|
@@ -98,6 +111,42 @@ module GoodData
|
|
98
111
|
# Return results
|
99
112
|
results
|
100
113
|
end
|
114
|
+
|
115
|
+
def remove_multiple_workspace(params, segment_id, master_projects, number_of_deleted_projects)
|
116
|
+
removal_master_project_ids = []
|
117
|
+
need_to_delete_projects = master_projects.take(number_of_deleted_projects)
|
118
|
+
|
119
|
+
need_to_delete_projects.each do |project_wrapper|
|
120
|
+
master_project_id = project_wrapper[:master_project_id]
|
121
|
+
next if master_project_id.to_s.empty?
|
122
|
+
|
123
|
+
begin
|
124
|
+
project = params.gdc_gd_client.projects(master_project_id)
|
125
|
+
if project && !%w[deleted archived].include?(project.state.to_s)
|
126
|
+
GoodData.logger.info "Segment #{segment_id}: Deleting old master workspace, project: '#{project.title}', PID: (#{project.pid})."
|
127
|
+
project.delete
|
128
|
+
end
|
129
|
+
removal_master_project_ids << master_project_id
|
130
|
+
master_projects.delete_if { |p| p[:master_project_id] == master_project_id }
|
131
|
+
rescue Exception => ex # rubocop:disable RescueException
|
132
|
+
GoodData.logger.error "Unable to remove master workspace: '#{master_project_id}', Error: #{ex.message}"
|
133
|
+
end
|
134
|
+
end
|
135
|
+
removal_master_project_ids
|
136
|
+
end
|
137
|
+
|
138
|
+
# rubocop:disable Metrics/ParameterLists
|
139
|
+
def remove_old_workspaces_from_release_table(params, domain_id, data_product_id, segment_id, master_projects, removal_master_project_ids)
|
140
|
+
unless removal_master_project_ids.empty?
|
141
|
+
if params.ads_client
|
142
|
+
GoodData::LCM2::Helpers.delete_master_project_from_ads(params.release_table_name, params.ads_client, segment_id, removal_master_project_ids)
|
143
|
+
else
|
144
|
+
data = master_projects.sort_by { |master| master[:version] }
|
145
|
+
GoodData::LCM2::Helpers.update_master_project_to_nfs(domain_id, data_product_id, segment_id, data)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
# rubocop:enable Metrics/ParameterLists
|
101
150
|
end
|
102
151
|
end
|
103
152
|
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require_relative 'base_action'
|
9
|
+
|
10
|
+
module GoodData
|
11
|
+
module LCM2
|
12
|
+
class SynchronizeDataSetMapping < BaseAction
|
13
|
+
DESCRIPTION = 'Synchronize Dataset Mappings'
|
14
|
+
|
15
|
+
PARAMS = define_params(self) do
|
16
|
+
description 'Client Used for Connecting to GD'
|
17
|
+
param :gdc_gd_client, instance_of(Type::GdClientType), required: true
|
18
|
+
|
19
|
+
description 'Client used to connecting to development domain'
|
20
|
+
param :development_client, instance_of(Type::GdClientType), required: true
|
21
|
+
|
22
|
+
description 'Synchronization Info'
|
23
|
+
param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
|
24
|
+
|
25
|
+
description 'Logger'
|
26
|
+
param :gdc_logger, instance_of(Type::GdLogger), required: true
|
27
|
+
end
|
28
|
+
|
29
|
+
RESULT_HEADER = %i[from to count status]
|
30
|
+
|
31
|
+
class << self
|
32
|
+
def call(params)
|
33
|
+
results = []
|
34
|
+
|
35
|
+
client = params.gdc_gd_client
|
36
|
+
development_client = params.development_client
|
37
|
+
|
38
|
+
params.synchronize.peach do |info|
|
39
|
+
from_project = info.from
|
40
|
+
to_projects = info.to
|
41
|
+
|
42
|
+
from = development_client.projects(from_project) || fail("Invalid 'from' project specified - '#{from_project}'")
|
43
|
+
dataset_mapping = from.dataset_mapping
|
44
|
+
if dataset_mapping&.dig('datasetMappings', 'items').nil? || dataset_mapping['datasetMappings']['items'].empty?
|
45
|
+
params.gdc_logger.info "Project: '#{from.title}', PID: '#{from.pid}' has no model mapping, skip synchronizing model mapping."
|
46
|
+
else
|
47
|
+
to_projects.peach do |to|
|
48
|
+
pid = to[:pid]
|
49
|
+
to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
|
50
|
+
|
51
|
+
params.gdc_logger.info "Transferring model mapping, from project: '#{from.title}', PID: '#{from.pid}', to project: '#{to_project.title}', PID: '#{to_project.pid}'"
|
52
|
+
res = to_project.update_dataset_mapping(dataset_mapping)
|
53
|
+
res[:from] = from.pid
|
54
|
+
results << res
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
# Return results
|
59
|
+
results.flatten
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
@@ -83,18 +83,29 @@ module GoodData
|
|
83
83
|
segment_info[:from_blueprint] = blueprint
|
84
84
|
maql_diff = nil
|
85
85
|
previous_master = segment_info[:previous_master]
|
86
|
+
synchronize_ldm_mode = params[:synchronize_ldm].downcase
|
86
87
|
diff_against_master = %w(diff_against_master_with_fallback diff_against_master)
|
87
|
-
.include?(
|
88
|
-
GoodData.logger.info "Synchronize LDM mode: '#{
|
89
|
-
if previous_master && diff_against_master
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
88
|
+
.include?(synchronize_ldm_mode)
|
89
|
+
GoodData.logger.info "Synchronize LDM mode: '#{synchronize_ldm_mode}'"
|
90
|
+
if segment_info.key?(:previous_master) && diff_against_master
|
91
|
+
if previous_master
|
92
|
+
maql_diff_params = [:includeGrain]
|
93
|
+
maql_diff_params << :excludeFactRule if exclude_fact_rule
|
94
|
+
maql_diff_params << :includeDeprecated if include_deprecated
|
95
|
+
maql_diff = previous_master.maql_diff(blueprint: blueprint, params: maql_diff_params)
|
96
|
+
else
|
97
|
+
maql_diff = {
|
98
|
+
"projectModelDiff" =>
|
99
|
+
{
|
100
|
+
"updateOperations" => [],
|
101
|
+
"updateScripts" => []
|
102
|
+
}
|
103
|
+
}
|
104
|
+
end
|
94
105
|
chunks = maql_diff['projectModelDiff']['updateScripts']
|
95
106
|
if chunks.empty?
|
96
107
|
GoodData.logger.info "Synchronize LDM to clients will not proceed in mode \
|
97
|
-
'#{
|
108
|
+
'#{synchronize_ldm_mode}' due to no LDM changes in the segment master project. \
|
98
109
|
If you had changed LDM of clients manually, please use mode 'diff_against_clients' \
|
99
110
|
to force synchronize LDM to clients"
|
100
111
|
end
|