gooddata 2.1.19-java → 2.2.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +5 -5
  2. data/.gdc-ii-config.yaml +1 -1
  3. data/.github/workflows/build.yml +66 -0
  4. data/.github/workflows/pre-merge.yml +72 -0
  5. data/CHANGELOG.md +38 -0
  6. data/Dockerfile +21 -14
  7. data/Dockerfile.jruby +1 -11
  8. data/README.md +1 -2
  9. data/SDK_VERSION +1 -1
  10. data/VERSION +1 -1
  11. data/ci/mssql/pom.xml +62 -0
  12. data/ci/mysql/pom.xml +57 -0
  13. data/ci/redshift/pom.xml +1 -1
  14. data/docker-compose.lcm.yml +0 -3
  15. data/gooddata.gemspec +2 -1
  16. data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
  17. data/lcm.rake +2 -8
  18. data/lib/gooddata/bricks/middleware/aws_middleware.rb +35 -9
  19. data/lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb +98 -0
  20. data/lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme +0 -0
  21. data/lib/gooddata/cloud_resources/mssql/mssql_client.rb +122 -0
  22. data/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme +0 -0
  23. data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +111 -0
  24. data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +0 -1
  25. data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +18 -1
  26. data/lib/gooddata/helpers/data_helper.rb +9 -4
  27. data/lib/gooddata/lcm/actions/collect_meta.rb +3 -1
  28. data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +3 -2
  29. data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -7
  30. data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +64 -0
  31. data/lib/gooddata/lcm/actions/synchronize_ldm.rb +19 -8
  32. data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +12 -9
  33. data/lib/gooddata/lcm/actions/update_metric_formats.rb +185 -0
  34. data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb +5 -0
  35. data/lib/gooddata/lcm/helpers/release_table_helper.rb +42 -8
  36. data/lib/gooddata/lcm/lcm2.rb +5 -0
  37. data/lib/gooddata/mixins/md_object_query.rb +1 -0
  38. data/lib/gooddata/models/data_source.rb +5 -1
  39. data/lib/gooddata/models/dataset_mapping.rb +36 -0
  40. data/lib/gooddata/models/metadata/label.rb +26 -27
  41. data/lib/gooddata/models/project.rb +34 -9
  42. data/lib/gooddata/models/schedule.rb +13 -1
  43. data/lib/gooddata/models/user_filters/user_filter_builder.rb +58 -53
  44. data/lib/gooddata/rest/phmap.rb +1 -0
  45. metadata +44 -18
  46. data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb +0 -37
@@ -0,0 +1,98 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'pathname'
10
+ require "azure/storage/blob"
11
+
12
+ module GoodData
13
+ class BlobStorageClient
14
+ SAS_URL_PATTERN = %r{(^https?:\/\/[^\/]*)\/.*\?(.*)}
15
+ INVALID_BLOB_GENERAL_MESSAGE = "The connection string is not valid."
16
+ INVALID_BLOB_SIG_WELL_FORMED_MESSAGE = "The signature format is not valid."
17
+ INVALID_BLOB_CONTAINER_MESSAGE = "ContainerNotFound"
18
+ INVALID_BLOB_CONTAINER_FORMED_MESSAGE = "The container with the specified name is not found."
19
+ INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE = "Signature not valid in the specified time frame"
20
+ INVALID_BLOB_EXPIRED_MESSAGE = "The signature expired."
21
+ INVALID_BLOB_INVALID_CONNECTION_STRING_MESSAGE = "The connection string is not valid."
22
+ INVALID_BLOB_PATH_MESSAGE = "BlobNotFound"
23
+ INVALID_BLOB_INVALID_PATH_MESSAGE = "The path to the data is not found."
24
+
25
+ attr_reader :use_sas
26
+
27
+ def initialize(options = {})
28
+ raise("Data Source needs a client to Blob Storage to be able to get blob file but 'blobStorage_client' is empty.") unless options['blobStorage_client']
29
+
30
+ if options['blobStorage_client']['connectionString'] && options['blobStorage_client']['container']
31
+ @connection_string = options['blobStorage_client']['connectionString']
32
+ @container = options['blobStorage_client']['container']
33
+ @path = options['blobStorage_client']['path']
34
+ @use_sas = false
35
+ build_sas(@connection_string)
36
+ else
37
+ raise('Missing connection info for Blob Storage client')
38
+ end
39
+ end
40
+
41
+ def realize_blob(file, _params)
42
+ GoodData.gd_logger.info("Realizing download from Blob Storage. Container #{@container}.")
43
+ filename = ''
44
+ begin
45
+ connect
46
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
47
+ blob_name = get_blob_name(@path, file)
48
+
49
+ measure = Benchmark.measure do
50
+ _blob, content = @client.get_blob(@container, blob_name)
51
+ File.open(filename, "wb") { |f| f.write(content) }
52
+ end
53
+ rescue => e
54
+ raise_error(e)
55
+ end
56
+ GoodData.gd_logger.info("Done downloading file type=blobStorage status=finished duration=#{measure.real}")
57
+ filename
58
+ end
59
+
60
+ def connect
61
+ GoodData.logger.info "Setting up connection to Blob Storage"
62
+ if use_sas
63
+ @client = Azure::Storage::Blob::BlobService.create(:storage_blob_host => @host, :storage_sas_token => @sas_token)
64
+ else
65
+ @client = Azure::Storage::Blob::BlobService.create_from_connection_string(@connection_string)
66
+ end
67
+ end
68
+
69
+ def build_sas(url)
70
+ matches = url.scan(SAS_URL_PATTERN)
71
+ return unless matches && matches[0]
72
+
73
+ @use_sas = true
74
+ @host = matches[0][0]
75
+ @sas_token = matches[0][1]
76
+ end
77
+
78
+ def raise_error(e)
79
+ if e.message && e.message.include?(INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE)
80
+ raise INVALID_BLOB_EXPIRED_MESSAGE
81
+ elsif e.message && e.message.include?(INVALID_BLOB_SIG_WELL_FORMED_MESSAGE)
82
+ raise INVALID_BLOB_SIG_WELL_FORMED_MESSAGE
83
+ elsif e.message && e.message.include?(INVALID_BLOB_CONTAINER_MESSAGE)
84
+ raise INVALID_BLOB_CONTAINER_FORMED_MESSAGE
85
+ elsif e.message && e.message.include?(INVALID_BLOB_PATH_MESSAGE)
86
+ raise INVALID_BLOB_INVALID_PATH_MESSAGE
87
+ else
88
+ raise INVALID_BLOB_GENERAL_MESSAGE
89
+ end
90
+ end
91
+
92
+ def get_blob_name(path, file)
93
+ return file unless path
94
+
95
+ path.rindex('/') == path.length - 1 ? "#{path}#{file}" : "#{path}/#{file}"
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,122 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'java'
10
+ require 'pathname'
11
+ require_relative '../cloud_resource_client'
12
+
13
+ base = Pathname(__FILE__).dirname.expand_path
14
+ Dir.glob(base + 'drivers/*.jar').each do |file|
15
+ require file unless file.start_with?('lcm-mssql-driver')
16
+ end
17
+
18
+ module GoodData
19
+ module CloudResources
20
+ class MSSQLClient < CloudResourceClient
21
+ MSSQL_SEPARATOR_PARAM = ";"
22
+ MSSQL_URL_PATTERN = %r{jdbc:sqlserver://([^:]+)(:([0-9]+))?(/)?}
23
+ MSSQL_DEFAULT_PORT = 1433
24
+ LOGIN_TIME_OUT = 30
25
+ MSSQL_FETCH_SIZE = 10_000
26
+ VERIFY_FULL = 'verify-full'
27
+ PREFER = 'prefer'
28
+ REQUIRE = 'require'
29
+
30
+ class << self
31
+ def accept?(type)
32
+ type == 'mssql'
33
+ end
34
+ end
35
+
36
+ def initialize(options = {})
37
+ raise("Data Source needs a client to MSSQL to be able to query the storage but 'mssql_client' is empty.") unless options['mssql_client']
38
+
39
+ connection = options['mssql_client']['connection']
40
+ if connection.is_a?(Hash)
41
+ @database = connection['database']
42
+ @schema = connection['schema']
43
+ @authentication = connection['authentication']
44
+ @ssl_mode = connection['sslMode']
45
+ @url = connection['url']
46
+
47
+ validate
48
+ else
49
+ raise('Missing connection info for MSSQL client')
50
+ end
51
+
52
+ Java.com.microsoft.sqlserver.jdbc.SQLServerDriver
53
+ end
54
+
55
+ def realize_query(query, _params)
56
+ GoodData.gd_logger.info("Realize SQL query: type=mssql status=started")
57
+
58
+ connect
59
+
60
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
61
+ measure = Benchmark.measure do
62
+ statement = @connection.create_statement
63
+ statement.set_fetch_size(MSSQL_FETCH_SIZE)
64
+ has_result = statement.execute(query)
65
+ if has_result
66
+ result = statement.get_result_set
67
+ metadata = result.get_meta_data
68
+ col_count = metadata.column_count
69
+ CSV.open(filename, 'wb') do |csv|
70
+ csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
71
+ csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
72
+ end
73
+ end
74
+ end
75
+
76
+ GoodData.gd_logger.info("Realize SQL query: type=mssql status=finished duration=#{measure.real}")
77
+ filename
78
+ ensure
79
+ @connection&.close
80
+ @connection = nil
81
+ end
82
+
83
+ def connect
84
+ connection_string = build_connection_string
85
+ GoodData.logger.info "Setting up connection to MSSQL #{connection_string}"
86
+
87
+ authentication = @authentication['basic'] || @authentication['activeDirectoryPassword']
88
+
89
+ prop = java.util.Properties.new
90
+ prop.setProperty('userName', authentication['userName'])
91
+ prop.setProperty('password', authentication['password'])
92
+
93
+ @connection = java.sql.DriverManager.getConnection(connection_string, prop)
94
+ end
95
+
96
+ def validate
97
+ raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
98
+
99
+ raise "Instance name is not supported" if @url !~ /^[^\\]*$/
100
+
101
+ raise "The connection url is invalid. Parameter is not supported." if @url.include? MSSQL_SEPARATOR_PARAM
102
+
103
+ url_matches = @url.scan(MSSQL_URL_PATTERN)
104
+ raise "Cannot reach the url" if url_matches.nil? || url_matches.length.zero?
105
+
106
+ raise "The authentication method is not supported." unless @authentication['basic'] || @authentication['activeDirectoryPassword']
107
+ end
108
+
109
+ def build_connection_string
110
+ encrypt = @ssl_mode != PREFER
111
+ trust_server_certificate = @ssl_mode == REQUIRE
112
+
113
+ "#{@url};" \
114
+ "database=#{@database};" \
115
+ "encrypt=#{encrypt};" \
116
+ "trustServerCertificate=#{trust_server_certificate};" \
117
+ "loginTimeout=#{LOGIN_TIME_OUT};" \
118
+ "#{'authentication=ActiveDirectoryPassword;' if @authentication['activeDirectoryPassword']}"
119
+ end
120
+ end
121
+ end
122
+ end
@@ -0,0 +1,111 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'java'
10
+ require 'pathname'
11
+ require_relative '../cloud_resource_client'
12
+
13
+ base = Pathname(__FILE__).dirname.expand_path
14
+ Dir.glob(base + 'drivers/*.jar').each do |file|
15
+ require file unless file.start_with?('lcm-mysql-driver')
16
+ end
17
+
18
+ module GoodData
19
+ module CloudResources
20
+ class MysqlClient < CloudResourceClient
21
+ JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
22
+ MYSQL_DEFAULT_PORT = 3306
23
+ JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://'
24
+ VERIFY_FULL = 'VERIFY_IDENTITY'
25
+ PREFER = 'PREFERRED'
26
+ REQUIRE = 'REQUIRED'
27
+ MYSQL_FETCH_SIZE = 1000
28
+
29
+ class << self
30
+ def accept?(type)
31
+ type == 'mysql'
32
+ end
33
+ end
34
+
35
+ def initialize(options = {})
36
+ raise("Data Source needs a client to Mysql to be able to query the storage but 'mysql_client' is empty.") unless options['mysql_client']
37
+
38
+ if options['mysql_client']['connection'].is_a?(Hash)
39
+ @database = options['mysql_client']['connection']['database']
40
+ @authentication = options['mysql_client']['connection']['authentication']
41
+ @ssl_mode = options['mysql_client']['connection']['sslMode']
42
+ raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
43
+
44
+ @url = build_url(options['mysql_client']['connection']['url'])
45
+ else
46
+ raise('Missing connection info for Mysql client')
47
+ end
48
+
49
+ Java.com.mysql.cj.jdbc.Driver
50
+ end
51
+
52
+ def realize_query(query, _params)
53
+ GoodData.gd_logger.info("Realize SQL query: type=mysql status=started")
54
+
55
+ connect
56
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
57
+ measure = Benchmark.measure do
58
+ statement = @connection.create_statement
59
+ statement.set_fetch_size(MYSQL_FETCH_SIZE)
60
+ has_result = statement.execute(query)
61
+ if has_result
62
+ result = statement.get_result_set
63
+ metadata = result.get_meta_data
64
+ col_count = metadata.column_count
65
+ CSV.open(filename, 'wb') do |csv|
66
+ csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
67
+ csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
68
+ end
69
+ end
70
+ end
71
+ GoodData.gd_logger.info("Realize SQL query: type=mysql status=finished duration=#{measure.real}")
72
+ filename
73
+ ensure
74
+ @connection&.close
75
+ @connection = nil
76
+ end
77
+
78
+ def connect
79
+ GoodData.logger.info "Setting up connection to Mysql #{@url}"
80
+
81
+ prop = java.util.Properties.new
82
+ prop.setProperty('user', @authentication['basic']['userName'])
83
+ prop.setProperty('password', @authentication['basic']['password'])
84
+
85
+ @connection = java.sql.DriverManager.getConnection(@url, prop)
86
+ @connection.set_auto_commit(false)
87
+ end
88
+
89
+ def build_url(url)
90
+ matches = url.scan(JDBC_MYSQL_PATTERN)
91
+ raise 'Cannot reach the url' unless matches
92
+
93
+ host = matches[0][0]
94
+ port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT
95
+
96
+ "#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{get_ssl_mode(@ssl_mode)}&useCursorFetch=true&enabledTLSProtocols=TLSv1.2"
97
+ end
98
+
99
+ def get_ssl_mode(ssl_mode)
100
+ mode = PREFER
101
+ if ssl_mode == 'verify-full'
102
+ mode = VERIFY_FULL
103
+ elsif ssl_mode == 'require'
104
+ mode = REQUIRE
105
+ end
106
+
107
+ mode
108
+ end
109
+ end
110
+ end
111
+ end
@@ -98,7 +98,6 @@ module GoodData
98
98
 
99
99
  host = matches[0][0]
100
100
  port = matches[0][2]&.to_i || POSTGRES_DEFAULT_PORT
101
- raise "Custom port #{port} is not supported. Remove it or use the default port '5432'" if POSTGRES_DEFAULT_PORT != port
102
101
 
103
102
  "#{JDBC_POSTGRES_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{@ssl_mode}#{VERIFY_FULL == @ssl_mode ? SSL_JAVA_FACTORY : ''}"
104
103
  end
@@ -18,6 +18,9 @@ end
18
18
  module GoodData
19
19
  module CloudResources
20
20
  class SnowflakeClient < CloudResourceClient
21
+ SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=GoodData_Platform'
22
+ SNOWFLAKE_SEPARATOR_PARAM = '?'
23
+
21
24
  class << self
22
25
  def accept?(type)
23
26
  type == 'snowflake'
@@ -31,7 +34,7 @@ module GoodData
31
34
  @database = options['snowflake_client']['connection']['database']
32
35
  @schema = options['snowflake_client']['connection']['schema'] || 'public'
33
36
  @warehouse = options['snowflake_client']['connection']['warehouse']
34
- @url = options['snowflake_client']['connection']['url']
37
+ @url = build_url(options['snowflake_client']['connection']['url'])
35
38
  @authentication = options['snowflake_client']['connection']['authentication']
36
39
  else
37
40
  raise('Missing connection info for Snowflake client')
@@ -79,6 +82,20 @@ module GoodData
79
82
 
80
83
  @connection = java.sql.DriverManager.getConnection(@url, prop)
81
84
  end
85
+
86
+ def build_url(url)
87
+ is_contain = url.include?(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
88
+ unless is_contain
89
+ if url.include?(SNOWFLAKE_SEPARATOR_PARAM)
90
+ url.concat("&")
91
+ else
92
+ url.concat(SNOWFLAKE_SEPARATOR_PARAM)
93
+ end
94
+ url.concat(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
95
+ end
96
+
97
+ url
98
+ end
82
99
  end
83
100
  end
84
101
  end
@@ -44,10 +44,14 @@ module GoodData
44
44
  realize_link
45
45
  when 's3'
46
46
  realize_s3(params)
47
- when 'redshift', 'snowflake', 'bigquery', 'postgresql'
47
+ when 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql'
48
48
  raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
49
49
  require_relative '../cloud_resources/cloud_resources'
50
50
  realize_cloud_resource(source, params)
51
+ when 'blobStorage'
52
+ require_relative '../cloud_resources/blobstorage/blobstorage_client'
53
+ blob_storage_client = GoodData::BlobStorageClient.new(params)
54
+ blob_storage_client.realize_blob(@options[:file], params)
51
55
  else
52
56
  raise "DataSource does not support type \"#{source}\""
53
57
  end
@@ -111,12 +115,13 @@ module GoodData
111
115
  end
112
116
 
113
117
  def realize_s3(params)
114
- s3_client = params['aws_client'] && params['aws_client']['s3_client']
118
+ s3_client = params['s3_client'] && params['s3_client']['client']
115
119
  raise 'AWS client not present. Perhaps S3Middleware is missing in the brick definition?' if !s3_client || !s3_client.respond_to?(:bucket)
116
120
  bucket_name = @options[:bucket]
117
- key = @options[:key]
121
+ key = @options[:key].present? ? @options[:key] : @options[:file]
118
122
  raise 'Key "bucket" is missing in S3 datasource' if bucket_name.blank?
119
- raise 'Key "key" is missing in S3 datasource' if key.blank?
123
+ raise 'Key "key" or "file" is missing in S3 datasource' if key.blank?
124
+
120
125
  GoodData.logger.info("Realizing download from S3. Bucket #{bucket_name}, object with key #{key}.")
121
126
  filename = Digest::SHA256.new.hexdigest(@options.to_json)
122
127
  bucket = s3_client.bucket(bucket_name)
@@ -53,7 +53,9 @@ module GoodData
53
53
  client: development_client
54
54
  )
55
55
  kpi_dashboards = MdObject.query('analyticalDashboard', MdObject, client: development_client, project: from_project)
56
- objects = old_dashboards.to_a + kpi_dashboards.to_a
56
+ kpi_dashboard_plugin = MdObject.query('dashboardPlugin', MdObject, client: development_client, project: from_project)
57
+ kpi_date_filter_config = MdObject.query('dateFilterConfig', MdObject, client: development_client, project: from_project)
58
+ objects = old_dashboards.to_a + kpi_dashboards.to_a + kpi_dashboard_plugin.to_a + kpi_date_filter_config.to_a
57
59
  else
58
60
  objects = GoodData::Dashboard.find_by_tag(
59
61
  production_tags,
@@ -50,6 +50,7 @@ module GoodData
50
50
  segment_info[:to].pmap do |entry|
51
51
  pid = entry[:pid]
52
52
  to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
53
+ GoodData.logger.info "Migrating date dimension, project: '#{to_project.title}', PID: #{pid}"
53
54
  to_blueprint = to_project.blueprint
54
55
  upgrade_datasets = get_upgrade_dates(latest_blueprint, to_blueprint)
55
56
  next if upgrade_datasets.empty?
@@ -71,9 +72,9 @@ module GoodData
71
72
  dest_dates = get_date_dimensions(dest_blueprint) if dest_blueprint
72
73
  src_dates = get_date_dimensions(src_blueprint) if src_blueprint
73
74
 
74
- return false if dest_dates.empty? || src_dates.empty?
75
-
76
75
  upgrade_datasets = []
76
+ return upgrade_datasets if dest_dates.empty? || src_dates.empty?
77
+
77
78
  dest_dates.each do |dest|
78
79
  src_dim = get_date_dimension(src_blueprint, dest[:id])
79
80
  next unless src_dim
@@ -33,6 +33,9 @@ module GoodData
33
33
  description 'ADS Client'
34
34
  param :ads_client, instance_of(Type::AdsClientType), required: false
35
35
 
36
+ description 'Keep number of old master workspace excluding the latest one'
37
+ param :keep_only_previous_masters_count, instance_of(Type::StringType), required: false, default: '-1'
38
+
36
39
  description 'Additional Hidden Parameters'
37
40
  param :additional_hidden_params, instance_of(Type::HashType), required: false
38
41
  end
@@ -53,6 +56,7 @@ module GoodData
53
56
  domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
54
57
  data_product = params.data_product
55
58
  domain_segments = domain.segments(:all, data_product)
59
+ keep_only_previous_masters_count = Integer(params.keep_only_previous_masters_count || "-1")
56
60
 
57
61
  segments = params.segments.map do |seg|
58
62
  domain_segments.find do |s|
@@ -62,18 +66,14 @@ module GoodData
62
66
 
63
67
  results = segments.map do |segment|
64
68
  if params.ads_client
65
- current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
66
- params.release_table_name,
67
- params.ads_client,
68
- segment.segment_id
69
- )
69
+ master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_ads(params.release_table_name, params.ads_client, segment.segment_id)
70
70
  else
71
- current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
71
+ master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
72
72
  end
73
73
 
74
+ current_master = master_projects.last
74
75
  # TODO: Check res.first.nil? || res.first[:master_project_id].nil?
75
76
  master = client.projects(current_master[:master_project_id])
76
-
77
77
  segment.master_project = master
78
78
  segment.save
79
79
 
@@ -87,6 +87,19 @@ module GoodData
87
87
  "Details: #{sync_result['links']['details']}")
88
88
  end
89
89
 
90
+ if keep_only_previous_masters_count >= 0
91
+ number_of_deleted_projects = master_projects.count - (keep_only_previous_masters_count + 1)
92
+
93
+ if number_of_deleted_projects.positive?
94
+ begin
95
+ removal_master_project_ids = remove_multiple_workspace(params, segment.segment_id, master_projects, number_of_deleted_projects)
96
+ remove_old_workspaces_from_release_table(params, domain_name, data_product.data_product_id, segment.segment_id, master_projects, removal_master_project_ids)
97
+ rescue Exception => e # rubocop:disable RescueException
98
+ GoodData.logger.error "Problem occurs when removing old master workspace, reason: #{e.message}"
99
+ end
100
+ end
101
+ end
102
+
90
103
  {
91
104
  segment: segment.id,
92
105
  master_pid: master.pid,
@@ -98,6 +111,42 @@ module GoodData
98
111
  # Return results
99
112
  results
100
113
  end
114
+
115
+ def remove_multiple_workspace(params, segment_id, master_projects, number_of_deleted_projects)
116
+ removal_master_project_ids = []
117
+ need_to_delete_projects = master_projects.take(number_of_deleted_projects)
118
+
119
+ need_to_delete_projects.each do |project_wrapper|
120
+ master_project_id = project_wrapper[:master_project_id]
121
+ next if master_project_id.to_s.empty?
122
+
123
+ begin
124
+ project = params.gdc_gd_client.projects(master_project_id)
125
+ if project && !%w[deleted archived].include?(project.state.to_s)
126
+ GoodData.logger.info "Segment #{segment_id}: Deleting old master workspace, project: '#{project.title}', PID: (#{project.pid})."
127
+ project.delete
128
+ end
129
+ removal_master_project_ids << master_project_id
130
+ master_projects.delete_if { |p| p[:master_project_id] == master_project_id }
131
+ rescue Exception => ex # rubocop:disable RescueException
132
+ GoodData.logger.error "Unable to remove master workspace: '#{master_project_id}', Error: #{ex.message}"
133
+ end
134
+ end
135
+ removal_master_project_ids
136
+ end
137
+
138
+ # rubocop:disable Metrics/ParameterLists
139
+ def remove_old_workspaces_from_release_table(params, domain_id, data_product_id, segment_id, master_projects, removal_master_project_ids)
140
+ unless removal_master_project_ids.empty?
141
+ if params.ads_client
142
+ GoodData::LCM2::Helpers.delete_master_project_from_ads(params.release_table_name, params.ads_client, segment_id, removal_master_project_ids)
143
+ else
144
+ data = master_projects.sort_by { |master| master[:version] }
145
+ GoodData::LCM2::Helpers.update_master_project_to_nfs(domain_id, data_product_id, segment_id, data)
146
+ end
147
+ end
148
+ end
149
+ # rubocop:enable Metrics/ParameterLists
101
150
  end
102
151
  end
103
152
  end
@@ -0,0 +1,64 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require_relative 'base_action'
9
+
10
+ module GoodData
11
+ module LCM2
12
+ class SynchronizeDataSetMapping < BaseAction
13
+ DESCRIPTION = 'Synchronize Dataset Mappings'
14
+
15
+ PARAMS = define_params(self) do
16
+ description 'Client Used for Connecting to GD'
17
+ param :gdc_gd_client, instance_of(Type::GdClientType), required: true
18
+
19
+ description 'Client used to connecting to development domain'
20
+ param :development_client, instance_of(Type::GdClientType), required: true
21
+
22
+ description 'Synchronization Info'
23
+ param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
24
+
25
+ description 'Logger'
26
+ param :gdc_logger, instance_of(Type::GdLogger), required: true
27
+ end
28
+
29
+ RESULT_HEADER = %i[from to count status]
30
+
31
+ class << self
32
+ def call(params)
33
+ results = []
34
+
35
+ client = params.gdc_gd_client
36
+ development_client = params.development_client
37
+
38
+ params.synchronize.peach do |info|
39
+ from_project = info.from
40
+ to_projects = info.to
41
+
42
+ from = development_client.projects(from_project) || fail("Invalid 'from' project specified - '#{from_project}'")
43
+ dataset_mapping = from.dataset_mapping
44
+ if dataset_mapping&.dig('datasetMappings', 'items').nil? || dataset_mapping['datasetMappings']['items'].empty?
45
+ params.gdc_logger.info "Project: '#{from.title}', PID: '#{from.pid}' has no model mapping, skip synchronizing model mapping."
46
+ else
47
+ to_projects.peach do |to|
48
+ pid = to[:pid]
49
+ to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
50
+
51
+ params.gdc_logger.info "Transferring model mapping, from project: '#{from.title}', PID: '#{from.pid}', to project: '#{to_project.title}', PID: '#{to_project.pid}'"
52
+ res = to_project.update_dataset_mapping(dataset_mapping)
53
+ res[:from] = from.pid
54
+ results << res
55
+ end
56
+ end
57
+ end
58
+ # Return results
59
+ results.flatten
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end
@@ -83,18 +83,29 @@ module GoodData
83
83
  segment_info[:from_blueprint] = blueprint
84
84
  maql_diff = nil
85
85
  previous_master = segment_info[:previous_master]
86
+ synchronize_ldm_mode = params[:synchronize_ldm].downcase
86
87
  diff_against_master = %w(diff_against_master_with_fallback diff_against_master)
87
- .include?(params[:synchronize_ldm].downcase)
88
- GoodData.logger.info "Synchronize LDM mode: '#{params[:synchronize_ldm].downcase}'"
89
- if previous_master && diff_against_master
90
- maql_diff_params = [:includeGrain]
91
- maql_diff_params << :excludeFactRule if exclude_fact_rule
92
- maql_diff_params << :includeDeprecated if include_deprecated
93
- maql_diff = previous_master.maql_diff(blueprint: blueprint, params: maql_diff_params)
88
+ .include?(synchronize_ldm_mode)
89
+ GoodData.logger.info "Synchronize LDM mode: '#{synchronize_ldm_mode}'"
90
+ if segment_info.key?(:previous_master) && diff_against_master
91
+ if previous_master
92
+ maql_diff_params = [:includeGrain]
93
+ maql_diff_params << :excludeFactRule if exclude_fact_rule
94
+ maql_diff_params << :includeDeprecated if include_deprecated
95
+ maql_diff = previous_master.maql_diff(blueprint: blueprint, params: maql_diff_params)
96
+ else
97
+ maql_diff = {
98
+ "projectModelDiff" =>
99
+ {
100
+ "updateOperations" => [],
101
+ "updateScripts" => []
102
+ }
103
+ }
104
+ end
94
105
  chunks = maql_diff['projectModelDiff']['updateScripts']
95
106
  if chunks.empty?
96
107
  GoodData.logger.info "Synchronize LDM to clients will not proceed in mode \
97
- '#{params[:synchronize_ldm].downcase}' due to no LDM changes in the new master project. \
108
+ '#{synchronize_ldm_mode}' due to no LDM changes in the segment master project. \
98
109
  If you had changed LDM of clients manually, please use mode 'diff_against_clients' \
99
110
  to force synchronize LDM to clients"
100
111
  end