gooddata 2.1.19-java → 2.2.0-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (46) hide show
  1. checksums.yaml +5 -5
  2. data/.gdc-ii-config.yaml +1 -1
  3. data/.github/workflows/build.yml +66 -0
  4. data/.github/workflows/pre-merge.yml +72 -0
  5. data/CHANGELOG.md +38 -0
  6. data/Dockerfile +21 -14
  7. data/Dockerfile.jruby +1 -11
  8. data/README.md +1 -2
  9. data/SDK_VERSION +1 -1
  10. data/VERSION +1 -1
  11. data/ci/mssql/pom.xml +62 -0
  12. data/ci/mysql/pom.xml +57 -0
  13. data/ci/redshift/pom.xml +1 -1
  14. data/docker-compose.lcm.yml +0 -3
  15. data/gooddata.gemspec +2 -1
  16. data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
  17. data/lcm.rake +2 -8
  18. data/lib/gooddata/bricks/middleware/aws_middleware.rb +35 -9
  19. data/lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb +98 -0
  20. data/lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme +0 -0
  21. data/lib/gooddata/cloud_resources/mssql/mssql_client.rb +122 -0
  22. data/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme +0 -0
  23. data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +111 -0
  24. data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +0 -1
  25. data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +18 -1
  26. data/lib/gooddata/helpers/data_helper.rb +9 -4
  27. data/lib/gooddata/lcm/actions/collect_meta.rb +3 -1
  28. data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +3 -2
  29. data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -7
  30. data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +64 -0
  31. data/lib/gooddata/lcm/actions/synchronize_ldm.rb +19 -8
  32. data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +12 -9
  33. data/lib/gooddata/lcm/actions/update_metric_formats.rb +185 -0
  34. data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb +5 -0
  35. data/lib/gooddata/lcm/helpers/release_table_helper.rb +42 -8
  36. data/lib/gooddata/lcm/lcm2.rb +5 -0
  37. data/lib/gooddata/mixins/md_object_query.rb +1 -0
  38. data/lib/gooddata/models/data_source.rb +5 -1
  39. data/lib/gooddata/models/dataset_mapping.rb +36 -0
  40. data/lib/gooddata/models/metadata/label.rb +26 -27
  41. data/lib/gooddata/models/project.rb +34 -9
  42. data/lib/gooddata/models/schedule.rb +13 -1
  43. data/lib/gooddata/models/user_filters/user_filter_builder.rb +58 -53
  44. data/lib/gooddata/rest/phmap.rb +1 -0
  45. metadata +44 -18
  46. data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb +0 -37
@@ -0,0 +1,98 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'pathname'
10
+ require "azure/storage/blob"
11
+
12
+ module GoodData
13
+ class BlobStorageClient
14
+ SAS_URL_PATTERN = %r{(^https?:\/\/[^\/]*)\/.*\?(.*)}
15
+ INVALID_BLOB_GENERAL_MESSAGE = "The connection string is not valid."
16
+ INVALID_BLOB_SIG_WELL_FORMED_MESSAGE = "The signature format is not valid."
17
+ INVALID_BLOB_CONTAINER_MESSAGE = "ContainerNotFound"
18
+ INVALID_BLOB_CONTAINER_FORMED_MESSAGE = "The container with the specified name is not found."
19
+ INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE = "Signature not valid in the specified time frame"
20
+ INVALID_BLOB_EXPIRED_MESSAGE = "The signature expired."
21
+ INVALID_BLOB_INVALID_CONNECTION_STRING_MESSAGE = "The connection string is not valid."
22
+ INVALID_BLOB_PATH_MESSAGE = "BlobNotFound"
23
+ INVALID_BLOB_INVALID_PATH_MESSAGE = "The path to the data is not found."
24
+
25
+ attr_reader :use_sas
26
+
27
+ def initialize(options = {})
28
+ raise("Data Source needs a client to Blob Storage to be able to get blob file but 'blobStorage_client' is empty.") unless options['blobStorage_client']
29
+
30
+ if options['blobStorage_client']['connectionString'] && options['blobStorage_client']['container']
31
+ @connection_string = options['blobStorage_client']['connectionString']
32
+ @container = options['blobStorage_client']['container']
33
+ @path = options['blobStorage_client']['path']
34
+ @use_sas = false
35
+ build_sas(@connection_string)
36
+ else
37
+ raise('Missing connection info for Blob Storage client')
38
+ end
39
+ end
40
+
41
+ def realize_blob(file, _params)
42
+ GoodData.gd_logger.info("Realizing download from Blob Storage. Container #{@container}.")
43
+ filename = ''
44
+ begin
45
+ connect
46
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
47
+ blob_name = get_blob_name(@path, file)
48
+
49
+ measure = Benchmark.measure do
50
+ _blob, content = @client.get_blob(@container, blob_name)
51
+ File.open(filename, "wb") { |f| f.write(content) }
52
+ end
53
+ rescue => e
54
+ raise_error(e)
55
+ end
56
+ GoodData.gd_logger.info("Done downloading file type=blobStorage status=finished duration=#{measure.real}")
57
+ filename
58
+ end
59
+
60
+ def connect
61
+ GoodData.logger.info "Setting up connection to Blob Storage"
62
+ if use_sas
63
+ @client = Azure::Storage::Blob::BlobService.create(:storage_blob_host => @host, :storage_sas_token => @sas_token)
64
+ else
65
+ @client = Azure::Storage::Blob::BlobService.create_from_connection_string(@connection_string)
66
+ end
67
+ end
68
+
69
+ def build_sas(url)
70
+ matches = url.scan(SAS_URL_PATTERN)
71
+ return unless matches && matches[0]
72
+
73
+ @use_sas = true
74
+ @host = matches[0][0]
75
+ @sas_token = matches[0][1]
76
+ end
77
+
78
+ def raise_error(e)
79
+ if e.message && e.message.include?(INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE)
80
+ raise INVALID_BLOB_EXPIRED_MESSAGE
81
+ elsif e.message && e.message.include?(INVALID_BLOB_SIG_WELL_FORMED_MESSAGE)
82
+ raise INVALID_BLOB_SIG_WELL_FORMED_MESSAGE
83
+ elsif e.message && e.message.include?(INVALID_BLOB_CONTAINER_MESSAGE)
84
+ raise INVALID_BLOB_CONTAINER_FORMED_MESSAGE
85
+ elsif e.message && e.message.include?(INVALID_BLOB_PATH_MESSAGE)
86
+ raise INVALID_BLOB_INVALID_PATH_MESSAGE
87
+ else
88
+ raise INVALID_BLOB_GENERAL_MESSAGE
89
+ end
90
+ end
91
+
92
+ def get_blob_name(path, file)
93
+ return file unless path
94
+
95
+ path.rindex('/') == path.length - 1 ? "#{path}#{file}" : "#{path}/#{file}"
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,122 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'java'
10
+ require 'pathname'
11
+ require_relative '../cloud_resource_client'
12
+
13
+ base = Pathname(__FILE__).dirname.expand_path
14
+ Dir.glob(base + 'drivers/*.jar').each do |file|
15
+ require file unless file.start_with?('lcm-mssql-driver')
16
+ end
17
+
18
+ module GoodData
19
+ module CloudResources
20
+ class MSSQLClient < CloudResourceClient
21
+ MSSQL_SEPARATOR_PARAM = ";"
22
+ MSSQL_URL_PATTERN = %r{jdbc:sqlserver://([^:]+)(:([0-9]+))?(/)?}
23
+ MSSQL_DEFAULT_PORT = 1433
24
+ LOGIN_TIME_OUT = 30
25
+ MSSQL_FETCH_SIZE = 10_000
26
+ VERIFY_FULL = 'verify-full'
27
+ PREFER = 'prefer'
28
+ REQUIRE = 'require'
29
+
30
+ class << self
31
+ def accept?(type)
32
+ type == 'mssql'
33
+ end
34
+ end
35
+
36
+ def initialize(options = {})
37
+ raise("Data Source needs a client to MSSQL to be able to query the storage but 'mssql_client' is empty.") unless options['mssql_client']
38
+
39
+ connection = options['mssql_client']['connection']
40
+ if connection.is_a?(Hash)
41
+ @database = connection['database']
42
+ @schema = connection['schema']
43
+ @authentication = connection['authentication']
44
+ @ssl_mode = connection['sslMode']
45
+ @url = connection['url']
46
+
47
+ validate
48
+ else
49
+ raise('Missing connection info for MSSQL client')
50
+ end
51
+
52
+ Java.com.microsoft.sqlserver.jdbc.SQLServerDriver
53
+ end
54
+
55
+ def realize_query(query, _params)
56
+ GoodData.gd_logger.info("Realize SQL query: type=mssql status=started")
57
+
58
+ connect
59
+
60
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
61
+ measure = Benchmark.measure do
62
+ statement = @connection.create_statement
63
+ statement.set_fetch_size(MSSQL_FETCH_SIZE)
64
+ has_result = statement.execute(query)
65
+ if has_result
66
+ result = statement.get_result_set
67
+ metadata = result.get_meta_data
68
+ col_count = metadata.column_count
69
+ CSV.open(filename, 'wb') do |csv|
70
+ csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
71
+ csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
72
+ end
73
+ end
74
+ end
75
+
76
+ GoodData.gd_logger.info("Realize SQL query: type=mssql status=finished duration=#{measure.real}")
77
+ filename
78
+ ensure
79
+ @connection&.close
80
+ @connection = nil
81
+ end
82
+
83
+ def connect
84
+ connection_string = build_connection_string
85
+ GoodData.logger.info "Setting up connection to MSSQL #{connection_string}"
86
+
87
+ authentication = @authentication['basic'] || @authentication['activeDirectoryPassword']
88
+
89
+ prop = java.util.Properties.new
90
+ prop.setProperty('userName', authentication['userName'])
91
+ prop.setProperty('password', authentication['password'])
92
+
93
+ @connection = java.sql.DriverManager.getConnection(connection_string, prop)
94
+ end
95
+
96
+ def validate
97
+ raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
98
+
99
+ raise "Instance name is not supported" if @url !~ /^[^\\]*$/
100
+
101
+ raise "The connection url is invalid. Parameter is not supported." if @url.include? MSSQL_SEPARATOR_PARAM
102
+
103
+ url_matches = @url.scan(MSSQL_URL_PATTERN)
104
+ raise "Cannot reach the url" if url_matches.nil? || url_matches.length.zero?
105
+
106
+ raise "The authentication method is not supported." unless @authentication['basic'] || @authentication['activeDirectoryPassword']
107
+ end
108
+
109
+ def build_connection_string
110
+ encrypt = @ssl_mode != PREFER
111
+ trust_server_certificate = @ssl_mode == REQUIRE
112
+
113
+ "#{@url};" \
114
+ "database=#{@database};" \
115
+ "encrypt=#{encrypt};" \
116
+ "trustServerCertificate=#{trust_server_certificate};" \
117
+ "loginTimeout=#{LOGIN_TIME_OUT};" \
118
+ "#{'authentication=ActiveDirectoryPassword;' if @authentication['activeDirectoryPassword']}"
119
+ end
120
+ end
121
+ end
122
+ end
@@ -0,0 +1,111 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'java'
10
+ require 'pathname'
11
+ require_relative '../cloud_resource_client'
12
+
13
+ base = Pathname(__FILE__).dirname.expand_path
14
+ Dir.glob(base + 'drivers/*.jar').each do |file|
15
+ require file unless file.start_with?('lcm-mysql-driver')
16
+ end
17
+
18
+ module GoodData
19
+ module CloudResources
20
+ class MysqlClient < CloudResourceClient
21
+ JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
22
+ MYSQL_DEFAULT_PORT = 3306
23
+ JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://'
24
+ VERIFY_FULL = 'VERIFY_IDENTITY'
25
+ PREFER = 'PREFERRED'
26
+ REQUIRE = 'REQUIRED'
27
+ MYSQL_FETCH_SIZE = 1000
28
+
29
+ class << self
30
+ def accept?(type)
31
+ type == 'mysql'
32
+ end
33
+ end
34
+
35
+ def initialize(options = {})
36
+ raise("Data Source needs a client to Mysql to be able to query the storage but 'mysql_client' is empty.") unless options['mysql_client']
37
+
38
+ if options['mysql_client']['connection'].is_a?(Hash)
39
+ @database = options['mysql_client']['connection']['database']
40
+ @authentication = options['mysql_client']['connection']['authentication']
41
+ @ssl_mode = options['mysql_client']['connection']['sslMode']
42
+ raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
43
+
44
+ @url = build_url(options['mysql_client']['connection']['url'])
45
+ else
46
+ raise('Missing connection info for Mysql client')
47
+ end
48
+
49
+ Java.com.mysql.cj.jdbc.Driver
50
+ end
51
+
52
+ def realize_query(query, _params)
53
+ GoodData.gd_logger.info("Realize SQL query: type=mysql status=started")
54
+
55
+ connect
56
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
57
+ measure = Benchmark.measure do
58
+ statement = @connection.create_statement
59
+ statement.set_fetch_size(MYSQL_FETCH_SIZE)
60
+ has_result = statement.execute(query)
61
+ if has_result
62
+ result = statement.get_result_set
63
+ metadata = result.get_meta_data
64
+ col_count = metadata.column_count
65
+ CSV.open(filename, 'wb') do |csv|
66
+ csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
67
+ csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
68
+ end
69
+ end
70
+ end
71
+ GoodData.gd_logger.info("Realize SQL query: type=mysql status=finished duration=#{measure.real}")
72
+ filename
73
+ ensure
74
+ @connection&.close
75
+ @connection = nil
76
+ end
77
+
78
+ def connect
79
+ GoodData.logger.info "Setting up connection to Mysql #{@url}"
80
+
81
+ prop = java.util.Properties.new
82
+ prop.setProperty('user', @authentication['basic']['userName'])
83
+ prop.setProperty('password', @authentication['basic']['password'])
84
+
85
+ @connection = java.sql.DriverManager.getConnection(@url, prop)
86
+ @connection.set_auto_commit(false)
87
+ end
88
+
89
+ def build_url(url)
90
+ matches = url.scan(JDBC_MYSQL_PATTERN)
91
+ raise 'Cannot reach the url' unless matches
92
+
93
+ host = matches[0][0]
94
+ port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT
95
+
96
+ "#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{get_ssl_mode(@ssl_mode)}&useCursorFetch=true&enabledTLSProtocols=TLSv1.2"
97
+ end
98
+
99
+ def get_ssl_mode(ssl_mode)
100
+ mode = PREFER
101
+ if ssl_mode == 'verify-full'
102
+ mode = VERIFY_FULL
103
+ elsif ssl_mode == 'require'
104
+ mode = REQUIRE
105
+ end
106
+
107
+ mode
108
+ end
109
+ end
110
+ end
111
+ end
@@ -98,7 +98,6 @@ module GoodData
98
98
 
99
99
  host = matches[0][0]
100
100
  port = matches[0][2]&.to_i || POSTGRES_DEFAULT_PORT
101
- raise "Custom port #{port} is not supported. Remove it or use the default port '5432'" if POSTGRES_DEFAULT_PORT != port
102
101
 
103
102
  "#{JDBC_POSTGRES_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{@ssl_mode}#{VERIFY_FULL == @ssl_mode ? SSL_JAVA_FACTORY : ''}"
104
103
  end
@@ -18,6 +18,9 @@ end
18
18
  module GoodData
19
19
  module CloudResources
20
20
  class SnowflakeClient < CloudResourceClient
21
+ SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=GoodData_Platform'
22
+ SNOWFLAKE_SEPARATOR_PARAM = '?'
23
+
21
24
  class << self
22
25
  def accept?(type)
23
26
  type == 'snowflake'
@@ -31,7 +34,7 @@ module GoodData
31
34
  @database = options['snowflake_client']['connection']['database']
32
35
  @schema = options['snowflake_client']['connection']['schema'] || 'public'
33
36
  @warehouse = options['snowflake_client']['connection']['warehouse']
34
- @url = options['snowflake_client']['connection']['url']
37
+ @url = build_url(options['snowflake_client']['connection']['url'])
35
38
  @authentication = options['snowflake_client']['connection']['authentication']
36
39
  else
37
40
  raise('Missing connection info for Snowflake client')
@@ -79,6 +82,20 @@ module GoodData
79
82
 
80
83
  @connection = java.sql.DriverManager.getConnection(@url, prop)
81
84
  end
85
+
86
+ def build_url(url)
87
+ is_contain = url.include?(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
88
+ unless is_contain
89
+ if url.include?(SNOWFLAKE_SEPARATOR_PARAM)
90
+ url.concat("&")
91
+ else
92
+ url.concat(SNOWFLAKE_SEPARATOR_PARAM)
93
+ end
94
+ url.concat(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
95
+ end
96
+
97
+ url
98
+ end
82
99
  end
83
100
  end
84
101
  end
@@ -44,10 +44,14 @@ module GoodData
44
44
  realize_link
45
45
  when 's3'
46
46
  realize_s3(params)
47
- when 'redshift', 'snowflake', 'bigquery', 'postgresql'
47
+ when 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql'
48
48
  raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
49
49
  require_relative '../cloud_resources/cloud_resources'
50
50
  realize_cloud_resource(source, params)
51
+ when 'blobStorage'
52
+ require_relative '../cloud_resources/blobstorage/blobstorage_client'
53
+ blob_storage_client = GoodData::BlobStorageClient.new(params)
54
+ blob_storage_client.realize_blob(@options[:file], params)
51
55
  else
52
56
  raise "DataSource does not support type \"#{source}\""
53
57
  end
@@ -111,12 +115,13 @@ module GoodData
111
115
  end
112
116
 
113
117
  def realize_s3(params)
114
- s3_client = params['aws_client'] && params['aws_client']['s3_client']
118
+ s3_client = params['s3_client'] && params['s3_client']['client']
115
119
  raise 'AWS client not present. Perhaps S3Middleware is missing in the brick definition?' if !s3_client || !s3_client.respond_to?(:bucket)
116
120
  bucket_name = @options[:bucket]
117
- key = @options[:key]
121
+ key = @options[:key].present? ? @options[:key] : @options[:file]
118
122
  raise 'Key "bucket" is missing in S3 datasource' if bucket_name.blank?
119
- raise 'Key "key" is missing in S3 datasource' if key.blank?
123
+ raise 'Key "key" or "file" is missing in S3 datasource' if key.blank?
124
+
120
125
  GoodData.logger.info("Realizing download from S3. Bucket #{bucket_name}, object with key #{key}.")
121
126
  filename = Digest::SHA256.new.hexdigest(@options.to_json)
122
127
  bucket = s3_client.bucket(bucket_name)
@@ -53,7 +53,9 @@ module GoodData
53
53
  client: development_client
54
54
  )
55
55
  kpi_dashboards = MdObject.query('analyticalDashboard', MdObject, client: development_client, project: from_project)
56
- objects = old_dashboards.to_a + kpi_dashboards.to_a
56
+ kpi_dashboard_plugin = MdObject.query('dashboardPlugin', MdObject, client: development_client, project: from_project)
57
+ kpi_date_filter_config = MdObject.query('dateFilterConfig', MdObject, client: development_client, project: from_project)
58
+ objects = old_dashboards.to_a + kpi_dashboards.to_a + kpi_dashboard_plugin.to_a + kpi_date_filter_config.to_a
57
59
  else
58
60
  objects = GoodData::Dashboard.find_by_tag(
59
61
  production_tags,
@@ -50,6 +50,7 @@ module GoodData
50
50
  segment_info[:to].pmap do |entry|
51
51
  pid = entry[:pid]
52
52
  to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
53
+ GoodData.logger.info "Migrating date dimension, project: '#{to_project.title}', PID: #{pid}"
53
54
  to_blueprint = to_project.blueprint
54
55
  upgrade_datasets = get_upgrade_dates(latest_blueprint, to_blueprint)
55
56
  next if upgrade_datasets.empty?
@@ -71,9 +72,9 @@ module GoodData
71
72
  dest_dates = get_date_dimensions(dest_blueprint) if dest_blueprint
72
73
  src_dates = get_date_dimensions(src_blueprint) if src_blueprint
73
74
 
74
- return false if dest_dates.empty? || src_dates.empty?
75
-
76
75
  upgrade_datasets = []
76
+ return upgrade_datasets if dest_dates.empty? || src_dates.empty?
77
+
77
78
  dest_dates.each do |dest|
78
79
  src_dim = get_date_dimension(src_blueprint, dest[:id])
79
80
  next unless src_dim
@@ -33,6 +33,9 @@ module GoodData
33
33
  description 'ADS Client'
34
34
  param :ads_client, instance_of(Type::AdsClientType), required: false
35
35
 
36
+ description 'Keep number of old master workspace excluding the latest one'
37
+ param :keep_only_previous_masters_count, instance_of(Type::StringType), required: false, default: '-1'
38
+
36
39
  description 'Additional Hidden Parameters'
37
40
  param :additional_hidden_params, instance_of(Type::HashType), required: false
38
41
  end
@@ -53,6 +56,7 @@ module GoodData
53
56
  domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
54
57
  data_product = params.data_product
55
58
  domain_segments = domain.segments(:all, data_product)
59
+ keep_only_previous_masters_count = Integer(params.keep_only_previous_masters_count || "-1")
56
60
 
57
61
  segments = params.segments.map do |seg|
58
62
  domain_segments.find do |s|
@@ -62,18 +66,14 @@ module GoodData
62
66
 
63
67
  results = segments.map do |segment|
64
68
  if params.ads_client
65
- current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
66
- params.release_table_name,
67
- params.ads_client,
68
- segment.segment_id
69
- )
69
+ master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_ads(params.release_table_name, params.ads_client, segment.segment_id)
70
70
  else
71
- current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
71
+ master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
72
72
  end
73
73
 
74
+ current_master = master_projects.last
74
75
  # TODO: Check res.first.nil? || res.first[:master_project_id].nil?
75
76
  master = client.projects(current_master[:master_project_id])
76
-
77
77
  segment.master_project = master
78
78
  segment.save
79
79
 
@@ -87,6 +87,19 @@ module GoodData
87
87
  "Details: #{sync_result['links']['details']}")
88
88
  end
89
89
 
90
+ if keep_only_previous_masters_count >= 0
91
+ number_of_deleted_projects = master_projects.count - (keep_only_previous_masters_count + 1)
92
+
93
+ if number_of_deleted_projects.positive?
94
+ begin
95
+ removal_master_project_ids = remove_multiple_workspace(params, segment.segment_id, master_projects, number_of_deleted_projects)
96
+ remove_old_workspaces_from_release_table(params, domain_name, data_product.data_product_id, segment.segment_id, master_projects, removal_master_project_ids)
97
+ rescue Exception => e # rubocop:disable RescueException
98
+ GoodData.logger.error "Problem occurs when removing old master workspace, reason: #{e.message}"
99
+ end
100
+ end
101
+ end
102
+
90
103
  {
91
104
  segment: segment.id,
92
105
  master_pid: master.pid,
@@ -98,6 +111,42 @@ module GoodData
98
111
  # Return results
99
112
  results
100
113
  end
114
+
115
+ def remove_multiple_workspace(params, segment_id, master_projects, number_of_deleted_projects)
116
+ removal_master_project_ids = []
117
+ need_to_delete_projects = master_projects.take(number_of_deleted_projects)
118
+
119
+ need_to_delete_projects.each do |project_wrapper|
120
+ master_project_id = project_wrapper[:master_project_id]
121
+ next if master_project_id.to_s.empty?
122
+
123
+ begin
124
+ project = params.gdc_gd_client.projects(master_project_id)
125
+ if project && !%w[deleted archived].include?(project.state.to_s)
126
+ GoodData.logger.info "Segment #{segment_id}: Deleting old master workspace, project: '#{project.title}', PID: (#{project.pid})."
127
+ project.delete
128
+ end
129
+ removal_master_project_ids << master_project_id
130
+ master_projects.delete_if { |p| p[:master_project_id] == master_project_id }
131
+ rescue Exception => ex # rubocop:disable RescueException
132
+ GoodData.logger.error "Unable to remove master workspace: '#{master_project_id}', Error: #{ex.message}"
133
+ end
134
+ end
135
+ removal_master_project_ids
136
+ end
137
+
138
+ # rubocop:disable Metrics/ParameterLists
139
+ def remove_old_workspaces_from_release_table(params, domain_id, data_product_id, segment_id, master_projects, removal_master_project_ids)
140
+ unless removal_master_project_ids.empty?
141
+ if params.ads_client
142
+ GoodData::LCM2::Helpers.delete_master_project_from_ads(params.release_table_name, params.ads_client, segment_id, removal_master_project_ids)
143
+ else
144
+ data = master_projects.sort_by { |master| master[:version] }
145
+ GoodData::LCM2::Helpers.update_master_project_to_nfs(domain_id, data_product_id, segment_id, data)
146
+ end
147
+ end
148
+ end
149
+ # rubocop:enable Metrics/ParameterLists
101
150
  end
102
151
  end
103
152
  end
@@ -0,0 +1,64 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require_relative 'base_action'
9
+
10
+ module GoodData
11
+ module LCM2
12
+ class SynchronizeDataSetMapping < BaseAction
13
+ DESCRIPTION = 'Synchronize Dataset Mappings'
14
+
15
+ PARAMS = define_params(self) do
16
+ description 'Client Used for Connecting to GD'
17
+ param :gdc_gd_client, instance_of(Type::GdClientType), required: true
18
+
19
+ description 'Client used to connecting to development domain'
20
+ param :development_client, instance_of(Type::GdClientType), required: true
21
+
22
+ description 'Synchronization Info'
23
+ param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
24
+
25
+ description 'Logger'
26
+ param :gdc_logger, instance_of(Type::GdLogger), required: true
27
+ end
28
+
29
+ RESULT_HEADER = %i[from to count status]
30
+
31
+ class << self
32
+ def call(params)
33
+ results = []
34
+
35
+ client = params.gdc_gd_client
36
+ development_client = params.development_client
37
+
38
+ params.synchronize.peach do |info|
39
+ from_project = info.from
40
+ to_projects = info.to
41
+
42
+ from = development_client.projects(from_project) || fail("Invalid 'from' project specified - '#{from_project}'")
43
+ dataset_mapping = from.dataset_mapping
44
+ if dataset_mapping&.dig('datasetMappings', 'items').nil? || dataset_mapping['datasetMappings']['items'].empty?
45
+ params.gdc_logger.info "Project: '#{from.title}', PID: '#{from.pid}' has no model mapping, skip synchronizing model mapping."
46
+ else
47
+ to_projects.peach do |to|
48
+ pid = to[:pid]
49
+ to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
50
+
51
+ params.gdc_logger.info "Transferring model mapping, from project: '#{from.title}', PID: '#{from.pid}', to project: '#{to_project.title}', PID: '#{to_project.pid}'"
52
+ res = to_project.update_dataset_mapping(dataset_mapping)
53
+ res[:from] = from.pid
54
+ results << res
55
+ end
56
+ end
57
+ end
58
+ # Return results
59
+ results.flatten
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end
@@ -83,18 +83,29 @@ module GoodData
83
83
  segment_info[:from_blueprint] = blueprint
84
84
  maql_diff = nil
85
85
  previous_master = segment_info[:previous_master]
86
+ synchronize_ldm_mode = params[:synchronize_ldm].downcase
86
87
  diff_against_master = %w(diff_against_master_with_fallback diff_against_master)
87
- .include?(params[:synchronize_ldm].downcase)
88
- GoodData.logger.info "Synchronize LDM mode: '#{params[:synchronize_ldm].downcase}'"
89
- if previous_master && diff_against_master
90
- maql_diff_params = [:includeGrain]
91
- maql_diff_params << :excludeFactRule if exclude_fact_rule
92
- maql_diff_params << :includeDeprecated if include_deprecated
93
- maql_diff = previous_master.maql_diff(blueprint: blueprint, params: maql_diff_params)
88
+ .include?(synchronize_ldm_mode)
89
+ GoodData.logger.info "Synchronize LDM mode: '#{synchronize_ldm_mode}'"
90
+ if segment_info.key?(:previous_master) && diff_against_master
91
+ if previous_master
92
+ maql_diff_params = [:includeGrain]
93
+ maql_diff_params << :excludeFactRule if exclude_fact_rule
94
+ maql_diff_params << :includeDeprecated if include_deprecated
95
+ maql_diff = previous_master.maql_diff(blueprint: blueprint, params: maql_diff_params)
96
+ else
97
+ maql_diff = {
98
+ "projectModelDiff" =>
99
+ {
100
+ "updateOperations" => [],
101
+ "updateScripts" => []
102
+ }
103
+ }
104
+ end
94
105
  chunks = maql_diff['projectModelDiff']['updateScripts']
95
106
  if chunks.empty?
96
107
  GoodData.logger.info "Synchronize LDM to clients will not proceed in mode \
97
- '#{params[:synchronize_ldm].downcase}' due to no LDM changes in the new master project. \
108
+ '#{synchronize_ldm_mode}' due to no LDM changes in the segment master project. \
98
109
  If you had changed LDM of clients manually, please use mode 'diff_against_clients' \
99
110
  to force synchronize LDM to clients"
100
111
  end