gooddata 2.1.19-java → 2.3.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. checksums.yaml +4 -4
  2. data/.gdc-ii-config.yaml +42 -1
  3. data/.github/workflows/build.yml +67 -0
  4. data/.github/workflows/pre-merge.yml +72 -0
  5. data/.pronto.yml +1 -0
  6. data/.rubocop.yml +2 -14
  7. data/CHANGELOG.md +47 -0
  8. data/Dockerfile +27 -14
  9. data/Dockerfile.jruby +5 -15
  10. data/Dockerfile.ruby +5 -7
  11. data/Gemfile +4 -2
  12. data/README.md +6 -6
  13. data/Rakefile +1 -1
  14. data/SDK_VERSION +1 -1
  15. data/VERSION +1 -1
  16. data/bin/run_brick.rb +7 -0
  17. data/ci/mssql/pom.xml +62 -0
  18. data/ci/mysql/pom.xml +62 -0
  19. data/ci/redshift/pom.xml +4 -5
  20. data/docker-compose.lcm.yml +42 -4
  21. data/docker-compose.yml +42 -0
  22. data/gooddata.gemspec +21 -21
  23. data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
  24. data/lcm.rake +11 -8
  25. data/lib/gooddata/bricks/base_pipeline.rb +26 -0
  26. data/lib/gooddata/bricks/brick.rb +0 -1
  27. data/lib/gooddata/bricks/middleware/aws_middleware.rb +35 -9
  28. data/lib/gooddata/bricks/middleware/execution_result_middleware.rb +3 -3
  29. data/lib/gooddata/bricks/pipeline.rb +2 -14
  30. data/lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb +98 -0
  31. data/lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme +0 -0
  32. data/lib/gooddata/cloud_resources/mssql/mssql_client.rb +122 -0
  33. data/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme +0 -0
  34. data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +121 -0
  35. data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +0 -1
  36. data/lib/gooddata/cloud_resources/redshift/drivers/.gitkeepme +0 -0
  37. data/lib/gooddata/cloud_resources/redshift/redshift_client.rb +0 -2
  38. data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +18 -1
  39. data/lib/gooddata/helpers/data_helper.rb +9 -4
  40. data/lib/gooddata/lcm/actions/base_action.rb +157 -0
  41. data/lib/gooddata/lcm/actions/collect_data_product.rb +2 -1
  42. data/lib/gooddata/lcm/actions/collect_meta.rb +3 -1
  43. data/lib/gooddata/lcm/actions/collect_projects_warning_status.rb +53 -0
  44. data/lib/gooddata/lcm/actions/collect_segment_clients.rb +14 -0
  45. data/lib/gooddata/lcm/actions/initialize_continue_on_error_option.rb +87 -0
  46. data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +31 -4
  47. data/lib/gooddata/lcm/actions/provision_clients.rb +34 -5
  48. data/lib/gooddata/lcm/actions/synchronize_cas.rb +24 -4
  49. data/lib/gooddata/lcm/actions/synchronize_clients.rb +112 -11
  50. data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +89 -0
  51. data/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb +48 -11
  52. data/lib/gooddata/lcm/actions/synchronize_kd_dashboard_permission.rb +103 -0
  53. data/lib/gooddata/lcm/actions/synchronize_ldm.rb +79 -23
  54. data/lib/gooddata/lcm/actions/synchronize_ldm_layout.rb +98 -0
  55. data/lib/gooddata/lcm/actions/synchronize_pp_dashboard_permission.rb +108 -0
  56. data/lib/gooddata/lcm/actions/synchronize_schedules.rb +31 -1
  57. data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +26 -18
  58. data/lib/gooddata/lcm/actions/synchronize_user_groups.rb +30 -4
  59. data/lib/gooddata/lcm/actions/synchronize_users.rb +11 -10
  60. data/lib/gooddata/lcm/actions/update_metric_formats.rb +202 -0
  61. data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb +5 -0
  62. data/lib/gooddata/lcm/exceptions/lcm_execution_warning.rb +15 -0
  63. data/lib/gooddata/lcm/helpers/check_helper.rb +19 -0
  64. data/lib/gooddata/lcm/helpers/release_table_helper.rb +42 -8
  65. data/lib/gooddata/lcm/lcm2.rb +50 -4
  66. data/lib/gooddata/lcm/user_bricks_helper.rb +9 -0
  67. data/lib/gooddata/mixins/inspector.rb +1 -1
  68. data/lib/gooddata/mixins/md_object_query.rb +1 -0
  69. data/lib/gooddata/models/data_source.rb +5 -1
  70. data/lib/gooddata/models/dataset_mapping.rb +36 -0
  71. data/lib/gooddata/models/ldm_layout.rb +38 -0
  72. data/lib/gooddata/models/metadata/label.rb +26 -27
  73. data/lib/gooddata/models/project.rb +230 -30
  74. data/lib/gooddata/models/project_creator.rb +83 -6
  75. data/lib/gooddata/models/schedule.rb +13 -1
  76. data/lib/gooddata/models/segment.rb +2 -1
  77. data/lib/gooddata/models/user_filters/user_filter_builder.rb +162 -68
  78. data/lib/gooddata/rest/connection.rb +5 -3
  79. data/lib/gooddata/rest/phmap.rb +2 -0
  80. data/lib/gooddata.rb +1 -0
  81. data/lib/gooddata_brick_base.rb +35 -0
  82. data/sonar-project.properties +6 -0
  83. metadata +96 -65
  84. data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb +0 -37
  85. data/lib/gooddata/cloud_resources/redshift/drivers/log4j.properties +0 -15
@@ -0,0 +1,87 @@
1
+ # frozen_string_literal: true
2
+ # (C) 2019-2022 GoodData Corporation
3
+ require_relative 'base_action'
4
+
5
+ module GoodData
6
+ module LCM2
7
+ class InitializeContinueOnErrorOption < BaseAction
8
+ DESCRIPTION = 'Initialize continue on error option'
9
+
10
+ PARAMS = define_params(self) do
11
+ description 'Organization Name'
12
+ param :organization, instance_of(Type::StringType), required: false
13
+
14
+ description 'Domain'
15
+ param :domain, instance_of(Type::StringType), required: false
16
+
17
+ description 'DataProduct'
18
+ param :data_product, instance_of(Type::GDDataProductType), required: false
19
+
20
+ description 'Restricts synchronization to specified segments'
21
+ param :segments_filter, array_of(instance_of(Type::StringType)), required: false
22
+
23
+ description 'Abort on error'
24
+ param :abort_on_error, instance_of(Type::StringType), required: false
25
+
26
+ description 'Client Used for Connecting to GD'
27
+ param :gdc_gd_client, instance_of(Type::GdClientType), required: true
28
+ end
29
+
30
+ class << self
31
+ def call(params)
32
+ project_mappings = ThreadSafe::Hash.new
33
+ client_mappings = ThreadSafe::Hash.new
34
+ continue_on_error = continue_on_error(params)
35
+
36
+ if continue_on_error
37
+ client = params.gdc_gd_client
38
+ domain_name = params.organization || params.domain
39
+ domain = client.domain(domain_name)
40
+ data_product = params.data_product
41
+ data_product_segments = domain.segments(:all, data_product)
42
+
43
+ if params.segments_filter
44
+ data_product_segments.select! do |segment|
45
+ params.segments_filter.include?(segment.segment_id)
46
+ end
47
+ end
48
+
49
+ data_product_segments.pmap do |segment|
50
+ segment.clients.map do |segment_client|
51
+ project = segment_client.project
52
+ next unless project
53
+
54
+ project_mappings[project.pid.to_sym] = {
55
+ client_id: segment_client.client_id,
56
+ segment_id: segment.segment_id
57
+ }
58
+ client_mappings[segment_client.client_id.to_sym] = {
59
+ project_id: project.pid,
60
+ segment_id: segment.segment_id
61
+ }
62
+ end
63
+ end
64
+ end
65
+
66
+ {
67
+ params: {
68
+ collect_synced_status: continue_on_error,
69
+ sync_failed_list: {
70
+ project_client_mappings: project_mappings,
71
+ client_project_mappings: client_mappings,
72
+ failed_detailed_projects: [],
73
+ failed_projects: [],
74
+ failed_clients: [],
75
+ failed_segments: []
76
+ }
77
+ }
78
+ }
79
+ end
80
+
81
+ def print_result(_params)
82
+ false
83
+ end
84
+ end
85
+ end
86
+ end
87
+ end
@@ -19,6 +19,18 @@ module GoodData
19
19
 
20
20
  description 'Synchronization Info'
21
21
  param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
22
+
23
+ description 'Abort on error'
24
+ param :abort_on_error, instance_of(Type::StringType), required: false
25
+
26
+ description 'Logger'
27
+ param :gdc_logger, instance_of(Type::GdLogger), required: false
28
+
29
+ description 'Collect synced status'
30
+ param :collect_synced_status, instance_of(Type::BooleanType), required: false
31
+
32
+ description 'Sync failed list'
33
+ param :sync_failed_list, instance_of(Type::HashType), required: false
22
34
  end
23
35
 
24
36
  RESULT_HEADER = %i[from to status]
@@ -27,6 +39,8 @@ module GoodData
27
39
  def call(params)
28
40
  results = []
29
41
  params.synchronize.map do |segment_info|
42
+ next if sync_failed_segment(segment_info[:segment_id], params)
43
+
30
44
  result = migrate_date_dimension(params, segment_info)
31
45
  results.concat(result)
32
46
  end
@@ -47,21 +61,32 @@ module GoodData
47
61
  # check latest master and previous master
48
62
  master_upgrade_datasets = get_upgrade_dates(latest_blueprint, previous_blueprint) if params[:synchronize_ldm].downcase == 'diff_against_master' && previous_blueprint
49
63
  unless master_upgrade_datasets&.empty?
64
+ collect_synced_status = collect_synced_status(params)
65
+ failed_projects = ThreadSafe::Array.new
66
+
50
67
  segment_info[:to].pmap do |entry|
51
68
  pid = entry[:pid]
69
+ next if sync_failed_project(pid, params)
70
+
52
71
  to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
72
+ GoodData.logger.info "Migrating date dimension, project: '#{to_project.title}', PID: #{pid}"
53
73
  to_blueprint = to_project.blueprint
54
74
  upgrade_datasets = get_upgrade_dates(latest_blueprint, to_blueprint)
55
75
  next if upgrade_datasets.empty?
56
76
 
57
77
  message = get_upgrade_message(upgrade_datasets)
78
+ failed_message = "Failed to migrate date dimension for project #{pid}"
79
+ update_status = to_project.upgrade_custom_v2(message)
80
+ process_failed_project(pid, failed_message, failed_projects, collect_synced_status) if collect_synced_status && update_status != 'OK'
58
81
 
59
82
  results << {
60
83
  from: segment_info[:from],
61
84
  to: pid,
62
- status: to_project.upgrade_custom_v2(message)
85
+ status: update_status
63
86
  }
64
87
  end
88
+
89
+ process_failed_projects(failed_projects, short_name, params) if collect_synced_status
65
90
  end
66
91
 
67
92
  results
@@ -71,9 +96,9 @@ module GoodData
71
96
  dest_dates = get_date_dimensions(dest_blueprint) if dest_blueprint
72
97
  src_dates = get_date_dimensions(src_blueprint) if src_blueprint
73
98
 
74
- return false if dest_dates.empty? || src_dates.empty?
75
-
76
99
  upgrade_datasets = []
100
+ return upgrade_datasets if dest_dates.empty? || src_dates.empty?
101
+
77
102
  dest_dates.each do |dest|
78
103
  src_dim = get_date_dimension(src_blueprint, dest[:id])
79
104
  next unless src_dim
@@ -103,8 +128,10 @@ module GoodData
103
128
  get_date_dimensions(blueprint).any? { |e| e[:urn] == DATE_DIMENSION_CUSTOM_V2 }
104
129
  end
105
130
 
131
+ # Get date dimension from blue print. Return nil if date dimension not existing
106
132
  def get_date_dimension(blueprint, id)
107
- GoodData::Model::ProjectBlueprint.find_date_dimension(blueprint, id)
133
+ date_dimensions = get_date_dimensions(blueprint)
134
+ date_dimensions.find { |d| d[:id] == id }
108
135
  end
109
136
 
110
137
  def get_date_dimensions(blueprint)
@@ -29,7 +29,16 @@ module GoodData
29
29
  param :data_product, instance_of(Type::GDDataProductType), required: false
30
30
 
31
31
  description 'Logger'
32
- param :gdc_logger, instance_of(Type::GdLogger), required: true
32
+ param :gdc_logger, instance_of(Type::GdLogger), required: false
33
+
34
+ description 'Abort on error'
35
+ param :abort_on_error, instance_of(Type::StringType), required: false
36
+
37
+ description 'Collect synced status'
38
+ param :collect_synced_status, instance_of(Type::BooleanType), required: false
39
+
40
+ description 'Sync failed list'
41
+ param :sync_failed_list, instance_of(Type::HashType), required: false
33
42
  end
34
43
 
35
44
  RESULT_HEADER = [
@@ -45,6 +54,8 @@ module GoodData
45
54
  synchronize_projects = []
46
55
  data_product = params.data_product
47
56
  client = params.gdc_gd_client
57
+ collect_synced_status = collect_synced_status(params)
58
+ continue_on_error = continue_on_error(params)
48
59
  domain_name = params.organization || params.domain
49
60
  fail "Either organisation or domain has to be specified in params" unless domain_name
50
61
  domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
@@ -52,6 +63,8 @@ module GoodData
52
63
  invalid_client_ids = []
53
64
  begin
54
65
  results = params.segments.map do |segment|
66
+ next if sync_failed_segment(segment.segment_id, params)
67
+
55
68
  segment_object = domain.segments(segment.segment_id, data_product)
56
69
  tmp = segment_object.provision_client_projects.map do |m|
57
70
  Hash[m.each_pair.to_a].merge(type: :provision_result)
@@ -65,10 +78,21 @@ module GoodData
65
78
  unless entry[:project_uri]
66
79
  error_message = "There was error during provisioning clients: #{entry[:error]}" unless error_message
67
80
  invalid_client_ids << entry[:id]
81
+ if collect_synced_status
82
+ failed_message = "Failed to provision client #{entry[:id]} in segment #{segment.segment_id}. Error: #{entry[:error]}"
83
+ add_failed_client(entry[:id], failed_message, short_name, params)
84
+ end
68
85
  next
69
86
  end
87
+
88
+ project_id = entry[:project_uri].split('/').last
89
+ if collect_synced_status && entry[:status] == 'CREATED' && entry[:id]
90
+ # Update project client mappings when there are create new clients during provision clients of the segment
91
+ add_new_clients_to_project_client_mapping(project_id, entry[:id], segment.segment_id, params)
92
+ end
93
+
70
94
  {
71
- pid: entry[:project_uri].split('/').last,
95
+ pid: project_id,
72
96
  client_id: entry[:id]
73
97
  }
74
98
  end.compact
@@ -93,15 +117,20 @@ module GoodData
93
117
  end
94
118
 
95
119
  params.gdc_logger.debug "Deleted clients: #{deleted_client_ids.join(', ')}"
96
- raise error_message unless error_message['TooManyProjectsCreatedException'] || error_message['Max number registered projects']
120
+ unless error_message['TooManyProjectsCreatedException'] || error_message['Max number registered projects']
121
+ raise error_message unless continue_on_error
122
+
123
+ next
124
+ end
125
+
97
126
  break tmp
98
127
  end
99
128
 
100
129
  tmp
101
130
  end
102
131
  rescue => e
103
- params.gdc_logger.error "Problem occurs when provisioning clients."
104
- raise e
132
+ params.gdc_logger.error "Problem occurs when provisioning clients. Error: #{e}"
133
+ raise e unless continue_on_error
105
134
  end
106
135
 
107
136
  results.flatten! if results
@@ -32,6 +32,15 @@ module GoodData
32
32
 
33
33
  description 'Additional Hidden Parameters'
34
34
  param :additional_hidden_params, instance_of(Type::HashType), required: false
35
+
36
+ description 'Abort on error'
37
+ param :abort_on_error, instance_of(Type::StringType), required: false
38
+
39
+ description 'Collect synced status'
40
+ param :collect_synced_status, instance_of(Type::BooleanType), required: false
41
+
42
+ description 'Sync failed list'
43
+ param :sync_failed_list, instance_of(Type::HashType), required: false
35
44
  end
36
45
 
37
46
  class << self
@@ -46,6 +55,8 @@ module GoodData
46
55
  return results unless include_ca
47
56
 
48
57
  client = params.gdc_gd_client
58
+ collect_synced_status = collect_synced_status(params)
59
+ failed_projects = ThreadSafe::Array.new
49
60
 
50
61
  params.synchronize.each do |info|
51
62
  from = info.from
@@ -58,24 +69,33 @@ module GoodData
58
69
  next unless ca_scripts
59
70
 
60
71
  pid = entry[:pid]
72
+ next if sync_failed_project(pid, params)
73
+
61
74
  ca_chunks = ca_scripts[:maqlDdlChunks]
62
- to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
63
- params.gdc_logger.info "Synchronizing Computed Attributes to project: '#{to_project.title}', PID: #{pid}"
75
+ to_project = client.projects(pid)
76
+ unless to_project
77
+ process_failed_project(pid, "Invalid 'to' project specified - '#{pid}'", failed_projects, collect_synced_status)
78
+ next
79
+ end
64
80
 
81
+ params.gdc_logger.info "Synchronizing Computed Attributes to project: '#{to_project.title}', PID: #{pid}"
82
+ error_message = nil
65
83
  begin
66
84
  ca_chunks.each { |chunk| to_project.execute_maql(chunk) }
67
85
  rescue => e
68
- raise "Error occured when executing MAQL, project: \"#{to_project.title}\" reason: \"#{e.message}\", chunks: #{ca_chunks.inspect}"
86
+ error_message = "Error occurred when executing MAQL for project: \"#{to_project.title}\" reason: \"#{e.message}\", chunks: #{ca_chunks.inspect}"
87
+ process_failed_project(pid, error_message, failed_projects, collect_synced_status)
69
88
  end
70
89
 
71
90
  results << {
72
91
  from: from,
73
92
  to: pid,
74
- status: 'ok'
93
+ status: error_message.nil? ? 'ok' : 'failed'
75
94
  }
76
95
  end
77
96
  end
78
97
 
98
+ process_failed_projects(failed_projects, short_name, params) if collect_synced_status
79
99
  results
80
100
  end
81
101
  end
@@ -33,8 +33,23 @@ module GoodData
33
33
  description 'ADS Client'
34
34
  param :ads_client, instance_of(Type::AdsClientType), required: false
35
35
 
36
+ description 'Keep number of old master workspace excluding the latest one'
37
+ param :keep_only_previous_masters_count, instance_of(Type::StringType), required: false, default: '-1'
38
+
36
39
  description 'Additional Hidden Parameters'
37
40
  param :additional_hidden_params, instance_of(Type::HashType), required: false
41
+
42
+ description 'Abort on error'
43
+ param :abort_on_error, instance_of(Type::StringType), required: false
44
+
45
+ description 'Logger'
46
+ param :gdc_logger, instance_of(Type::GdLogger), required: false
47
+
48
+ description 'Collect synced status'
49
+ param :collect_synced_status, instance_of(Type::BooleanType), required: false
50
+
51
+ description 'Sync failed list'
52
+ param :sync_failed_list, instance_of(Type::HashType), required: false
38
53
  end
39
54
 
40
55
  RESULT_HEADER = [
@@ -53,6 +68,7 @@ module GoodData
53
68
  domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
54
69
  data_product = params.data_product
55
70
  domain_segments = domain.segments(:all, data_product)
71
+ keep_only_previous_masters_count = Integer(params.keep_only_previous_masters_count || "-1")
56
72
 
57
73
  segments = params.segments.map do |seg|
58
74
  domain_segments.find do |s|
@@ -61,19 +77,17 @@ module GoodData
61
77
  end
62
78
 
63
79
  results = segments.map do |segment|
80
+ next if sync_failed_segment(segment.segment_id, params)
81
+
64
82
  if params.ads_client
65
- current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
66
- params.release_table_name,
67
- params.ads_client,
68
- segment.segment_id
69
- )
83
+ master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_ads(params.release_table_name, params.ads_client, segment.segment_id)
70
84
  else
71
- current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
85
+ master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
72
86
  end
73
87
 
88
+ current_master = master_projects.last
74
89
  # TODO: Check res.first.nil? || res.first[:master_project_id].nil?
75
90
  master = client.projects(current_master[:master_project_id])
76
-
77
91
  segment.master_project = master
78
92
  segment.save
79
93
 
@@ -83,14 +97,26 @@ module GoodData
83
97
  failed_count = sync_result['failedClients']['count']
84
98
 
85
99
  if failed_count.to_i > 0
86
- fail("#{failed_count} clients failed to synchronize. " \
87
- "Details: #{sync_result['links']['details']}")
100
+ error_handle(segment, res, collect_synced_status(params), params)
101
+ end
102
+
103
+ if keep_only_previous_masters_count >= 0
104
+ number_of_deleted_projects = master_projects.count - (keep_only_previous_masters_count + 1)
105
+
106
+ if number_of_deleted_projects.positive?
107
+ begin
108
+ removal_master_project_ids = remove_multiple_workspace(params, segment.segment_id, master_projects, number_of_deleted_projects)
109
+ remove_old_workspaces_from_release_table(params, domain_name, data_product.data_product_id, segment.segment_id, master_projects, removal_master_project_ids)
110
+ rescue Exception => e # rubocop:disable RescueException
111
+ GoodData.logger.error "Problem occurs when removing old master workspace, reason: #{e.message}"
112
+ end
113
+ end
88
114
  end
89
115
 
90
116
  {
91
117
  segment: segment.id,
92
- master_pid: master.pid,
93
- master_name: master.title,
118
+ master_pid: master.nil? ? '' : master.pid,
119
+ master_name: master.nil? ? '' : master.title,
94
120
  successful_count: sync_result['successfulClients']['count']
95
121
  }
96
122
  end
@@ -98,6 +124,81 @@ module GoodData
98
124
  # Return results
99
125
  results
100
126
  end
127
+
128
+ def remove_multiple_workspace(params, segment_id, master_projects, number_of_deleted_projects)
129
+ removal_master_project_ids = []
130
+ need_to_delete_projects = master_projects.take(number_of_deleted_projects)
131
+
132
+ need_to_delete_projects.each do |project_wrapper|
133
+ master_project_id = project_wrapper[:master_project_id]
134
+ next if master_project_id.to_s.empty?
135
+
136
+ begin
137
+ project = params.gdc_gd_client.projects(master_project_id)
138
+ if project && !%w[deleted archived].include?(project.state.to_s)
139
+ GoodData.logger.info "Segment #{segment_id}: Deleting old master workspace, project: '#{project.title}', PID: (#{project.pid})."
140
+ project.delete
141
+ end
142
+ removal_master_project_ids << master_project_id
143
+ master_projects.delete_if { |p| p[:master_project_id] == master_project_id }
144
+ rescue Exception => ex # rubocop:disable RescueException
145
+ GoodData.logger.error "Unable to remove master workspace: '#{master_project_id}', Error: #{ex.message}"
146
+ end
147
+ end
148
+ removal_master_project_ids
149
+ end
150
+
151
+ # rubocop:disable Metrics/ParameterLists
152
+ def remove_old_workspaces_from_release_table(params, domain_id, data_product_id, segment_id, master_projects, removal_master_project_ids)
153
+ unless removal_master_project_ids.empty?
154
+ if params.ads_client
155
+ GoodData::LCM2::Helpers.delete_master_project_from_ads(params.release_table_name, params.ads_client, segment_id, removal_master_project_ids)
156
+ else
157
+ data = master_projects.sort_by { |master| master[:version] }
158
+ GoodData::LCM2::Helpers.update_master_project_to_nfs(domain_id, data_product_id, segment_id, data)
159
+ end
160
+ end
161
+ end
162
+ # rubocop:enable Metrics/ParameterLists
163
+
164
+ private
165
+
166
+ def error_handle(segment, error_result, continue_on_error, params)
167
+ sync_result = error_result.json['synchronizationResult']
168
+ success_count = sync_result['successfulClients']['count'].to_i
169
+ failed_count = sync_result['failedClients']['count'].to_i
170
+
171
+ # Synchronize failure for all clients in segment
172
+ if continue_on_error && success_count.zero? && failed_count.positive?
173
+ segment_warning_message = "Failed to synchronize clients for #{segment.segment_id} segment. Details: #{sync_result['links']['details']}"
174
+ add_failed_segment(segment.segment_id, segment_warning_message, short_name, params)
175
+ return
176
+ end
177
+
178
+ summary_error_message = "#{failed_count} clients failed to synchronize. Details: #{sync_result['links']['details']}"
179
+ fail(summary_error_message) unless continue_on_error
180
+
181
+ GoodData.logger.warn summary_error_message
182
+ if error_result.details # rubocop:disable Style/SafeNavigation
183
+ error_result.details.items.each do |item|
184
+ next if item['status'] == 'OK'
185
+
186
+ error_client_id = item['id']
187
+ error_message = error_message(item, segment)
188
+ add_failed_client(error_client_id, error_message, short_name, params)
189
+ end
190
+ end
191
+ end
192
+
193
+ def error_message(error_item, segment)
194
+ error_client_id = error_item['id']
195
+ error_message = "Failed to synchronize #{error_client_id} client in #{segment.segment_id} segment."
196
+ error_message = "#{error_message}. Detail: #{error_item['error']['message']}" if error_item['error'] && error_item['error']['message']
197
+
198
+ error_message = "#{error_message}. Error items: #{error_item['error']['parameters']}" if error_item['error'] && error_item['error']['parameters']
199
+
200
+ error_message
201
+ end
101
202
  end
102
203
  end
103
204
  end
@@ -0,0 +1,89 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require_relative 'base_action'
9
+
10
+ module GoodData
11
+ module LCM2
12
+ class SynchronizeDataSetMapping < BaseAction
13
+ DESCRIPTION = 'Synchronize Dataset Mappings'
14
+
15
+ PARAMS = define_params(self) do
16
+ description 'Client Used for Connecting to GD'
17
+ param :gdc_gd_client, instance_of(Type::GdClientType), required: true
18
+
19
+ description 'Client used to connecting to development domain'
20
+ param :development_client, instance_of(Type::GdClientType), required: true
21
+
22
+ description 'Synchronization Info'
23
+ param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
24
+
25
+ description 'Logger'
26
+ param :gdc_logger, instance_of(Type::GdLogger), required: true
27
+
28
+ description 'Abort on error'
29
+ param :abort_on_error, instance_of(Type::StringType), required: false
30
+
31
+ description 'Collect synced status'
32
+ param :collect_synced_status, instance_of(Type::BooleanType), required: false
33
+
34
+ description 'Sync failed list'
35
+ param :sync_failed_list, instance_of(Type::HashType), required: false
36
+ end
37
+
38
+ RESULT_HEADER = %i[from to count status]
39
+
40
+ class << self
41
+ def call(params)
42
+ results = []
43
+ collect_synced_status = collect_synced_status(params)
44
+ failed_projects = ThreadSafe::Array.new
45
+
46
+ client = params.gdc_gd_client
47
+ development_client = params.development_client
48
+
49
+ params.synchronize.peach do |info|
50
+ from_project = info.from
51
+ to_projects = info.to
52
+
53
+ from = development_client.projects(from_project)
54
+ unless from
55
+ process_failed_project(from_project, "Invalid 'from' project specified - '#{from_project}'", failed_projects, collect_synced_status)
56
+ next
57
+ end
58
+
59
+ dataset_mapping = from.dataset_mapping
60
+ if dataset_mapping&.dig('datasetMappings', 'items').nil? || dataset_mapping['datasetMappings']['items'].empty?
61
+ params.gdc_logger.info "Project: '#{from.title}', PID: '#{from.pid}' has no model mapping, skip synchronizing model mapping."
62
+ else
63
+ to_projects.peach do |to|
64
+ pid = to[:pid]
65
+ next if sync_failed_project(pid, params)
66
+
67
+ to_project = client.projects(pid)
68
+ process_failed_project(pid, "Invalid 'to' project specified - '#{pid}'", failed_projects, collect_synced_status) unless to_project
69
+
70
+ message_to_project = "to project: '#{to_project.title}', PID: '#{to_project.pid}'"
71
+ params.gdc_logger.info "Transferring model mapping, from project: '#{from.title}', PID: '#{from.pid}', #{message_to_project}"
72
+ res = to_project.update_dataset_mapping(dataset_mapping)
73
+ res[:from] = from.pid
74
+ results << res
75
+
76
+ failed_message = "Failed to transfer model mapping from project '#{from.pid}' to project '#{to_project.pid}'"
77
+ process_failed_project(pid, failed_message, failed_projects, collect_synced_status) if collect_synced_status && res[:status] != 'OK'
78
+ end
79
+ end
80
+ end
81
+
82
+ process_failed_projects(failed_projects, short_name, params) if collect_synced_status
83
+ # Return results
84
+ results.flatten
85
+ end
86
+ end
87
+ end
88
+ end
89
+ end