gooddata 2.2.0 → 2.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (67) hide show
  1. checksums.yaml +5 -5
  2. data/.gdc-ii-config.yaml +42 -1
  3. data/.github/workflows/build.yml +14 -13
  4. data/.github/workflows/pre-merge.yml +13 -13
  5. data/.pronto.yml +1 -0
  6. data/.rubocop.yml +2 -14
  7. data/CHANGELOG.md +9 -0
  8. data/Dockerfile +13 -7
  9. data/Dockerfile.jruby +5 -5
  10. data/Dockerfile.ruby +5 -7
  11. data/Gemfile +4 -2
  12. data/README.md +5 -4
  13. data/Rakefile +1 -1
  14. data/SDK_VERSION +1 -1
  15. data/VERSION +1 -1
  16. data/bin/run_brick.rb +7 -0
  17. data/ci/mysql/pom.xml +6 -1
  18. data/ci/redshift/pom.xml +3 -4
  19. data/docker-compose.lcm.yml +42 -1
  20. data/docker-compose.yml +42 -0
  21. data/gooddata.gemspec +21 -22
  22. data/lcm.rake +9 -0
  23. data/lib/gooddata/bricks/base_pipeline.rb +26 -0
  24. data/lib/gooddata/bricks/brick.rb +0 -1
  25. data/lib/gooddata/bricks/middleware/execution_result_middleware.rb +3 -3
  26. data/lib/gooddata/bricks/pipeline.rb +2 -14
  27. data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +18 -8
  28. data/lib/gooddata/cloud_resources/redshift/drivers/.gitkeepme +0 -0
  29. data/lib/gooddata/cloud_resources/redshift/redshift_client.rb +0 -2
  30. data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +1 -1
  31. data/lib/gooddata/lcm/actions/base_action.rb +157 -0
  32. data/lib/gooddata/lcm/actions/collect_data_product.rb +2 -1
  33. data/lib/gooddata/lcm/actions/collect_projects_warning_status.rb +53 -0
  34. data/lib/gooddata/lcm/actions/collect_segment_clients.rb +14 -0
  35. data/lib/gooddata/lcm/actions/initialize_continue_on_error_option.rb +87 -0
  36. data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +28 -2
  37. data/lib/gooddata/lcm/actions/provision_clients.rb +34 -5
  38. data/lib/gooddata/lcm/actions/synchronize_cas.rb +24 -4
  39. data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -4
  40. data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +28 -3
  41. data/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb +48 -11
  42. data/lib/gooddata/lcm/actions/synchronize_kd_dashboard_permission.rb +103 -0
  43. data/lib/gooddata/lcm/actions/synchronize_ldm.rb +60 -15
  44. data/lib/gooddata/lcm/actions/synchronize_ldm_layout.rb +98 -0
  45. data/lib/gooddata/lcm/actions/synchronize_pp_dashboard_permission.rb +108 -0
  46. data/lib/gooddata/lcm/actions/synchronize_schedules.rb +31 -1
  47. data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +14 -9
  48. data/lib/gooddata/lcm/actions/synchronize_user_groups.rb +30 -4
  49. data/lib/gooddata/lcm/actions/synchronize_users.rb +11 -10
  50. data/lib/gooddata/lcm/actions/update_metric_formats.rb +21 -4
  51. data/lib/gooddata/lcm/exceptions/lcm_execution_warning.rb +15 -0
  52. data/lib/gooddata/lcm/helpers/check_helper.rb +19 -0
  53. data/lib/gooddata/lcm/lcm2.rb +45 -4
  54. data/lib/gooddata/lcm/user_bricks_helper.rb +9 -0
  55. data/lib/gooddata/mixins/inspector.rb +1 -1
  56. data/lib/gooddata/models/ldm_layout.rb +38 -0
  57. data/lib/gooddata/models/project.rb +197 -22
  58. data/lib/gooddata/models/project_creator.rb +83 -6
  59. data/lib/gooddata/models/segment.rb +2 -1
  60. data/lib/gooddata/models/user_filters/user_filter_builder.rb +104 -15
  61. data/lib/gooddata/rest/connection.rb +5 -3
  62. data/lib/gooddata/rest/phmap.rb +1 -0
  63. data/lib/gooddata.rb +1 -0
  64. data/lib/gooddata_brick_base.rb +35 -0
  65. data/sonar-project.properties +6 -0
  66. metadata +64 -59
  67. data/lib/gooddata/cloud_resources/redshift/drivers/log4j.properties +0 -15
@@ -5,6 +5,7 @@
5
5
  # LICENSE file in the root directory of this source tree.
6
6
 
7
7
  require_relative 'base_action'
8
+ require_relative '../helpers/helpers'
8
9
 
9
10
  using TrueExtensions
10
11
  using FalseExtensions
@@ -47,6 +48,15 @@ module GoodData
47
48
 
48
49
  description 'Enables handling of deprecated objects in the logical data model.'
49
50
  param :include_deprecated, instance_of(Type::BooleanType), required: false, default: false
51
+
52
+ description 'Abort on error'
53
+ param :abort_on_error, instance_of(Type::StringType), required: false
54
+
55
+ description 'Collect synced status'
56
+ param :collect_synced_status, instance_of(Type::BooleanType), required: false
57
+
58
+ description 'Sync failed list'
59
+ param :sync_failed_list, instance_of(Type::HashType), required: false
50
60
  end
51
61
 
52
62
  RESULT_HEADER = %i[from to status]
@@ -56,6 +66,8 @@ module GoodData
56
66
  results = []
57
67
  synchronize = []
58
68
  params.synchronize.map do |segment_info|
69
+ next if sync_failed_segment(segment_info[:segment_id], params)
70
+
59
71
  new_segment_info, segment_results = sync_segment_ldm(params, segment_info)
60
72
  results.concat(segment_results)
61
73
  synchronize << new_segment_info
@@ -72,12 +84,27 @@ module GoodData
72
84
  private
73
85
 
74
86
  def sync_segment_ldm(params, segment_info)
75
- results = []
87
+ collect_synced_status = collect_synced_status(params)
88
+ failed_projects = ThreadSafe::Array.new
89
+ results = ThreadSafe::Array.new
76
90
  client = params.gdc_gd_client
77
91
  exclude_fact_rule = params.exclude_fact_rule.to_b
78
92
  include_deprecated = params.include_deprecated.to_b
93
+ update_preference = params[:update_preference]
94
+ exist_fallback_to_hard_sync_config = !update_preference.nil? && !update_preference[:fallback_to_hard_sync].nil?
95
+ include_maql_fallback_hard_sync = exist_fallback_to_hard_sync_config && Helpers.to_bool('fallback_to_hard_sync', update_preference[:fallback_to_hard_sync])
96
+
79
97
  from_pid = segment_info[:from]
80
- from = params.development_client.projects(from_pid) || fail("Invalid 'from' project specified - '#{from_pid}'")
98
+ from = params.development_client.projects(from_pid)
99
+ unless from
100
+ segment_id = segment_info[:segment_id]
101
+ error_message = "Failed to sync LDM for segment #{segment_id}. Error: Invalid 'from' project specified - '#{from_pid}'"
102
+ fail(error_message) unless collect_synced_status
103
+
104
+ add_failed_segment(segment_id, error_message, short_name, params)
105
+ return [segment_info, results]
106
+ end
107
+
81
108
  GoodData.logger.info "Creating Blueprint, project: '#{from.title}', PID: #{from_pid}"
82
109
  blueprint = from.blueprint(include_ca: params.include_computed_attributes.to_b)
83
110
  segment_info[:from_blueprint] = blueprint
@@ -92,6 +119,8 @@ module GoodData
92
119
  maql_diff_params = [:includeGrain]
93
120
  maql_diff_params << :excludeFactRule if exclude_fact_rule
94
121
  maql_diff_params << :includeDeprecated if include_deprecated
122
+ maql_diff_params << :includeMaqlFallbackHardSync if include_maql_fallback_hard_sync
123
+
95
124
  maql_diff = previous_master.maql_diff(blueprint: blueprint, params: maql_diff_params)
96
125
  else
97
126
  maql_diff = {
@@ -102,6 +131,7 @@ module GoodData
102
131
  }
103
132
  }
104
133
  end
134
+
105
135
  chunks = maql_diff['projectModelDiff']['updateScripts']
106
136
  if chunks.empty?
107
137
  GoodData.logger.info "Synchronize LDM to clients will not proceed in mode \
@@ -112,40 +142,55 @@ to force synchronize LDM to clients"
112
142
  end
113
143
 
114
144
  segment_info[:to] = segment_info[:to].pmap do |entry|
145
+ update_status = true
115
146
  pid = entry[:pid]
116
- to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
147
+ next if sync_failed_project(pid, params)
148
+
149
+ to_project = client.projects(pid)
150
+ unless to_project
151
+ process_failed_project(pid, "Invalid 'to' project specified - '#{pid}'", failed_projects, collect_synced_status)
152
+ next
153
+ end
117
154
 
118
155
  GoodData.logger.info "Updating from Blueprint, project: '#{to_project.title}', PID: #{pid}"
119
156
  begin
120
157
  entry[:ca_scripts] = to_project.update_from_blueprint(
121
158
  blueprint,
122
- update_preference: params[:update_preference],
159
+ update_preference: update_preference,
123
160
  exclude_fact_rule: exclude_fact_rule,
124
161
  execute_ca_scripts: false,
125
162
  maql_diff: maql_diff,
126
163
  include_deprecated: include_deprecated
127
164
  )
128
165
  rescue MaqlExecutionError => e
129
- GoodData.logger.info("Applying MAQL to project #{to_project.title} - #{pid} failed. Reason: #{e}")
130
- fail e unless previous_master && params[:synchronize_ldm] == 'diff_against_master_with_fallback'
131
- GoodData.logger.info("Restoring the client project #{to_project.title} from master.")
132
- entry[:ca_scripts] = to_project.update_from_blueprint(
133
- blueprint,
134
- update_preference: params[:update_preference],
135
- exclude_fact_rule: exclude_fact_rule,
136
- execute_ca_scripts: false,
137
- include_deprecated: include_deprecated
138
- )
166
+ if collect_synced_status
167
+ update_status = false
168
+ failed_message = "Applying MAQL to project #{to_project.title} - #{pid} failed. Reason: #{e}"
169
+ process_failed_project(pid, failed_message, failed_projects, collect_synced_status)
170
+ else
171
+ fail e unless previous_master && params[:synchronize_ldm] == 'diff_against_master_with_fallback'
172
+
173
+ GoodData.logger.info("Restoring the client project #{to_project.title} from master.")
174
+ entry[:ca_scripts] = to_project.update_from_blueprint(
175
+ blueprint,
176
+ update_preference: update_preference,
177
+ exclude_fact_rule: exclude_fact_rule,
178
+ execute_ca_scripts: false,
179
+ include_deprecated: include_deprecated
180
+ )
181
+ end
139
182
  end
140
183
 
141
184
  results << {
142
185
  from: from_pid,
143
186
  to: pid,
144
187
  status: 'ok'
145
- }
188
+ } if update_status
189
+
146
190
  entry
147
191
  end
148
192
 
193
+ process_failed_projects(failed_projects, short_name, params) if collect_synced_status
149
194
  [segment_info, results]
150
195
  end
151
196
  end
@@ -0,0 +1,98 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2022 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require_relative 'base_action'
9
+
10
+ module GoodData
11
+ module LCM2
12
+ class SynchronizeLdmLayout < BaseAction
13
+ DESCRIPTION = 'Synchronize LDM Layout'
14
+
15
+ PARAMS = define_params(self) do
16
+ description 'Logger'
17
+ param :gdc_logger, instance_of(Type::GdLogger), required: true
18
+
19
+ description 'Client Used for Connecting to GD'
20
+ param :gdc_gd_client, instance_of(Type::GdClientType), required: true
21
+
22
+ description 'Client used to connecting to development domain'
23
+ param :development_client, instance_of(Type::GdClientType), required: true
24
+
25
+ description 'Synchronization Info'
26
+ param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
27
+
28
+ description 'Abort on error'
29
+ param :abort_on_error, instance_of(Type::StringType), required: false
30
+
31
+ description 'Collect synced status'
32
+ param :collect_synced_status, instance_of(Type::BooleanType), required: false
33
+
34
+ description 'Sync failed list'
35
+ param :sync_failed_list, instance_of(Type::HashType), required: false
36
+ end
37
+
38
+ RESULT_HEADER = %i[from to status]
39
+
40
+ class << self
41
+ def call(params)
42
+ results = []
43
+
44
+ client = params.gdc_gd_client
45
+ development_client = params.development_client
46
+ gdc_logger = params.gdc_logger
47
+ collect_synced_status = collect_synced_status(params)
48
+ failed_projects = ThreadSafe::Array.new
49
+
50
+ params.synchronize.peach do |info|
51
+ from_project = info.from
52
+ to_projects = info.to
53
+
54
+ from = development_client.projects(from_project)
55
+ unless from
56
+ process_failed_project(from_project, "Invalid 'from' project specified - '#{from_project}'", failed_projects, collect_synced_status)
57
+ next
58
+ end
59
+
60
+ from_pid = from.pid
61
+ from_title = from.title
62
+ from_ldm_layout = from.ldm_layout
63
+
64
+ if from_ldm_layout&.dig('ldmLayout', 'layout').nil? || from_ldm_layout['ldmLayout']['layout'].empty?
65
+ gdc_logger.info "Project: '#{from_title}', PID: '#{from_pid}' has no ldm layout, skip synchronizing ldm layout."
66
+ else
67
+ to_projects.peach do |to|
68
+ pid = to[:pid]
69
+ to_project = client.projects(pid)
70
+ unless to_project
71
+ process_failed_project(pid, "Invalid 'from' project specified - '#{pid}'", failed_projects, collect_synced_status)
72
+ next
73
+ end
74
+
75
+ next if sync_failed_project(pid, params)
76
+
77
+ gdc_logger.info "Transferring ldm layout, from project: '#{from_title}', PID: '#{from_pid}', to project: '#{to_project.title}', PID: '#{to_project.pid}'"
78
+ res = to_project.save_ldm_layout(from_ldm_layout)
79
+
80
+ if res[:status] == 'OK' || !collect_synced_status
81
+ res[:from] = from_pid
82
+ results << res
83
+ else
84
+ warning_message = "Failed to transfer ldm layout from project: '#{from_pid}' to project: '#{to_project.title}'"
85
+ process_failed_project(pid, warning_message, failed_projects, collect_synced_status)
86
+ end
87
+ end
88
+ end
89
+ end
90
+
91
+ process_failed_projects(failed_projects, short_name, params) if collect_synced_status
92
+ # Return results
93
+ results.flatten
94
+ end
95
+ end
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,108 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require_relative 'base_action'
9
+
10
+ require 'thread_safe'
11
+ require 'json'
12
+
13
+ module GoodData
14
+ module LCM2
15
+ class SynchronizePPDashboardPermissions < BaseAction
16
+ DESCRIPTION = 'Synchronize Pixel Perfect Dashboard Permission'
17
+
18
+ PARAMS = define_params(self) do
19
+ description 'Client Used for Connecting to GD'
20
+ param :gdc_gd_client, instance_of(Type::GdClientType), required: true
21
+
22
+ description 'Client used to connecting to development domain'
23
+ param :development_client, instance_of(Type::GdClientType), required: true
24
+
25
+ description 'Synchronization Info'
26
+ param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
27
+
28
+ description 'Logger'
29
+ param :gdc_logger, instance_of(Type::GdLogger), required: true
30
+
31
+ description 'Additional Hidden Parameters'
32
+ param :additional_hidden_params, instance_of(Type::HashType), required: false
33
+
34
+ description 'Disable synchronizing dashboard permissions for Pixel Perfect dashboards'
35
+ param :disable_pp_dashboard_permission, instance_of(Type::BooleanType), required: false, default: false
36
+
37
+ description 'Abort on error'
38
+ param :abort_on_error, instance_of(Type::StringType), required: false
39
+
40
+ description 'Collect synced status'
41
+ param :collect_synced_status, instance_of(Type::BooleanType), required: false
42
+
43
+ description 'Sync failed list'
44
+ param :sync_failed_list, instance_of(Type::HashType), required: false
45
+ end
46
+
47
+ class << self
48
+ def call(params)
49
+ results = []
50
+ disable_pp_dashboard_permission = GoodData::Helpers.to_boolean(params.disable_pp_dashboard_permission)
51
+ collect_synced_status = collect_synced_status(params)
52
+ failed_projects = ThreadSafe::Array.new
53
+
54
+ if disable_pp_dashboard_permission
55
+ params.gdc_logger.info "Skip synchronize Pixel Perfect dashboard permission."
56
+ else
57
+ client = params.gdc_gd_client
58
+ development_client = params.development_client
59
+ failed_projects = ThreadSafe::Array.new
60
+
61
+ params.synchronize.peach do |info|
62
+ from_project = info.from
63
+ to_projects = info.to
64
+ sync_success = false
65
+
66
+ from = development_client.projects(from_project)
67
+ unless from
68
+ process_failed_project(from_project, "Invalid 'from' project specified - '#{from_project}'", failed_projects, collect_synced_status)
69
+ next
70
+ end
71
+
72
+ source_dashboards = from.dashboards
73
+
74
+ params.gdc_logger.info "Transferring Pixel Perfect Dashboard permission, from project: '#{from.title}', PID: '#{from.pid}' for dashboard(s): #{source_dashboards.map { |d| "#{d.title.inspect}" }.join(', ')}" # rubocop:disable Metrics/LineLength
75
+ to_projects.peach do |entry|
76
+ pid = entry[:pid]
77
+ next if sync_failed_project(pid, params)
78
+
79
+ to_project = client.projects(pid)
80
+ unless to_project
81
+ process_failed_project(pid, "Invalid 'to' project specified - '#{pid}'", failed_projects, collect_synced_status)
82
+ next
83
+ end
84
+
85
+ target_dashboards = to_project.dashboards
86
+ begin
87
+ GoodData::Project.transfer_dashboard_permission(from, to_project, source_dashboards, target_dashboards)
88
+ sync_success = true
89
+ rescue StandardError => err
90
+ process_failed_project(pid, err.message, failed_projects, collect_synced_status)
91
+ end
92
+ end
93
+
94
+ results << {
95
+ from_project_name: from.title,
96
+ from_project_pid: from.pid,
97
+ status: 'ok'
98
+ } if sync_success
99
+ end
100
+ end
101
+
102
+ process_failed_projects(failed_projects, short_name, params) if collect_synced_status
103
+ results
104
+ end
105
+ end
106
+ end
107
+ end
108
+ end
@@ -61,12 +61,13 @@ module GoodData
61
61
  to_projects = info.to
62
62
 
63
63
  from = development_client.projects(from_project) || fail("Invalid 'from' project specified - '#{from_project}'")
64
+ has_cycle_trigger = exist_cycle_trigger(from)
64
65
  to_projects.peach do |entry|
65
66
  pid = entry[:pid]
66
67
  to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
67
68
 
68
69
  params.gdc_logger.info "Transferring Schedules, from project: '#{from.title}', PID: '#{from.pid}', to project: '#{to_project.title}', PID: '#{to_project.pid}'"
69
- res = GoodData::Project.transfer_schedules(from, to_project).sort_by do |item|
70
+ res = GoodData::Project.transfer_schedules(from, to_project, has_cycle_trigger).sort_by do |item|
70
71
  item[:status]
71
72
  end
72
73
 
@@ -97,6 +98,35 @@ module GoodData
97
98
  # Return results
98
99
  results
99
100
  end
101
+
102
+ private
103
+
104
+ def exist_cycle_trigger(project)
105
+ schedules = project.schedules
106
+ triggers = {}
107
+ schedules.each do |schedule|
108
+ triggers[schedule.obj_id] = schedule.trigger_id if schedule.trigger_id
109
+ end
110
+
111
+ triggers.each do |schedule_id, trigger_id|
112
+ checking_id = trigger_id
113
+ if checking_id == schedule_id
114
+ return true
115
+ else
116
+ max_step = triggers.length
117
+ count = 1
118
+ loop do
119
+ checking_id = triggers[checking_id]
120
+ count += 1
121
+ break if !checking_id || count > max_step
122
+
123
+ return true if checking_id == schedule_id
124
+ end
125
+ end
126
+ end
127
+
128
+ false
129
+ end
100
130
  end
101
131
  end
102
132
  end
@@ -121,7 +121,7 @@ module GoodData
121
121
  users_brick_input: params.users_brick_users
122
122
  }
123
123
  all_clients = domain.clients(:all, data_product).to_a
124
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, number_of_clients=#{all_clients.size}, data_rows=#{user_filters.size}")
124
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, number_of_clients=#{all_clients.size}, data_rows=#{user_filters.size} ,")
125
125
 
126
126
  GoodData.logger.info("Synchronizing in mode \"#{mode}\"")
127
127
  results = []
@@ -134,7 +134,7 @@ module GoodData
134
134
  end
135
135
  user_filters = user_filters.select { |f| f[:pid] == filter } if filter
136
136
 
137
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{user_filters.size}")
137
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{user_filters.size} ,")
138
138
  current_results = sync_user_filters(project, user_filters, run_params, symbolized_config)
139
139
 
140
140
  results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
@@ -151,7 +151,7 @@ module GoodData
151
151
  current_project = client.projects(id)
152
152
  end
153
153
 
154
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{id}, data_rows=#{new_filters.size}")
154
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{id}, data_rows=#{new_filters.size} ,")
155
155
  current_results = sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
156
156
 
157
157
  results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
@@ -164,6 +164,7 @@ module GoodData
164
164
  end
165
165
 
166
166
  working_client_ids = []
167
+ semaphore = Mutex.new
167
168
 
168
169
  users_by_project = run_params[:users_brick_input].group_by { |u| u[:pid] }
169
170
  user_filters.group_by { |u| u[multiple_projects_column] }.flat_map.pmap do |client_id, new_filters|
@@ -182,23 +183,26 @@ module GoodData
182
183
  current_project = c.project
183
184
  fail "Client #{client_id} does not have project." unless current_project
184
185
 
185
- working_client_ids << client_id
186
+ semaphore.synchronize do
187
+ working_client_ids << client_id.to_s
188
+ end
186
189
 
187
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, client_id=#{client_id}, data_rows=#{new_filters.size}")
190
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, client_id=#{client_id}, data_rows=#{new_filters.size} ,")
188
191
  partial_results = sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
189
192
  results.concat(partial_results[:results]) unless partial_results.nil? || partial_results[:results].empty?
190
193
  end
191
194
 
192
- unless run_params[:do_not_touch_filters_that_are_not_mentioned]
193
- domain_clients.peach do |c|
194
- next if working_client_ids.include?(c.client_id)
195
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, working_client_ids=#{working_client_ids.join(', ')} ,") if working_client_ids.size < 50
195
196
 
197
+ unless run_params[:do_not_touch_filters_that_are_not_mentioned]
198
+ to_be_deleted_clients = UserBricksHelper.non_working_clients(domain_clients, working_client_ids)
199
+ to_be_deleted_clients.peach do |c|
196
200
  begin
197
201
  current_project = c.project
198
202
  users = users_by_project[c.client_id]
199
203
  params.gdc_logger.info "Delete all filters in project #{current_project.pid} of client #{c.client_id}"
200
204
 
201
- GoodData.gd_logger.info("Delete all filters in project_id=#{current_project.pid}, client_id=#{c.client_id}")
205
+ GoodData.gd_logger.info("Delete all filters in project_id=#{current_project.pid}, client_id=#{c.client_id} ,")
202
206
  current_results = sync_user_filters(current_project, [], run_params.merge(users_brick_input: users), symbolized_config)
203
207
 
204
208
  results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
@@ -208,6 +212,7 @@ module GoodData
208
212
  end
209
213
  end
210
214
  end
215
+
211
216
  {
212
217
  results: results
213
218
  }
@@ -27,11 +27,22 @@ module GoodData
27
27
 
28
28
  description 'Additional Hidden Parameters'
29
29
  param :additional_hidden_params, instance_of(Type::HashType), required: false
30
+
31
+ description 'Abort on error'
32
+ param :abort_on_error, instance_of(Type::StringType), required: false
33
+
34
+ description 'Collect synced status'
35
+ param :collect_synced_status, instance_of(Type::BooleanType), required: false
36
+
37
+ description 'Sync failed list'
38
+ param :sync_failed_list, instance_of(Type::HashType), required: false
30
39
  end
31
40
 
32
41
  class << self
33
42
  def call(params)
34
43
  results = ThreadSafe::Array.new
44
+ collect_synced_status = collect_synced_status(params)
45
+ failed_projects = ThreadSafe::Array.new
35
46
 
36
47
  client = params.gdc_gd_client
37
48
  development_client = params.development_client
@@ -40,17 +51,32 @@ module GoodData
40
51
  from_project = info.from
41
52
  to_projects = info.to
42
53
 
43
- from = development_client.projects(from_project) || fail("Invalid 'from' project specified - '#{from_project}'")
54
+ from = development_client.projects(from_project)
55
+ unless from
56
+ process_failed_project(from_project, "Invalid 'from' project specified - '#{from_project}'", failed_projects, collect_synced_status)
57
+ next
58
+ end
44
59
 
45
60
  to_projects.peach do |entry|
46
61
  pid = entry[:pid]
47
- to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
62
+ next if sync_failed_project(pid, params)
63
+
64
+ to_project = client.projects(pid)
65
+ unless to_project
66
+ process_failed_project(pid, "Invalid 'to' project specified - '#{pid}'", failed_projects, collect_synced_status)
67
+ next
68
+ end
48
69
 
49
- params.gdc_logger.info "Transferring User Groups, from project: '#{from.title}', PID: '#{from.pid}', to project: '#{to_project.title}', PID: '#{to_project.pid}'"
50
- results += GoodData::Project.transfer_user_groups(from, to_project)
70
+ begin
71
+ params.gdc_logger.info "Transferring User Groups, from project: '#{from.title}', PID: '#{from.pid}', to project: '#{to_project.title}', PID: '#{to_project.pid}'"
72
+ results += GoodData::Project.transfer_user_groups(from, to_project)
73
+ rescue StandardError => err
74
+ process_failed_project(pid, err.message, failed_projects, collect_synced_status)
75
+ end
51
76
  end
52
77
  end
53
78
 
79
+ process_failed_projects(failed_projects, short_name, params) if collect_synced_status
54
80
  results.uniq
55
81
  end
56
82
  end
@@ -190,7 +190,7 @@ module GoodData
190
190
  create_non_existing_user_groups: create_non_existing_user_groups,
191
191
  user_groups_cache: nil
192
192
  }
193
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, data_rows=#{new_users.size}")
193
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, data_rows=#{new_users.size} ,")
194
194
 
195
195
  GoodData.logger.info("Synchronizing in mode \"#{mode}\"")
196
196
  results = case mode
@@ -202,7 +202,7 @@ module GoodData
202
202
  params.gdc_logger.info "#{user_ids.count - users.count} users were not found (or were deleted) in domain #{domain_name}" if user_ids.count > users.count
203
203
  params.gdc_logger.warn "Deleting #{users.count} users from domain #{domain_name}"
204
204
 
205
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, domain=#{domain_name}, data_rows=#{users.count}")
205
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, domain=#{domain_name}, data_rows=#{users.count} ,")
206
206
  users.map(&:delete)
207
207
  when 'sync_project'
208
208
  project.import_users(new_users, common_params)
@@ -211,7 +211,7 @@ module GoodData
211
211
  begin
212
212
  project = client.projects(project_id)
213
213
 
214
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project_id}, data_rows=#{users.count}")
214
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project_id}, data_rows=#{users.count} ,")
215
215
  project.import_users(users, common_params)
216
216
  rescue RestClient::ResourceNotFound
217
217
  fail "Project \"#{project_id}\" was not found. Please check your project ids in the source file"
@@ -224,7 +224,7 @@ module GoodData
224
224
  when 'sync_one_project_based_on_pid'
225
225
  filtered_users = new_users.select { |u| u[:pid] == project.pid }
226
226
 
227
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, data_rows=#{filtered_users.count}")
227
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, data_rows=#{filtered_users.count} ,")
228
228
  project.import_users(filtered_users, common_params)
229
229
  when 'sync_one_project_based_on_custom_id'
230
230
  filter_value = UserBricksHelper.resolve_client_id(domain, project, data_product)
@@ -245,7 +245,7 @@ module GoodData
245
245
  end
246
246
 
247
247
  GoodData.logger.info("Project #{project.pid} will receive #{filtered_users.count} from #{new_users.count} users")
248
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, filtered_users=#{filtered_users.count}, data_rows=#{new_users.count}")
248
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, filtered_users=#{filtered_users.count}, data_rows=#{new_users.count} ,")
249
249
  project.import_users(filtered_users, common_params)
250
250
  when 'sync_multiple_projects_based_on_custom_id'
251
251
  all_clients = domain.clients(:all, data_product).to_a
@@ -260,7 +260,7 @@ module GoodData
260
260
 
261
261
  GoodData.logger.info("Project #{project.pid} of client #{client_id} will receive #{users.count} users")
262
262
 
263
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{users.count}")
263
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{users.count} ,")
264
264
  project.import_users(users, common_params)
265
265
  end
266
266
  when 'sync_domain_client_workspaces'
@@ -294,7 +294,7 @@ module GoodData
294
294
  working_client_ids << client_id.to_s
295
295
  GoodData.logger.info("Project #{project.pid} of client #{client_id} will receive #{users.count} users")
296
296
 
297
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{users.count}")
297
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{users.count} ,")
298
298
  project.import_users(users, common_params)
299
299
  end
300
300
 
@@ -319,17 +319,17 @@ module GoodData
319
319
  end
320
320
  GoodData.logger.info("Synchronizing all users in project #{project.pid} of client #{c.client_id}")
321
321
 
322
- GoodData.gd_logger.info("Synchronizing all users in project_id=#{project.pid}, client_id=#{c.client_id}")
322
+ GoodData.gd_logger.info("Synchronizing all users in project_id=#{project.pid}, client_id=#{c.client_id} ,")
323
323
  res += project.import_users([], common_params)
324
324
  end
325
325
  end
326
326
 
327
327
  res
328
328
  when 'sync_domain_and_project'
329
- GoodData.gd_logger.info("Create users in mode=#{mode}, data_rows=#{new_users.count}")
329
+ GoodData.gd_logger.info("Create users in mode=#{mode}, data_rows=#{new_users.count} ,")
330
330
  domain.create_users(new_users, ignore_failures: ignore_failures)
331
331
 
332
- GoodData.gd_logger.info("Import users in mode=#{mode}, data_rows=#{new_users.count}")
332
+ GoodData.gd_logger.info("Import users in mode=#{mode}, data_rows=#{new_users.count} ,")
333
333
  project.import_users(new_users, common_params)
334
334
  end
335
335
 
@@ -390,6 +390,7 @@ module GoodData
390
390
 
391
391
  user_group = row[user_groups_column] || row[user_groups_column.to_sym]
392
392
  user_group = user_group.split(',').map(&:strip) if user_group
393
+ user_group = [] if row.headers.include?(user_groups_column) && !user_group
393
394
 
394
395
  ip_whitelist = row[ip_whitelist_column] || row[ip_whitelist_column.to_sym]
395
396
  ip_whitelist = ip_whitelist.split(',').map(&:strip) if ip_whitelist