gooddata 2.2.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. checksums.yaml +5 -5
  2. data/.gdc-ii-config.yaml +42 -1
  3. data/.github/workflows/build.yml +14 -13
  4. data/.github/workflows/pre-merge.yml +13 -13
  5. data/.pronto.yml +1 -0
  6. data/.rubocop.yml +2 -14
  7. data/CHANGELOG.md +9 -0
  8. data/Dockerfile +13 -7
  9. data/Dockerfile.jruby +5 -5
  10. data/Dockerfile.ruby +5 -7
  11. data/Gemfile +4 -2
  12. data/README.md +5 -4
  13. data/Rakefile +1 -1
  14. data/SDK_VERSION +1 -1
  15. data/VERSION +1 -1
  16. data/bin/run_brick.rb +7 -0
  17. data/ci/mysql/pom.xml +6 -1
  18. data/ci/redshift/pom.xml +3 -4
  19. data/docker-compose.lcm.yml +42 -1
  20. data/docker-compose.yml +42 -0
  21. data/gooddata.gemspec +21 -22
  22. data/lcm.rake +9 -0
  23. data/lib/gooddata/bricks/base_pipeline.rb +26 -0
  24. data/lib/gooddata/bricks/brick.rb +0 -1
  25. data/lib/gooddata/bricks/middleware/execution_result_middleware.rb +3 -3
  26. data/lib/gooddata/bricks/pipeline.rb +2 -14
  27. data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +18 -8
  28. data/lib/gooddata/cloud_resources/redshift/drivers/.gitkeepme +0 -0
  29. data/lib/gooddata/cloud_resources/redshift/redshift_client.rb +0 -2
  30. data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +1 -1
  31. data/lib/gooddata/lcm/actions/base_action.rb +157 -0
  32. data/lib/gooddata/lcm/actions/collect_data_product.rb +2 -1
  33. data/lib/gooddata/lcm/actions/collect_projects_warning_status.rb +53 -0
  34. data/lib/gooddata/lcm/actions/collect_segment_clients.rb +14 -0
  35. data/lib/gooddata/lcm/actions/initialize_continue_on_error_option.rb +87 -0
  36. data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +28 -2
  37. data/lib/gooddata/lcm/actions/provision_clients.rb +34 -5
  38. data/lib/gooddata/lcm/actions/synchronize_cas.rb +24 -4
  39. data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -4
  40. data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +28 -3
  41. data/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb +48 -11
  42. data/lib/gooddata/lcm/actions/synchronize_kd_dashboard_permission.rb +103 -0
  43. data/lib/gooddata/lcm/actions/synchronize_ldm.rb +60 -15
  44. data/lib/gooddata/lcm/actions/synchronize_ldm_layout.rb +98 -0
  45. data/lib/gooddata/lcm/actions/synchronize_pp_dashboard_permission.rb +108 -0
  46. data/lib/gooddata/lcm/actions/synchronize_schedules.rb +31 -1
  47. data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +14 -9
  48. data/lib/gooddata/lcm/actions/synchronize_user_groups.rb +30 -4
  49. data/lib/gooddata/lcm/actions/synchronize_users.rb +11 -10
  50. data/lib/gooddata/lcm/actions/update_metric_formats.rb +21 -4
  51. data/lib/gooddata/lcm/exceptions/lcm_execution_warning.rb +15 -0
  52. data/lib/gooddata/lcm/helpers/check_helper.rb +19 -0
  53. data/lib/gooddata/lcm/lcm2.rb +45 -4
  54. data/lib/gooddata/lcm/user_bricks_helper.rb +9 -0
  55. data/lib/gooddata/mixins/inspector.rb +1 -1
  56. data/lib/gooddata/models/ldm_layout.rb +38 -0
  57. data/lib/gooddata/models/project.rb +197 -22
  58. data/lib/gooddata/models/project_creator.rb +83 -6
  59. data/lib/gooddata/models/segment.rb +2 -1
  60. data/lib/gooddata/models/user_filters/user_filter_builder.rb +104 -15
  61. data/lib/gooddata/rest/connection.rb +5 -3
  62. data/lib/gooddata/rest/phmap.rb +1 -0
  63. data/lib/gooddata.rb +1 -0
  64. data/lib/gooddata_brick_base.rb +35 -0
  65. data/sonar-project.properties +6 -0
  66. metadata +64 -59
  67. data/lib/gooddata/cloud_resources/redshift/drivers/log4j.properties +0 -15
@@ -5,6 +5,7 @@
5
5
  # LICENSE file in the root directory of this source tree.
6
6
 
7
7
  require_relative 'base_action'
8
+ require_relative '../helpers/helpers'
8
9
 
9
10
  using TrueExtensions
10
11
  using FalseExtensions
@@ -47,6 +48,15 @@ module GoodData
47
48
 
48
49
  description 'Enables handling of deprecated objects in the logical data model.'
49
50
  param :include_deprecated, instance_of(Type::BooleanType), required: false, default: false
51
+
52
+ description 'Abort on error'
53
+ param :abort_on_error, instance_of(Type::StringType), required: false
54
+
55
+ description 'Collect synced status'
56
+ param :collect_synced_status, instance_of(Type::BooleanType), required: false
57
+
58
+ description 'Sync failed list'
59
+ param :sync_failed_list, instance_of(Type::HashType), required: false
50
60
  end
51
61
 
52
62
  RESULT_HEADER = %i[from to status]
@@ -56,6 +66,8 @@ module GoodData
56
66
  results = []
57
67
  synchronize = []
58
68
  params.synchronize.map do |segment_info|
69
+ next if sync_failed_segment(segment_info[:segment_id], params)
70
+
59
71
  new_segment_info, segment_results = sync_segment_ldm(params, segment_info)
60
72
  results.concat(segment_results)
61
73
  synchronize << new_segment_info
@@ -72,12 +84,27 @@ module GoodData
72
84
  private
73
85
 
74
86
  def sync_segment_ldm(params, segment_info)
75
- results = []
87
+ collect_synced_status = collect_synced_status(params)
88
+ failed_projects = ThreadSafe::Array.new
89
+ results = ThreadSafe::Array.new
76
90
  client = params.gdc_gd_client
77
91
  exclude_fact_rule = params.exclude_fact_rule.to_b
78
92
  include_deprecated = params.include_deprecated.to_b
93
+ update_preference = params[:update_preference]
94
+ exist_fallback_to_hard_sync_config = !update_preference.nil? && !update_preference[:fallback_to_hard_sync].nil?
95
+ include_maql_fallback_hard_sync = exist_fallback_to_hard_sync_config && Helpers.to_bool('fallback_to_hard_sync', update_preference[:fallback_to_hard_sync])
96
+
79
97
  from_pid = segment_info[:from]
80
- from = params.development_client.projects(from_pid) || fail("Invalid 'from' project specified - '#{from_pid}'")
98
+ from = params.development_client.projects(from_pid)
99
+ unless from
100
+ segment_id = segment_info[:segment_id]
101
+ error_message = "Failed to sync LDM for segment #{segment_id}. Error: Invalid 'from' project specified - '#{from_pid}'"
102
+ fail(error_message) unless collect_synced_status
103
+
104
+ add_failed_segment(segment_id, error_message, short_name, params)
105
+ return [segment_info, results]
106
+ end
107
+
81
108
  GoodData.logger.info "Creating Blueprint, project: '#{from.title}', PID: #{from_pid}"
82
109
  blueprint = from.blueprint(include_ca: params.include_computed_attributes.to_b)
83
110
  segment_info[:from_blueprint] = blueprint
@@ -92,6 +119,8 @@ module GoodData
92
119
  maql_diff_params = [:includeGrain]
93
120
  maql_diff_params << :excludeFactRule if exclude_fact_rule
94
121
  maql_diff_params << :includeDeprecated if include_deprecated
122
+ maql_diff_params << :includeMaqlFallbackHardSync if include_maql_fallback_hard_sync
123
+
95
124
  maql_diff = previous_master.maql_diff(blueprint: blueprint, params: maql_diff_params)
96
125
  else
97
126
  maql_diff = {
@@ -102,6 +131,7 @@ module GoodData
102
131
  }
103
132
  }
104
133
  end
134
+
105
135
  chunks = maql_diff['projectModelDiff']['updateScripts']
106
136
  if chunks.empty?
107
137
  GoodData.logger.info "Synchronize LDM to clients will not proceed in mode \
@@ -112,40 +142,55 @@ to force synchronize LDM to clients"
112
142
  end
113
143
 
114
144
  segment_info[:to] = segment_info[:to].pmap do |entry|
145
+ update_status = true
115
146
  pid = entry[:pid]
116
- to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
147
+ next if sync_failed_project(pid, params)
148
+
149
+ to_project = client.projects(pid)
150
+ unless to_project
151
+ process_failed_project(pid, "Invalid 'to' project specified - '#{pid}'", failed_projects, collect_synced_status)
152
+ next
153
+ end
117
154
 
118
155
  GoodData.logger.info "Updating from Blueprint, project: '#{to_project.title}', PID: #{pid}"
119
156
  begin
120
157
  entry[:ca_scripts] = to_project.update_from_blueprint(
121
158
  blueprint,
122
- update_preference: params[:update_preference],
159
+ update_preference: update_preference,
123
160
  exclude_fact_rule: exclude_fact_rule,
124
161
  execute_ca_scripts: false,
125
162
  maql_diff: maql_diff,
126
163
  include_deprecated: include_deprecated
127
164
  )
128
165
  rescue MaqlExecutionError => e
129
- GoodData.logger.info("Applying MAQL to project #{to_project.title} - #{pid} failed. Reason: #{e}")
130
- fail e unless previous_master && params[:synchronize_ldm] == 'diff_against_master_with_fallback'
131
- GoodData.logger.info("Restoring the client project #{to_project.title} from master.")
132
- entry[:ca_scripts] = to_project.update_from_blueprint(
133
- blueprint,
134
- update_preference: params[:update_preference],
135
- exclude_fact_rule: exclude_fact_rule,
136
- execute_ca_scripts: false,
137
- include_deprecated: include_deprecated
138
- )
166
+ if collect_synced_status
167
+ update_status = false
168
+ failed_message = "Applying MAQL to project #{to_project.title} - #{pid} failed. Reason: #{e}"
169
+ process_failed_project(pid, failed_message, failed_projects, collect_synced_status)
170
+ else
171
+ fail e unless previous_master && params[:synchronize_ldm] == 'diff_against_master_with_fallback'
172
+
173
+ GoodData.logger.info("Restoring the client project #{to_project.title} from master.")
174
+ entry[:ca_scripts] = to_project.update_from_blueprint(
175
+ blueprint,
176
+ update_preference: update_preference,
177
+ exclude_fact_rule: exclude_fact_rule,
178
+ execute_ca_scripts: false,
179
+ include_deprecated: include_deprecated
180
+ )
181
+ end
139
182
  end
140
183
 
141
184
  results << {
142
185
  from: from_pid,
143
186
  to: pid,
144
187
  status: 'ok'
145
- }
188
+ } if update_status
189
+
146
190
  entry
147
191
  end
148
192
 
193
+ process_failed_projects(failed_projects, short_name, params) if collect_synced_status
149
194
  [segment_info, results]
150
195
  end
151
196
  end
@@ -0,0 +1,98 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2022 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require_relative 'base_action'
9
+
10
+ module GoodData
11
+ module LCM2
12
+ class SynchronizeLdmLayout < BaseAction
13
+ DESCRIPTION = 'Synchronize LDM Layout'
14
+
15
+ PARAMS = define_params(self) do
16
+ description 'Logger'
17
+ param :gdc_logger, instance_of(Type::GdLogger), required: true
18
+
19
+ description 'Client Used for Connecting to GD'
20
+ param :gdc_gd_client, instance_of(Type::GdClientType), required: true
21
+
22
+ description 'Client used to connecting to development domain'
23
+ param :development_client, instance_of(Type::GdClientType), required: true
24
+
25
+ description 'Synchronization Info'
26
+ param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
27
+
28
+ description 'Abort on error'
29
+ param :abort_on_error, instance_of(Type::StringType), required: false
30
+
31
+ description 'Collect synced status'
32
+ param :collect_synced_status, instance_of(Type::BooleanType), required: false
33
+
34
+ description 'Sync failed list'
35
+ param :sync_failed_list, instance_of(Type::HashType), required: false
36
+ end
37
+
38
+ RESULT_HEADER = %i[from to status]
39
+
40
+ class << self
41
+ def call(params)
42
+ results = []
43
+
44
+ client = params.gdc_gd_client
45
+ development_client = params.development_client
46
+ gdc_logger = params.gdc_logger
47
+ collect_synced_status = collect_synced_status(params)
48
+ failed_projects = ThreadSafe::Array.new
49
+
50
+ params.synchronize.peach do |info|
51
+ from_project = info.from
52
+ to_projects = info.to
53
+
54
+ from = development_client.projects(from_project)
55
+ unless from
56
+ process_failed_project(from_project, "Invalid 'from' project specified - '#{from_project}'", failed_projects, collect_synced_status)
57
+ next
58
+ end
59
+
60
+ from_pid = from.pid
61
+ from_title = from.title
62
+ from_ldm_layout = from.ldm_layout
63
+
64
+ if from_ldm_layout&.dig('ldmLayout', 'layout').nil? || from_ldm_layout['ldmLayout']['layout'].empty?
65
+ gdc_logger.info "Project: '#{from_title}', PID: '#{from_pid}' has no ldm layout, skip synchronizing ldm layout."
66
+ else
67
+ to_projects.peach do |to|
68
+ pid = to[:pid]
69
+ to_project = client.projects(pid)
70
+ unless to_project
71
+ process_failed_project(pid, "Invalid 'from' project specified - '#{pid}'", failed_projects, collect_synced_status)
72
+ next
73
+ end
74
+
75
+ next if sync_failed_project(pid, params)
76
+
77
+ gdc_logger.info "Transferring ldm layout, from project: '#{from_title}', PID: '#{from_pid}', to project: '#{to_project.title}', PID: '#{to_project.pid}'"
78
+ res = to_project.save_ldm_layout(from_ldm_layout)
79
+
80
+ if res[:status] == 'OK' || !collect_synced_status
81
+ res[:from] = from_pid
82
+ results << res
83
+ else
84
+ warning_message = "Failed to transfer ldm layout from project: '#{from_pid}' to project: '#{to_project.title}'"
85
+ process_failed_project(pid, warning_message, failed_projects, collect_synced_status)
86
+ end
87
+ end
88
+ end
89
+ end
90
+
91
+ process_failed_projects(failed_projects, short_name, params) if collect_synced_status
92
+ # Return results
93
+ results.flatten
94
+ end
95
+ end
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,108 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require_relative 'base_action'
9
+
10
+ require 'thread_safe'
11
+ require 'json'
12
+
13
+ module GoodData
14
+ module LCM2
15
+ class SynchronizePPDashboardPermissions < BaseAction
16
+ DESCRIPTION = 'Synchronize Pixel Perfect Dashboard Permission'
17
+
18
+ PARAMS = define_params(self) do
19
+ description 'Client Used for Connecting to GD'
20
+ param :gdc_gd_client, instance_of(Type::GdClientType), required: true
21
+
22
+ description 'Client used to connecting to development domain'
23
+ param :development_client, instance_of(Type::GdClientType), required: true
24
+
25
+ description 'Synchronization Info'
26
+ param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
27
+
28
+ description 'Logger'
29
+ param :gdc_logger, instance_of(Type::GdLogger), required: true
30
+
31
+ description 'Additional Hidden Parameters'
32
+ param :additional_hidden_params, instance_of(Type::HashType), required: false
33
+
34
+ description 'Disable synchronizing dashboard permissions for Pixel Perfect dashboards'
35
+ param :disable_pp_dashboard_permission, instance_of(Type::BooleanType), required: false, default: false
36
+
37
+ description 'Abort on error'
38
+ param :abort_on_error, instance_of(Type::StringType), required: false
39
+
40
+ description 'Collect synced status'
41
+ param :collect_synced_status, instance_of(Type::BooleanType), required: false
42
+
43
+ description 'Sync failed list'
44
+ param :sync_failed_list, instance_of(Type::HashType), required: false
45
+ end
46
+
47
+ class << self
48
+ def call(params)
49
+ results = []
50
+ disable_pp_dashboard_permission = GoodData::Helpers.to_boolean(params.disable_pp_dashboard_permission)
51
+ collect_synced_status = collect_synced_status(params)
52
+ failed_projects = ThreadSafe::Array.new
53
+
54
+ if disable_pp_dashboard_permission
55
+ params.gdc_logger.info "Skip synchronize Pixel Perfect dashboard permission."
56
+ else
57
+ client = params.gdc_gd_client
58
+ development_client = params.development_client
59
+ failed_projects = ThreadSafe::Array.new
60
+
61
+ params.synchronize.peach do |info|
62
+ from_project = info.from
63
+ to_projects = info.to
64
+ sync_success = false
65
+
66
+ from = development_client.projects(from_project)
67
+ unless from
68
+ process_failed_project(from_project, "Invalid 'from' project specified - '#{from_project}'", failed_projects, collect_synced_status)
69
+ next
70
+ end
71
+
72
+ source_dashboards = from.dashboards
73
+
74
+ params.gdc_logger.info "Transferring Pixel Perfect Dashboard permission, from project: '#{from.title}', PID: '#{from.pid}' for dashboard(s): #{source_dashboards.map { |d| "#{d.title.inspect}" }.join(', ')}" # rubocop:disable Metrics/LineLength
75
+ to_projects.peach do |entry|
76
+ pid = entry[:pid]
77
+ next if sync_failed_project(pid, params)
78
+
79
+ to_project = client.projects(pid)
80
+ unless to_project
81
+ process_failed_project(pid, "Invalid 'to' project specified - '#{pid}'", failed_projects, collect_synced_status)
82
+ next
83
+ end
84
+
85
+ target_dashboards = to_project.dashboards
86
+ begin
87
+ GoodData::Project.transfer_dashboard_permission(from, to_project, source_dashboards, target_dashboards)
88
+ sync_success = true
89
+ rescue StandardError => err
90
+ process_failed_project(pid, err.message, failed_projects, collect_synced_status)
91
+ end
92
+ end
93
+
94
+ results << {
95
+ from_project_name: from.title,
96
+ from_project_pid: from.pid,
97
+ status: 'ok'
98
+ } if sync_success
99
+ end
100
+ end
101
+
102
+ process_failed_projects(failed_projects, short_name, params) if collect_synced_status
103
+ results
104
+ end
105
+ end
106
+ end
107
+ end
108
+ end
@@ -61,12 +61,13 @@ module GoodData
61
61
  to_projects = info.to
62
62
 
63
63
  from = development_client.projects(from_project) || fail("Invalid 'from' project specified - '#{from_project}'")
64
+ has_cycle_trigger = exist_cycle_trigger(from)
64
65
  to_projects.peach do |entry|
65
66
  pid = entry[:pid]
66
67
  to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
67
68
 
68
69
  params.gdc_logger.info "Transferring Schedules, from project: '#{from.title}', PID: '#{from.pid}', to project: '#{to_project.title}', PID: '#{to_project.pid}'"
69
- res = GoodData::Project.transfer_schedules(from, to_project).sort_by do |item|
70
+ res = GoodData::Project.transfer_schedules(from, to_project, has_cycle_trigger).sort_by do |item|
70
71
  item[:status]
71
72
  end
72
73
 
@@ -97,6 +98,35 @@ module GoodData
97
98
  # Return results
98
99
  results
99
100
  end
101
+
102
+ private
103
+
104
+ def exist_cycle_trigger(project)
105
+ schedules = project.schedules
106
+ triggers = {}
107
+ schedules.each do |schedule|
108
+ triggers[schedule.obj_id] = schedule.trigger_id if schedule.trigger_id
109
+ end
110
+
111
+ triggers.each do |schedule_id, trigger_id|
112
+ checking_id = trigger_id
113
+ if checking_id == schedule_id
114
+ return true
115
+ else
116
+ max_step = triggers.length
117
+ count = 1
118
+ loop do
119
+ checking_id = triggers[checking_id]
120
+ count += 1
121
+ break if !checking_id || count > max_step
122
+
123
+ return true if checking_id == schedule_id
124
+ end
125
+ end
126
+ end
127
+
128
+ false
129
+ end
100
130
  end
101
131
  end
102
132
  end
@@ -121,7 +121,7 @@ module GoodData
121
121
  users_brick_input: params.users_brick_users
122
122
  }
123
123
  all_clients = domain.clients(:all, data_product).to_a
124
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, number_of_clients=#{all_clients.size}, data_rows=#{user_filters.size}")
124
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, number_of_clients=#{all_clients.size}, data_rows=#{user_filters.size} ,")
125
125
 
126
126
  GoodData.logger.info("Synchronizing in mode \"#{mode}\"")
127
127
  results = []
@@ -134,7 +134,7 @@ module GoodData
134
134
  end
135
135
  user_filters = user_filters.select { |f| f[:pid] == filter } if filter
136
136
 
137
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{user_filters.size}")
137
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{user_filters.size} ,")
138
138
  current_results = sync_user_filters(project, user_filters, run_params, symbolized_config)
139
139
 
140
140
  results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
@@ -151,7 +151,7 @@ module GoodData
151
151
  current_project = client.projects(id)
152
152
  end
153
153
 
154
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{id}, data_rows=#{new_filters.size}")
154
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{id}, data_rows=#{new_filters.size} ,")
155
155
  current_results = sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
156
156
 
157
157
  results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
@@ -164,6 +164,7 @@ module GoodData
164
164
  end
165
165
 
166
166
  working_client_ids = []
167
+ semaphore = Mutex.new
167
168
 
168
169
  users_by_project = run_params[:users_brick_input].group_by { |u| u[:pid] }
169
170
  user_filters.group_by { |u| u[multiple_projects_column] }.flat_map.pmap do |client_id, new_filters|
@@ -182,23 +183,26 @@ module GoodData
182
183
  current_project = c.project
183
184
  fail "Client #{client_id} does not have project." unless current_project
184
185
 
185
- working_client_ids << client_id
186
+ semaphore.synchronize do
187
+ working_client_ids << client_id.to_s
188
+ end
186
189
 
187
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, client_id=#{client_id}, data_rows=#{new_filters.size}")
190
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, client_id=#{client_id}, data_rows=#{new_filters.size} ,")
188
191
  partial_results = sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
189
192
  results.concat(partial_results[:results]) unless partial_results.nil? || partial_results[:results].empty?
190
193
  end
191
194
 
192
- unless run_params[:do_not_touch_filters_that_are_not_mentioned]
193
- domain_clients.peach do |c|
194
- next if working_client_ids.include?(c.client_id)
195
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, working_client_ids=#{working_client_ids.join(', ')} ,") if working_client_ids.size < 50
195
196
 
197
+ unless run_params[:do_not_touch_filters_that_are_not_mentioned]
198
+ to_be_deleted_clients = UserBricksHelper.non_working_clients(domain_clients, working_client_ids)
199
+ to_be_deleted_clients.peach do |c|
196
200
  begin
197
201
  current_project = c.project
198
202
  users = users_by_project[c.client_id]
199
203
  params.gdc_logger.info "Delete all filters in project #{current_project.pid} of client #{c.client_id}"
200
204
 
201
- GoodData.gd_logger.info("Delete all filters in project_id=#{current_project.pid}, client_id=#{c.client_id}")
205
+ GoodData.gd_logger.info("Delete all filters in project_id=#{current_project.pid}, client_id=#{c.client_id} ,")
202
206
  current_results = sync_user_filters(current_project, [], run_params.merge(users_brick_input: users), symbolized_config)
203
207
 
204
208
  results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
@@ -208,6 +212,7 @@ module GoodData
208
212
  end
209
213
  end
210
214
  end
215
+
211
216
  {
212
217
  results: results
213
218
  }
@@ -27,11 +27,22 @@ module GoodData
27
27
 
28
28
  description 'Additional Hidden Parameters'
29
29
  param :additional_hidden_params, instance_of(Type::HashType), required: false
30
+
31
+ description 'Abort on error'
32
+ param :abort_on_error, instance_of(Type::StringType), required: false
33
+
34
+ description 'Collect synced status'
35
+ param :collect_synced_status, instance_of(Type::BooleanType), required: false
36
+
37
+ description 'Sync failed list'
38
+ param :sync_failed_list, instance_of(Type::HashType), required: false
30
39
  end
31
40
 
32
41
  class << self
33
42
  def call(params)
34
43
  results = ThreadSafe::Array.new
44
+ collect_synced_status = collect_synced_status(params)
45
+ failed_projects = ThreadSafe::Array.new
35
46
 
36
47
  client = params.gdc_gd_client
37
48
  development_client = params.development_client
@@ -40,17 +51,32 @@ module GoodData
40
51
  from_project = info.from
41
52
  to_projects = info.to
42
53
 
43
- from = development_client.projects(from_project) || fail("Invalid 'from' project specified - '#{from_project}'")
54
+ from = development_client.projects(from_project)
55
+ unless from
56
+ process_failed_project(from_project, "Invalid 'from' project specified - '#{from_project}'", failed_projects, collect_synced_status)
57
+ next
58
+ end
44
59
 
45
60
  to_projects.peach do |entry|
46
61
  pid = entry[:pid]
47
- to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
62
+ next if sync_failed_project(pid, params)
63
+
64
+ to_project = client.projects(pid)
65
+ unless to_project
66
+ process_failed_project(pid, "Invalid 'to' project specified - '#{pid}'", failed_projects, collect_synced_status)
67
+ next
68
+ end
48
69
 
49
- params.gdc_logger.info "Transferring User Groups, from project: '#{from.title}', PID: '#{from.pid}', to project: '#{to_project.title}', PID: '#{to_project.pid}'"
50
- results += GoodData::Project.transfer_user_groups(from, to_project)
70
+ begin
71
+ params.gdc_logger.info "Transferring User Groups, from project: '#{from.title}', PID: '#{from.pid}', to project: '#{to_project.title}', PID: '#{to_project.pid}'"
72
+ results += GoodData::Project.transfer_user_groups(from, to_project)
73
+ rescue StandardError => err
74
+ process_failed_project(pid, err.message, failed_projects, collect_synced_status)
75
+ end
51
76
  end
52
77
  end
53
78
 
79
+ process_failed_projects(failed_projects, short_name, params) if collect_synced_status
54
80
  results.uniq
55
81
  end
56
82
  end
@@ -190,7 +190,7 @@ module GoodData
190
190
  create_non_existing_user_groups: create_non_existing_user_groups,
191
191
  user_groups_cache: nil
192
192
  }
193
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, data_rows=#{new_users.size}")
193
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, data_rows=#{new_users.size} ,")
194
194
 
195
195
  GoodData.logger.info("Synchronizing in mode \"#{mode}\"")
196
196
  results = case mode
@@ -202,7 +202,7 @@ module GoodData
202
202
  params.gdc_logger.info "#{user_ids.count - users.count} users were not found (or were deleted) in domain #{domain_name}" if user_ids.count > users.count
203
203
  params.gdc_logger.warn "Deleting #{users.count} users from domain #{domain_name}"
204
204
 
205
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, domain=#{domain_name}, data_rows=#{users.count}")
205
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, domain=#{domain_name}, data_rows=#{users.count} ,")
206
206
  users.map(&:delete)
207
207
  when 'sync_project'
208
208
  project.import_users(new_users, common_params)
@@ -211,7 +211,7 @@ module GoodData
211
211
  begin
212
212
  project = client.projects(project_id)
213
213
 
214
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project_id}, data_rows=#{users.count}")
214
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project_id}, data_rows=#{users.count} ,")
215
215
  project.import_users(users, common_params)
216
216
  rescue RestClient::ResourceNotFound
217
217
  fail "Project \"#{project_id}\" was not found. Please check your project ids in the source file"
@@ -224,7 +224,7 @@ module GoodData
224
224
  when 'sync_one_project_based_on_pid'
225
225
  filtered_users = new_users.select { |u| u[:pid] == project.pid }
226
226
 
227
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, data_rows=#{filtered_users.count}")
227
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, data_rows=#{filtered_users.count} ,")
228
228
  project.import_users(filtered_users, common_params)
229
229
  when 'sync_one_project_based_on_custom_id'
230
230
  filter_value = UserBricksHelper.resolve_client_id(domain, project, data_product)
@@ -245,7 +245,7 @@ module GoodData
245
245
  end
246
246
 
247
247
  GoodData.logger.info("Project #{project.pid} will receive #{filtered_users.count} from #{new_users.count} users")
248
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, filtered_users=#{filtered_users.count}, data_rows=#{new_users.count}")
248
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, filtered_users=#{filtered_users.count}, data_rows=#{new_users.count} ,")
249
249
  project.import_users(filtered_users, common_params)
250
250
  when 'sync_multiple_projects_based_on_custom_id'
251
251
  all_clients = domain.clients(:all, data_product).to_a
@@ -260,7 +260,7 @@ module GoodData
260
260
 
261
261
  GoodData.logger.info("Project #{project.pid} of client #{client_id} will receive #{users.count} users")
262
262
 
263
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{users.count}")
263
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{users.count} ,")
264
264
  project.import_users(users, common_params)
265
265
  end
266
266
  when 'sync_domain_client_workspaces'
@@ -294,7 +294,7 @@ module GoodData
294
294
  working_client_ids << client_id.to_s
295
295
  GoodData.logger.info("Project #{project.pid} of client #{client_id} will receive #{users.count} users")
296
296
 
297
- GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{users.count}")
297
+ GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{users.count} ,")
298
298
  project.import_users(users, common_params)
299
299
  end
300
300
 
@@ -319,17 +319,17 @@ module GoodData
319
319
  end
320
320
  GoodData.logger.info("Synchronizing all users in project #{project.pid} of client #{c.client_id}")
321
321
 
322
- GoodData.gd_logger.info("Synchronizing all users in project_id=#{project.pid}, client_id=#{c.client_id}")
322
+ GoodData.gd_logger.info("Synchronizing all users in project_id=#{project.pid}, client_id=#{c.client_id} ,")
323
323
  res += project.import_users([], common_params)
324
324
  end
325
325
  end
326
326
 
327
327
  res
328
328
  when 'sync_domain_and_project'
329
- GoodData.gd_logger.info("Create users in mode=#{mode}, data_rows=#{new_users.count}")
329
+ GoodData.gd_logger.info("Create users in mode=#{mode}, data_rows=#{new_users.count} ,")
330
330
  domain.create_users(new_users, ignore_failures: ignore_failures)
331
331
 
332
- GoodData.gd_logger.info("Import users in mode=#{mode}, data_rows=#{new_users.count}")
332
+ GoodData.gd_logger.info("Import users in mode=#{mode}, data_rows=#{new_users.count} ,")
333
333
  project.import_users(new_users, common_params)
334
334
  end
335
335
 
@@ -390,6 +390,7 @@ module GoodData
390
390
 
391
391
  user_group = row[user_groups_column] || row[user_groups_column.to_sym]
392
392
  user_group = user_group.split(',').map(&:strip) if user_group
393
+ user_group = [] if row.headers.include?(user_groups_column) && !user_group
393
394
 
394
395
  ip_whitelist = row[ip_whitelist_column] || row[ip_whitelist_column.to_sym]
395
396
  ip_whitelist = ip_whitelist.split(',').map(&:strip) if ip_whitelist