gooddata 2.1.14 → 2.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (78) hide show
  1. checksums.yaml +5 -5
  2. data/.gdc-ii-config.yaml +1 -1
  3. data/.github/workflows/build.yml +66 -0
  4. data/.github/workflows/pre-merge.yml +72 -0
  5. data/.sonar.settings +4 -0
  6. data/.travis.yml +78 -12
  7. data/CHANGELOG.md +62 -0
  8. data/Dockerfile +25 -14
  9. data/LICENSE +4418 -17
  10. data/LICENSE.rb +1 -1
  11. data/README.md +3 -3
  12. data/Rakefile +8 -1
  13. data/SDK_VERSION +1 -1
  14. data/VERSION +1 -1
  15. data/bin/test_projects_cleanup.rb +45 -3
  16. data/ci/mssql/pom.xml +62 -0
  17. data/ci/mysql/pom.xml +57 -0
  18. data/ci/postgresql/pom.xml +57 -0
  19. data/ci/redshift/pom.xml +1 -1
  20. data/dev-gooddata-sso.pub.encrypted +40 -40
  21. data/gdc_fossa_ruby_sdk.yaml +1 -0
  22. data/gooddata.gemspec +10 -6
  23. data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
  24. data/k8s/charts/lcm-bricks/templates/prometheus/alertingRules.yaml +11 -1
  25. data/lcm.rake +2 -2
  26. data/lib/gooddata/bricks/middleware/aws_middleware.rb +35 -9
  27. data/lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb +98 -0
  28. data/lib/gooddata/cloud_resources/{cloud_resouce_factory.rb → cloud_resource_factory.rb} +8 -0
  29. data/lib/gooddata/cloud_resources/cloud_resources.rb +1 -1
  30. data/lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme +0 -0
  31. data/lib/gooddata/cloud_resources/mssql/mssql_client.rb +122 -0
  32. data/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme +0 -0
  33. data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +111 -0
  34. data/lib/gooddata/cloud_resources/postgresql/drivers/.gitkeepme +0 -0
  35. data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +106 -0
  36. data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +18 -1
  37. data/lib/gooddata/commands/scaffold.rb +9 -10
  38. data/lib/gooddata/core/nil_logger.rb +3 -1
  39. data/lib/gooddata/helpers/data_helper.rb +9 -5
  40. data/lib/gooddata/helpers/global_helpers.rb +6 -5
  41. data/lib/gooddata/lcm/actions/associate_clients.rb +8 -2
  42. data/lib/gooddata/lcm/actions/base_action.rb +0 -2
  43. data/lib/gooddata/lcm/actions/collect_meta.rb +3 -1
  44. data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +3 -2
  45. data/lib/gooddata/lcm/actions/provision_clients.rb +31 -10
  46. data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -7
  47. data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +64 -0
  48. data/lib/gooddata/lcm/actions/synchronize_ldm.rb +19 -8
  49. data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +16 -9
  50. data/lib/gooddata/lcm/actions/synchronize_users.rb +7 -6
  51. data/lib/gooddata/lcm/actions/update_metric_formats.rb +185 -0
  52. data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb +5 -0
  53. data/lib/gooddata/lcm/helpers/release_table_helper.rb +42 -8
  54. data/lib/gooddata/lcm/lcm2.rb +5 -2
  55. data/lib/gooddata/lcm/types/base_type.rb +0 -2
  56. data/lib/gooddata/mixins/md_object_query.rb +9 -6
  57. data/lib/gooddata/models/blueprint/project_blueprint.rb +0 -2
  58. data/lib/gooddata/models/client.rb +14 -12
  59. data/lib/gooddata/models/data_source.rb +668 -0
  60. data/lib/gooddata/models/dataset_mapping.rb +36 -0
  61. data/lib/gooddata/models/domain.rb +3 -2
  62. data/lib/gooddata/models/metadata/analytical_dashboard.rb +49 -0
  63. data/lib/gooddata/models/metadata/analytical_visualization_object.rb +30 -0
  64. data/lib/gooddata/models/metadata/label.rb +26 -27
  65. data/lib/gooddata/models/metadata/visualization_object.rb +50 -0
  66. data/lib/gooddata/models/project.rb +66 -19
  67. data/lib/gooddata/models/schedule.rb +13 -1
  68. data/lib/gooddata/models/user_filters/user_filter_builder.rb +58 -54
  69. data/lib/gooddata/models/user_group.rb +0 -1
  70. data/lib/gooddata/rest/connection.rb +6 -4
  71. data/lib/gooddata/rest/phmap.rb +2 -1
  72. data/lib/gooddata.rb +2 -0
  73. data/rubydev_public.gpg.encrypted +51 -51
  74. data/rubydev_secret_keys.gpg.encrypted +109 -109
  75. metadata +52 -27
  76. data/DEPENDENCIES.md +0 -880
  77. data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb +0 -37
  78. data/lib/gooddata/helpers/data_source_helpers.rb +0 -47
@@ -43,7 +43,7 @@ module GoodData
43
43
  RESULT_HEADER = [
44
44
  :id,
45
45
  :status,
46
- :originalProject,
46
+ :project,
47
47
  :client,
48
48
  :type
49
49
  ]
@@ -102,7 +102,13 @@ module GoodData
102
102
  options = { delete_projects: delete_projects }
103
103
  options.merge!(delete_extra_option(params, delete_extra)) if delete_extra
104
104
 
105
- domain.update_clients(params.clients, options)
105
+ results = domain.update_clients(params.clients, options)
106
+ # Update status to CREATED if the client has no project
107
+ results&.each do |r|
108
+ r[:status] = 'CREATED' if r[:originalProject].nil?
109
+ r[:project] = r[:originalProject]
110
+ end
111
+ results
106
112
  end
107
113
 
108
114
  private
@@ -8,8 +8,6 @@ require 'gooddata/extensions/integer'
8
8
  require 'gooddata/extensions/string'
9
9
  require 'gooddata/extensions/nil'
10
10
 
11
- require 'active_support/core_ext/hash/compact'
12
-
13
11
  require_relative '../dsl/dsl'
14
12
  require_relative '../helpers/helpers'
15
13
  require_relative '../types/types'
@@ -53,7 +53,9 @@ module GoodData
53
53
  client: development_client
54
54
  )
55
55
  kpi_dashboards = MdObject.query('analyticalDashboard', MdObject, client: development_client, project: from_project)
56
- objects = old_dashboards.to_a + kpi_dashboards.to_a
56
+ kpi_dashboard_plugin = MdObject.query('dashboardPlugin', MdObject, client: development_client, project: from_project)
57
+ kpi_date_filter_config = MdObject.query('dateFilterConfig', MdObject, client: development_client, project: from_project)
58
+ objects = old_dashboards.to_a + kpi_dashboards.to_a + kpi_dashboard_plugin.to_a + kpi_date_filter_config.to_a
57
59
  else
58
60
  objects = GoodData::Dashboard.find_by_tag(
59
61
  production_tags,
@@ -50,6 +50,7 @@ module GoodData
50
50
  segment_info[:to].pmap do |entry|
51
51
  pid = entry[:pid]
52
52
  to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
53
+ GoodData.logger.info "Migrating date dimension, project: '#{to_project.title}', PID: #{pid}"
53
54
  to_blueprint = to_project.blueprint
54
55
  upgrade_datasets = get_upgrade_dates(latest_blueprint, to_blueprint)
55
56
  next if upgrade_datasets.empty?
@@ -71,9 +72,9 @@ module GoodData
71
72
  dest_dates = get_date_dimensions(dest_blueprint) if dest_blueprint
72
73
  src_dates = get_date_dimensions(src_blueprint) if src_blueprint
73
74
 
74
- return false if dest_dates.empty? || src_dates.empty?
75
-
76
75
  upgrade_datasets = []
76
+ return upgrade_datasets if dest_dates.empty? || src_dates.empty?
77
+
77
78
  dest_dates.each do |dest|
78
79
  src_dim = get_date_dimension(src_blueprint, dest[:id])
79
80
  next unless src_dim
@@ -48,7 +48,8 @@ module GoodData
48
48
  domain_name = params.organization || params.domain
49
49
  fail "Either organisation or domain has to be specified in params" unless domain_name
50
50
  domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
51
-
51
+ error_message = nil
52
+ invalid_client_ids = []
52
53
  begin
53
54
  results = params.segments.map do |segment|
54
55
  segment_object = domain.segments(segment.segment_id, data_product)
@@ -57,33 +58,53 @@ module GoodData
57
58
  end
58
59
 
59
60
  unless tmp.empty?
60
- synchronize_projects << {
61
+ synchronize_project = {
61
62
  segment_id: segment.segment_id,
62
63
  from: segment.development_pid,
63
64
  to: tmp.map do |entry|
64
65
  unless entry[:project_uri]
65
- raise "Provisioning project for client id #{entry[:id]} has error: #{entry[:error]}"
66
+ error_message = "There was error during provisioning clients: #{entry[:error]}" unless error_message
67
+ invalid_client_ids << entry[:id]
68
+ next
66
69
  end
67
70
  {
68
71
  pid: entry[:project_uri].split('/').last,
69
72
  client_id: entry[:id]
70
73
  }
71
- end
74
+ end.compact
72
75
  }
76
+
77
+ synchronize_projects << synchronize_project unless synchronize_project[:to].empty?
78
+ end
79
+
80
+ if error_message
81
+ params.gdc_logger.debug "#{error_message}. Purge all invalid clients now ..."
82
+ deleted_client_ids = []
83
+
84
+ segment_object.clients.map do |segment_client|
85
+ project = segment_client.project
86
+ if (project.nil? || project.deleted?)
87
+ client_id = segment_client.client_id
88
+ if invalid_client_ids.include?(client_id)
89
+ segment_client.delete
90
+ deleted_client_ids << client_id
91
+ end
92
+ end
93
+ end
94
+
95
+ params.gdc_logger.debug "Deleted clients: #{deleted_client_ids.join(', ')}"
96
+ raise error_message unless error_message['TooManyProjectsCreatedException'] || error_message['Max number registered projects']
97
+ break tmp
73
98
  end
74
99
 
75
100
  tmp
76
101
  end
77
102
  rescue => e
78
- params.gdc_logger.error "Problem occurs when provisioning clients. Purge all invalid clients now ..."
79
- res = LCM2.run_action PurgeClients, params
80
- params.gdc_logger.debug "Purge clients result: #{res}"
81
- deleted_client_ids = res[:results].select { |r| r[:status] == 'purged' }.map { |r| r[:client_id] }
82
- params.gdc_logger.error "Deleted clients: #{deleted_client_ids.join(', ')}"
103
+ params.gdc_logger.error "Problem occurs when provisioning clients."
83
104
  raise e
84
105
  end
85
106
 
86
- results.flatten!
107
+ results.flatten! if results
87
108
 
88
109
  # Return results
89
110
  {
@@ -33,6 +33,9 @@ module GoodData
33
33
  description 'ADS Client'
34
34
  param :ads_client, instance_of(Type::AdsClientType), required: false
35
35
 
36
+ description 'Keep number of old master workspace excluding the latest one'
37
+ param :keep_only_previous_masters_count, instance_of(Type::StringType), required: false, default: '-1'
38
+
36
39
  description 'Additional Hidden Parameters'
37
40
  param :additional_hidden_params, instance_of(Type::HashType), required: false
38
41
  end
@@ -53,6 +56,7 @@ module GoodData
53
56
  domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
54
57
  data_product = params.data_product
55
58
  domain_segments = domain.segments(:all, data_product)
59
+ keep_only_previous_masters_count = Integer(params.keep_only_previous_masters_count || "-1")
56
60
 
57
61
  segments = params.segments.map do |seg|
58
62
  domain_segments.find do |s|
@@ -62,18 +66,14 @@ module GoodData
62
66
 
63
67
  results = segments.map do |segment|
64
68
  if params.ads_client
65
- current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
66
- params.release_table_name,
67
- params.ads_client,
68
- segment.segment_id
69
- )
69
+ master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_ads(params.release_table_name, params.ads_client, segment.segment_id)
70
70
  else
71
- current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
71
+ master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
72
72
  end
73
73
 
74
+ current_master = master_projects.last
74
75
  # TODO: Check res.first.nil? || res.first[:master_project_id].nil?
75
76
  master = client.projects(current_master[:master_project_id])
76
-
77
77
  segment.master_project = master
78
78
  segment.save
79
79
 
@@ -87,6 +87,19 @@ module GoodData
87
87
  "Details: #{sync_result['links']['details']}")
88
88
  end
89
89
 
90
+ if keep_only_previous_masters_count >= 0
91
+ number_of_deleted_projects = master_projects.count - (keep_only_previous_masters_count + 1)
92
+
93
+ if number_of_deleted_projects.positive?
94
+ begin
95
+ removal_master_project_ids = remove_multiple_workspace(params, segment.segment_id, master_projects, number_of_deleted_projects)
96
+ remove_old_workspaces_from_release_table(params, domain_name, data_product.data_product_id, segment.segment_id, master_projects, removal_master_project_ids)
97
+ rescue Exception => e # rubocop:disable RescueException
98
+ GoodData.logger.error "Problem occurs when removing old master workspace, reason: #{e.message}"
99
+ end
100
+ end
101
+ end
102
+
90
103
  {
91
104
  segment: segment.id,
92
105
  master_pid: master.pid,
@@ -98,6 +111,42 @@ module GoodData
98
111
  # Return results
99
112
  results
100
113
  end
114
+
115
+ def remove_multiple_workspace(params, segment_id, master_projects, number_of_deleted_projects)
116
+ removal_master_project_ids = []
117
+ need_to_delete_projects = master_projects.take(number_of_deleted_projects)
118
+
119
+ need_to_delete_projects.each do |project_wrapper|
120
+ master_project_id = project_wrapper[:master_project_id]
121
+ next if master_project_id.to_s.empty?
122
+
123
+ begin
124
+ project = params.gdc_gd_client.projects(master_project_id)
125
+ if project && !%w[deleted archived].include?(project.state.to_s)
126
+ GoodData.logger.info "Segment #{segment_id}: Deleting old master workspace, project: '#{project.title}', PID: (#{project.pid})."
127
+ project.delete
128
+ end
129
+ removal_master_project_ids << master_project_id
130
+ master_projects.delete_if { |p| p[:master_project_id] == master_project_id }
131
+ rescue Exception => ex # rubocop:disable RescueException
132
+ GoodData.logger.error "Unable to remove master workspace: '#{master_project_id}', Error: #{ex.message}"
133
+ end
134
+ end
135
+ removal_master_project_ids
136
+ end
137
+
138
+ # rubocop:disable Metrics/ParameterLists
139
+ def remove_old_workspaces_from_release_table(params, domain_id, data_product_id, segment_id, master_projects, removal_master_project_ids)
140
+ unless removal_master_project_ids.empty?
141
+ if params.ads_client
142
+ GoodData::LCM2::Helpers.delete_master_project_from_ads(params.release_table_name, params.ads_client, segment_id, removal_master_project_ids)
143
+ else
144
+ data = master_projects.sort_by { |master| master[:version] }
145
+ GoodData::LCM2::Helpers.update_master_project_to_nfs(domain_id, data_product_id, segment_id, data)
146
+ end
147
+ end
148
+ end
149
+ # rubocop:enable Metrics/ParameterLists
101
150
  end
102
151
  end
103
152
  end
@@ -0,0 +1,64 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require_relative 'base_action'
9
+
10
+ module GoodData
11
+ module LCM2
12
+ class SynchronizeDataSetMapping < BaseAction
13
+ DESCRIPTION = 'Synchronize Dataset Mappings'
14
+
15
+ PARAMS = define_params(self) do
16
+ description 'Client Used for Connecting to GD'
17
+ param :gdc_gd_client, instance_of(Type::GdClientType), required: true
18
+
19
+ description 'Client used to connecting to development domain'
20
+ param :development_client, instance_of(Type::GdClientType), required: true
21
+
22
+ description 'Synchronization Info'
23
+ param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
24
+
25
+ description 'Logger'
26
+ param :gdc_logger, instance_of(Type::GdLogger), required: true
27
+ end
28
+
29
+ RESULT_HEADER = %i[from to count status]
30
+
31
+ class << self
32
+ def call(params)
33
+ results = []
34
+
35
+ client = params.gdc_gd_client
36
+ development_client = params.development_client
37
+
38
+ params.synchronize.peach do |info|
39
+ from_project = info.from
40
+ to_projects = info.to
41
+
42
+ from = development_client.projects(from_project) || fail("Invalid 'from' project specified - '#{from_project}'")
43
+ dataset_mapping = from.dataset_mapping
44
+ if dataset_mapping&.dig('datasetMappings', 'items').nil? || dataset_mapping['datasetMappings']['items'].empty?
45
+ params.gdc_logger.info "Project: '#{from.title}', PID: '#{from.pid}' has no model mapping, skip synchronizing model mapping."
46
+ else
47
+ to_projects.peach do |to|
48
+ pid = to[:pid]
49
+ to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
50
+
51
+ params.gdc_logger.info "Transferring model mapping, from project: '#{from.title}', PID: '#{from.pid}', to project: '#{to_project.title}', PID: '#{to_project.pid}'"
52
+ res = to_project.update_dataset_mapping(dataset_mapping)
53
+ res[:from] = from.pid
54
+ results << res
55
+ end
56
+ end
57
+ end
58
+ # Return results
59
+ results.flatten
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end
@@ -83,18 +83,29 @@ module GoodData
83
83
  segment_info[:from_blueprint] = blueprint
84
84
  maql_diff = nil
85
85
  previous_master = segment_info[:previous_master]
86
+ synchronize_ldm_mode = params[:synchronize_ldm].downcase
86
87
  diff_against_master = %w(diff_against_master_with_fallback diff_against_master)
87
- .include?(params[:synchronize_ldm].downcase)
88
- GoodData.logger.info "Synchronize LDM mode: '#{params[:synchronize_ldm].downcase}'"
89
- if previous_master && diff_against_master
90
- maql_diff_params = [:includeGrain]
91
- maql_diff_params << :excludeFactRule if exclude_fact_rule
92
- maql_diff_params << :includeDeprecated if include_deprecated
93
- maql_diff = previous_master.maql_diff(blueprint: blueprint, params: maql_diff_params)
88
+ .include?(synchronize_ldm_mode)
89
+ GoodData.logger.info "Synchronize LDM mode: '#{synchronize_ldm_mode}'"
90
+ if segment_info.key?(:previous_master) && diff_against_master
91
+ if previous_master
92
+ maql_diff_params = [:includeGrain]
93
+ maql_diff_params << :excludeFactRule if exclude_fact_rule
94
+ maql_diff_params << :includeDeprecated if include_deprecated
95
+ maql_diff = previous_master.maql_diff(blueprint: blueprint, params: maql_diff_params)
96
+ else
97
+ maql_diff = {
98
+ "projectModelDiff" =>
99
+ {
100
+ "updateOperations" => [],
101
+ "updateScripts" => []
102
+ }
103
+ }
104
+ end
94
105
  chunks = maql_diff['projectModelDiff']['updateScripts']
95
106
  if chunks.empty?
96
107
  GoodData.logger.info "Synchronize LDM to clients will not proceed in mode \
97
- '#{params[:synchronize_ldm].downcase}' due to no LDM changes in the new master project. \
108
+ '#{synchronize_ldm_mode}' due to no LDM changes in the segment master project. \
98
109
  If you had changed LDM of clients manually, please use mode 'diff_against_clients' \
99
110
  to force synchronize LDM to clients"
100
111
  end
@@ -124,6 +124,7 @@ module GoodData
124
124
  GoodData.gd_logger.info("Synchronizing in mode=#{mode}, number_of_clients=#{all_clients.size}, data_rows=#{user_filters.size}")
125
125
 
126
126
  GoodData.logger.info("Synchronizing in mode \"#{mode}\"")
127
+ results = []
127
128
  case mode
128
129
  when 'sync_project', 'sync_one_project_based_on_pid', 'sync_one_project_based_on_custom_id'
129
130
  if mode == 'sync_one_project_based_on_pid'
@@ -134,7 +135,9 @@ module GoodData
134
135
  user_filters = user_filters.select { |f| f[:pid] == filter } if filter
135
136
 
136
137
  GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{user_filters.size}")
137
- sync_user_filters(project, user_filters, run_params, symbolized_config)
138
+ current_results = sync_user_filters(project, user_filters, run_params, symbolized_config)
139
+
140
+ results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
138
141
  when 'sync_multiple_projects_based_on_pid', 'sync_multiple_projects_based_on_custom_id'
139
142
  users_by_project = run_params[:users_brick_input].group_by { |u| u[:pid] }
140
143
  user_filters.group_by { |u| u[:pid] }.flat_map.pmap do |id, new_filters|
@@ -149,7 +152,9 @@ module GoodData
149
152
  end
150
153
 
151
154
  GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{id}, data_rows=#{new_filters.size}")
152
- sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
155
+ current_results = sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
156
+
157
+ results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
153
158
  end
154
159
  when 'sync_domain_client_workspaces'
155
160
  domain_clients = all_clients
@@ -161,12 +166,15 @@ module GoodData
161
166
  working_client_ids = []
162
167
 
163
168
  users_by_project = run_params[:users_brick_input].group_by { |u| u[:pid] }
164
- results = []
165
169
  user_filters.group_by { |u| u[multiple_projects_column] }.flat_map.pmap do |client_id, new_filters|
166
170
  users = users_by_project[client_id]
167
171
  fail "Client id cannot be empty" if client_id.blank?
168
172
 
169
173
  c = all_clients.detect { |specific_client| specific_client.id == client_id }
174
+ if c.nil?
175
+ params.gdc_logger.warn "Client #{client_id} is not found"
176
+ next
177
+ end
170
178
  if params.segments && !segment_uris.include?(c.segment_uri)
171
179
  params.gdc_logger.warn "Client #{client_id} is outside segments_filter #{params.segments}"
172
180
  next
@@ -178,7 +186,7 @@ module GoodData
178
186
 
179
187
  GoodData.gd_logger.info("Synchronizing in mode=#{mode}, client_id=#{client_id}, data_rows=#{new_filters.size}")
180
188
  partial_results = sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
181
- results.concat(partial_results[:results])
189
+ results.concat(partial_results[:results]) unless partial_results.nil? || partial_results[:results].empty?
182
190
  end
183
191
 
184
192
  unless run_params[:do_not_touch_filters_that_are_not_mentioned]
@@ -193,17 +201,16 @@ module GoodData
193
201
  GoodData.gd_logger.info("Delete all filters in project_id=#{current_project.pid}, client_id=#{c.client_id}")
194
202
  current_results = sync_user_filters(current_project, [], run_params.merge(users_brick_input: users), symbolized_config)
195
203
 
196
- results.concat(current_results[:results])
204
+ results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
197
205
  rescue StandardError => e
198
206
  params.gdc_logger.error "Failed to clear filters of #{c.client_id} due to: #{e.inspect}"
199
207
  end
200
208
  end
201
209
  end
202
-
203
- {
204
- results: results
205
- }
206
210
  end
211
+ {
212
+ results: results
213
+ }
207
214
  end
208
215
 
209
216
  def sync_user_filters(project, filters, params, filters_config)
@@ -182,11 +182,11 @@ module GoodData
182
182
  # value of a project id in the data since he does not know it upfront
183
183
  # and we cannot influence its value.
184
184
  common_params = {
185
- domain: domain,
186
- whitelists: whitelists,
187
- ignore_failures: ignore_failures,
188
- remove_users_from_project: remove_users_from_project,
189
- do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
185
+ domain: domain,
186
+ whitelists: whitelists,
187
+ ignore_failures: ignore_failures,
188
+ remove_users_from_project: remove_users_from_project,
189
+ do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
190
190
  create_non_existing_user_groups: create_non_existing_user_groups,
191
191
  user_groups_cache: nil
192
192
  }
@@ -198,7 +198,8 @@ module GoodData
198
198
  domain.create_users(new_users.uniq { |u| u[:login] || u[:email] })
199
199
  when 'remove_from_organization'
200
200
  user_ids = new_users.uniq { |u| u[:login] || u[:email] }.map { |u| u[:login] || u[:email] }
201
- users = user_ids.map { |u| domain.users(u, client: client) }
201
+ users = user_ids.map { |u| domain.users(u, client: client) }.reject(&:nil?)
202
+ params.gdc_logger.info "#{user_ids.count - users.count} users were not found (or were deleted) in domain #{domain_name}" if user_ids.count > users.count
202
203
  params.gdc_logger.warn "Deleting #{users.count} users from domain #{domain_name}"
203
204
 
204
205
  GoodData.gd_logger.info("Synchronizing in mode=#{mode}, domain=#{domain_name}, data_rows=#{users.count}")
@@ -0,0 +1,185 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ # Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
4
+ # This source code is licensed under the BSD-style license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ require_relative 'base_action'
8
+
9
+ module GoodData
10
+ module LCM2
11
+ class UpdateMetricFormats < BaseAction
12
+ DESCRIPTION = 'Localize Metric Formats'
13
+
14
+ PARAMS = define_params(self) do
15
+ description 'Synchronization Info'
16
+ param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
17
+
18
+ description 'Client Used for Connecting to GD'
19
+ param :gdc_gd_client, instance_of(Type::GdClientType), required: true
20
+
21
+ description 'Organization Name'
22
+ param :organization, instance_of(Type::StringType), required: false
23
+
24
+ description 'DataProduct to manage'
25
+ param :data_product, instance_of(Type::GDDataProductType), required: false
26
+
27
+ description 'Logger'
28
+ param :gdc_logger, instance_of(Type::GdLogger), required: true
29
+
30
+ description 'ADS Client'
31
+ param :ads_client, instance_of(Type::AdsClientType), required: false
32
+
33
+ description 'Input Source'
34
+ param :input_source, instance_of(Type::HashType), required: false
35
+
36
+ description 'Localization query'
37
+ param :localization_query, instance_of(Type::StringType), required: false
38
+ end
39
+
40
+ RESULT_HEADER = %i[action ok_clients error_clients]
41
+
42
+ class << self
43
+ def load_metric_data(params)
44
+ if params&.dig(:input_source, :metric_format) && params[:input_source][:metric_format].present?
45
+ metric_input_source = validate_input_source(params[:input_source])
46
+ else
47
+ return nil
48
+ end
49
+
50
+ metric_data_source = GoodData::Helpers::DataSource.new(metric_input_source)
51
+ begin
52
+ temp_csv = without_check(PARAMS, params) do
53
+ File.open(metric_data_source.realize(params), 'r:UTF-8')
54
+ end
55
+ rescue StandardError => e
56
+ GoodData.logger.warn("Unable to get metric input source, skip updating metric formats. Error: #{e.message} - #{e}")
57
+ return nil
58
+ end
59
+
60
+ metrics_hash = GoodData::Helpers::Csv.read_as_hash temp_csv
61
+ return nil if metrics_hash.empty?
62
+
63
+ expected_keys = %w[tag client_id format]
64
+ unless expected_keys.map(&:to_sym).all? { |s| metrics_hash.first.key? s }
65
+ GoodData.logger.warn("The input metric data is incorrect, expecting the following fields: #{expected_keys}")
66
+ return nil
67
+ end
68
+ metrics_hash
69
+ end
70
+
71
+ def validate_input_source(input_source)
72
+ type = input_source[:type] if input_source&.dig(:type)
73
+ metric_format = input_source[:metric_format]
74
+ raise "Incorrect configuration: 'type' of 'input_source' is required" if type.blank?
75
+
76
+ modified_input_source = input_source
77
+ case type
78
+ when 'ads', 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql'
79
+ if metric_format[:query].blank?
80
+ GoodData.logger.warn("The metric input_source '#{type}' is missing property 'query'")
81
+ return nil
82
+ end
83
+
84
+ modified_input_source[:query] = metric_format[:query]
85
+ return modified_input_source
86
+ when 's3'
87
+ if metric_format[:file].blank?
88
+ GoodData.logger.warn("The metric input_source '#{type}' is missing property 'file'")
89
+ return nil
90
+ end
91
+
92
+ if modified_input_source.key?(:key)
93
+ modified_input_source[:key] = metric_format[:file]
94
+ else
95
+ modified_input_source[:file] = metric_format[:file]
96
+ end
97
+ return modified_input_source
98
+ when 'blobStorage'
99
+ if metric_format[:file].blank?
100
+ GoodData.logger.warn("The metric input_source '#{type}' is missing property 'file'")
101
+ return nil
102
+ end
103
+
104
+ modified_input_source[:file] = metric_format[:file]
105
+ return modified_input_source
106
+ when 'staging'
107
+ if metric_format[:file].blank?
108
+ GoodData.logger.warn("The metric input_source '#{type}' is missing property 'file'")
109
+ return nil
110
+ end
111
+
112
+ modified_input_source[:path] = metric_format[:file]
113
+ return modified_input_source
114
+ when 'web'
115
+ if metric_format[:url].blank?
116
+ GoodData.logger.warn("The metric input_source '#{type}' is missing property 'url'")
117
+ return nil
118
+ end
119
+
120
+ modified_input_source[:url] = metric_format[:url]
121
+ return modified_input_source
122
+ else
123
+ return nil
124
+ end
125
+ end
126
+
127
+ def get_clients_metrics(metric_data)
128
+ return {} if metric_data.nil?
129
+
130
+ metric_groups = {}
131
+ clients = metric_data.map { |row| row[:client_id] }.uniq
132
+ clients.each do |client|
133
+ next if client.blank?
134
+
135
+ formats = {}
136
+ metric_data.select { |row| row[:client_id] == client && row[:tag].present? && row[:format].present? }.each { |row| formats[row[:tag]] = row[:format] }
137
+ metric_groups[client.to_s] ||= formats
138
+ end
139
+ metric_groups
140
+ end
141
+
142
+ def call(params)
143
+ data = load_metric_data(params)
144
+ result = []
145
+ return result if data.nil?
146
+
147
+ metric_group = get_clients_metrics(data)
148
+ return result if metric_group.empty?
149
+
150
+ GoodData.logger.debug("Clients have metrics which will be modified: #{metric_group.keys}")
151
+ updated_clients = params.synchronize.map { |segment| segment.to.map { |client| client[:client_id] } }.flatten.uniq
152
+ GoodData.logger.debug("Updating clients: #{updated_clients}")
153
+ data_product = params.data_product
154
+ data_product_clients = data_product.clients
155
+ number_client_ok = 0
156
+ number_client_error = 0
157
+ metric_group.each do |client_id, formats|
158
+ next unless updated_clients.include?(client_id)
159
+
160
+ client = data_product_clients.find { |c| c.id == client_id }
161
+ begin
162
+ GoodData.logger.info("Start updating metric format for client: '#{client_id}'")
163
+ metrics = client.project.metrics.to_a
164
+ formats.each do |tag, format|
165
+ next if tag.blank? || format.blank?
166
+
167
+ metrics_to_be_updated = metrics.select { |metric| metric.tags.include?(tag) }
168
+ metrics_to_be_updated.each do |metric|
169
+ metric.format = format
170
+ metric.save
171
+ end
172
+ end
173
+ number_client_ok += 1
174
+ GoodData.logger.info("Finished updating metric format for client: '#{client_id}'")
175
+ rescue StandardError => e
176
+ number_client_error += 1
177
+ GoodData.logger.warn("Failed to update metric format for client: '#{client_id}'. Error: #{e.message} - #{e}")
178
+ end
179
+ end
180
+ [{ :action => 'Update metric format', :ok_clients => number_client_ok, :error_clients => number_client_error }]
181
+ end
182
+ end
183
+ end
184
+ end
185
+ end
@@ -0,0 +1,5 @@
1
+ DELETE FROM "<%= table_name || 'LCM_RELEASE' %>"
2
+ WHERE
3
+ segment_id = '<%= segment_id %>'
4
+ AND master_project_id IN (<%= master_project_ids %>)
5
+ ;