gooddata 2.1.14 → 2.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gdc-ii-config.yaml +1 -1
- data/.github/workflows/build.yml +66 -0
- data/.github/workflows/pre-merge.yml +72 -0
- data/.sonar.settings +4 -0
- data/.travis.yml +78 -12
- data/CHANGELOG.md +62 -0
- data/Dockerfile +25 -14
- data/LICENSE +4418 -17
- data/LICENSE.rb +1 -1
- data/README.md +3 -3
- data/Rakefile +8 -1
- data/SDK_VERSION +1 -1
- data/VERSION +1 -1
- data/bin/test_projects_cleanup.rb +45 -3
- data/ci/mssql/pom.xml +62 -0
- data/ci/mysql/pom.xml +57 -0
- data/ci/postgresql/pom.xml +57 -0
- data/ci/redshift/pom.xml +1 -1
- data/dev-gooddata-sso.pub.encrypted +40 -40
- data/gdc_fossa_ruby_sdk.yaml +1 -0
- data/gooddata.gemspec +10 -6
- data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
- data/k8s/charts/lcm-bricks/templates/prometheus/alertingRules.yaml +11 -1
- data/lcm.rake +2 -2
- data/lib/gooddata/bricks/middleware/aws_middleware.rb +35 -9
- data/lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb +98 -0
- data/lib/gooddata/cloud_resources/{cloud_resouce_factory.rb → cloud_resource_factory.rb} +8 -0
- data/lib/gooddata/cloud_resources/cloud_resources.rb +1 -1
- data/lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mssql/mssql_client.rb +122 -0
- data/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +111 -0
- data/lib/gooddata/cloud_resources/postgresql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +106 -0
- data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +18 -1
- data/lib/gooddata/commands/scaffold.rb +9 -10
- data/lib/gooddata/core/nil_logger.rb +3 -1
- data/lib/gooddata/helpers/data_helper.rb +9 -5
- data/lib/gooddata/helpers/global_helpers.rb +6 -5
- data/lib/gooddata/lcm/actions/associate_clients.rb +8 -2
- data/lib/gooddata/lcm/actions/base_action.rb +0 -2
- data/lib/gooddata/lcm/actions/collect_meta.rb +3 -1
- data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +3 -2
- data/lib/gooddata/lcm/actions/provision_clients.rb +31 -10
- data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -7
- data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +64 -0
- data/lib/gooddata/lcm/actions/synchronize_ldm.rb +19 -8
- data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +16 -9
- data/lib/gooddata/lcm/actions/synchronize_users.rb +7 -6
- data/lib/gooddata/lcm/actions/update_metric_formats.rb +185 -0
- data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb +5 -0
- data/lib/gooddata/lcm/helpers/release_table_helper.rb +42 -8
- data/lib/gooddata/lcm/lcm2.rb +5 -2
- data/lib/gooddata/lcm/types/base_type.rb +0 -2
- data/lib/gooddata/mixins/md_object_query.rb +9 -6
- data/lib/gooddata/models/blueprint/project_blueprint.rb +0 -2
- data/lib/gooddata/models/client.rb +14 -12
- data/lib/gooddata/models/data_source.rb +668 -0
- data/lib/gooddata/models/dataset_mapping.rb +36 -0
- data/lib/gooddata/models/domain.rb +3 -2
- data/lib/gooddata/models/metadata/analytical_dashboard.rb +49 -0
- data/lib/gooddata/models/metadata/analytical_visualization_object.rb +30 -0
- data/lib/gooddata/models/metadata/label.rb +26 -27
- data/lib/gooddata/models/metadata/visualization_object.rb +50 -0
- data/lib/gooddata/models/project.rb +66 -19
- data/lib/gooddata/models/schedule.rb +13 -1
- data/lib/gooddata/models/user_filters/user_filter_builder.rb +58 -54
- data/lib/gooddata/models/user_group.rb +0 -1
- data/lib/gooddata/rest/connection.rb +6 -4
- data/lib/gooddata/rest/phmap.rb +2 -1
- data/lib/gooddata.rb +2 -0
- data/rubydev_public.gpg.encrypted +51 -51
- data/rubydev_secret_keys.gpg.encrypted +109 -109
- metadata +52 -27
- data/DEPENDENCIES.md +0 -880
- data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb +0 -37
- data/lib/gooddata/helpers/data_source_helpers.rb +0 -47
@@ -43,7 +43,7 @@ module GoodData
|
|
43
43
|
RESULT_HEADER = [
|
44
44
|
:id,
|
45
45
|
:status,
|
46
|
-
:
|
46
|
+
:project,
|
47
47
|
:client,
|
48
48
|
:type
|
49
49
|
]
|
@@ -102,7 +102,13 @@ module GoodData
|
|
102
102
|
options = { delete_projects: delete_projects }
|
103
103
|
options.merge!(delete_extra_option(params, delete_extra)) if delete_extra
|
104
104
|
|
105
|
-
domain.update_clients(params.clients, options)
|
105
|
+
results = domain.update_clients(params.clients, options)
|
106
|
+
# Update status to CREATED if the client has no project
|
107
|
+
results&.each do |r|
|
108
|
+
r[:status] = 'CREATED' if r[:originalProject].nil?
|
109
|
+
r[:project] = r[:originalProject]
|
110
|
+
end
|
111
|
+
results
|
106
112
|
end
|
107
113
|
|
108
114
|
private
|
@@ -8,8 +8,6 @@ require 'gooddata/extensions/integer'
|
|
8
8
|
require 'gooddata/extensions/string'
|
9
9
|
require 'gooddata/extensions/nil'
|
10
10
|
|
11
|
-
require 'active_support/core_ext/hash/compact'
|
12
|
-
|
13
11
|
require_relative '../dsl/dsl'
|
14
12
|
require_relative '../helpers/helpers'
|
15
13
|
require_relative '../types/types'
|
@@ -53,7 +53,9 @@ module GoodData
|
|
53
53
|
client: development_client
|
54
54
|
)
|
55
55
|
kpi_dashboards = MdObject.query('analyticalDashboard', MdObject, client: development_client, project: from_project)
|
56
|
-
|
56
|
+
kpi_dashboard_plugin = MdObject.query('dashboardPlugin', MdObject, client: development_client, project: from_project)
|
57
|
+
kpi_date_filter_config = MdObject.query('dateFilterConfig', MdObject, client: development_client, project: from_project)
|
58
|
+
objects = old_dashboards.to_a + kpi_dashboards.to_a + kpi_dashboard_plugin.to_a + kpi_date_filter_config.to_a
|
57
59
|
else
|
58
60
|
objects = GoodData::Dashboard.find_by_tag(
|
59
61
|
production_tags,
|
@@ -50,6 +50,7 @@ module GoodData
|
|
50
50
|
segment_info[:to].pmap do |entry|
|
51
51
|
pid = entry[:pid]
|
52
52
|
to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
|
53
|
+
GoodData.logger.info "Migrating date dimension, project: '#{to_project.title}', PID: #{pid}"
|
53
54
|
to_blueprint = to_project.blueprint
|
54
55
|
upgrade_datasets = get_upgrade_dates(latest_blueprint, to_blueprint)
|
55
56
|
next if upgrade_datasets.empty?
|
@@ -71,9 +72,9 @@ module GoodData
|
|
71
72
|
dest_dates = get_date_dimensions(dest_blueprint) if dest_blueprint
|
72
73
|
src_dates = get_date_dimensions(src_blueprint) if src_blueprint
|
73
74
|
|
74
|
-
return false if dest_dates.empty? || src_dates.empty?
|
75
|
-
|
76
75
|
upgrade_datasets = []
|
76
|
+
return upgrade_datasets if dest_dates.empty? || src_dates.empty?
|
77
|
+
|
77
78
|
dest_dates.each do |dest|
|
78
79
|
src_dim = get_date_dimension(src_blueprint, dest[:id])
|
79
80
|
next unless src_dim
|
@@ -48,7 +48,8 @@ module GoodData
|
|
48
48
|
domain_name = params.organization || params.domain
|
49
49
|
fail "Either organisation or domain has to be specified in params" unless domain_name
|
50
50
|
domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
|
51
|
-
|
51
|
+
error_message = nil
|
52
|
+
invalid_client_ids = []
|
52
53
|
begin
|
53
54
|
results = params.segments.map do |segment|
|
54
55
|
segment_object = domain.segments(segment.segment_id, data_product)
|
@@ -57,33 +58,53 @@ module GoodData
|
|
57
58
|
end
|
58
59
|
|
59
60
|
unless tmp.empty?
|
60
|
-
|
61
|
+
synchronize_project = {
|
61
62
|
segment_id: segment.segment_id,
|
62
63
|
from: segment.development_pid,
|
63
64
|
to: tmp.map do |entry|
|
64
65
|
unless entry[:project_uri]
|
65
|
-
|
66
|
+
error_message = "There was error during provisioning clients: #{entry[:error]}" unless error_message
|
67
|
+
invalid_client_ids << entry[:id]
|
68
|
+
next
|
66
69
|
end
|
67
70
|
{
|
68
71
|
pid: entry[:project_uri].split('/').last,
|
69
72
|
client_id: entry[:id]
|
70
73
|
}
|
71
|
-
end
|
74
|
+
end.compact
|
72
75
|
}
|
76
|
+
|
77
|
+
synchronize_projects << synchronize_project unless synchronize_project[:to].empty?
|
78
|
+
end
|
79
|
+
|
80
|
+
if error_message
|
81
|
+
params.gdc_logger.debug "#{error_message}. Purge all invalid clients now ..."
|
82
|
+
deleted_client_ids = []
|
83
|
+
|
84
|
+
segment_object.clients.map do |segment_client|
|
85
|
+
project = segment_client.project
|
86
|
+
if (project.nil? || project.deleted?)
|
87
|
+
client_id = segment_client.client_id
|
88
|
+
if invalid_client_ids.include?(client_id)
|
89
|
+
segment_client.delete
|
90
|
+
deleted_client_ids << client_id
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
params.gdc_logger.debug "Deleted clients: #{deleted_client_ids.join(', ')}"
|
96
|
+
raise error_message unless error_message['TooManyProjectsCreatedException'] || error_message['Max number registered projects']
|
97
|
+
break tmp
|
73
98
|
end
|
74
99
|
|
75
100
|
tmp
|
76
101
|
end
|
77
102
|
rescue => e
|
78
|
-
params.gdc_logger.error "Problem occurs when provisioning clients.
|
79
|
-
res = LCM2.run_action PurgeClients, params
|
80
|
-
params.gdc_logger.debug "Purge clients result: #{res}"
|
81
|
-
deleted_client_ids = res[:results].select { |r| r[:status] == 'purged' }.map { |r| r[:client_id] }
|
82
|
-
params.gdc_logger.error "Deleted clients: #{deleted_client_ids.join(', ')}"
|
103
|
+
params.gdc_logger.error "Problem occurs when provisioning clients."
|
83
104
|
raise e
|
84
105
|
end
|
85
106
|
|
86
|
-
results.flatten!
|
107
|
+
results.flatten! if results
|
87
108
|
|
88
109
|
# Return results
|
89
110
|
{
|
@@ -33,6 +33,9 @@ module GoodData
|
|
33
33
|
description 'ADS Client'
|
34
34
|
param :ads_client, instance_of(Type::AdsClientType), required: false
|
35
35
|
|
36
|
+
description 'Keep number of old master workspace excluding the latest one'
|
37
|
+
param :keep_only_previous_masters_count, instance_of(Type::StringType), required: false, default: '-1'
|
38
|
+
|
36
39
|
description 'Additional Hidden Parameters'
|
37
40
|
param :additional_hidden_params, instance_of(Type::HashType), required: false
|
38
41
|
end
|
@@ -53,6 +56,7 @@ module GoodData
|
|
53
56
|
domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
|
54
57
|
data_product = params.data_product
|
55
58
|
domain_segments = domain.segments(:all, data_product)
|
59
|
+
keep_only_previous_masters_count = Integer(params.keep_only_previous_masters_count || "-1")
|
56
60
|
|
57
61
|
segments = params.segments.map do |seg|
|
58
62
|
domain_segments.find do |s|
|
@@ -62,18 +66,14 @@ module GoodData
|
|
62
66
|
|
63
67
|
results = segments.map do |segment|
|
64
68
|
if params.ads_client
|
65
|
-
|
66
|
-
params.release_table_name,
|
67
|
-
params.ads_client,
|
68
|
-
segment.segment_id
|
69
|
-
)
|
69
|
+
master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_ads(params.release_table_name, params.ads_client, segment.segment_id)
|
70
70
|
else
|
71
|
-
|
71
|
+
master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
|
72
72
|
end
|
73
73
|
|
74
|
+
current_master = master_projects.last
|
74
75
|
# TODO: Check res.first.nil? || res.first[:master_project_id].nil?
|
75
76
|
master = client.projects(current_master[:master_project_id])
|
76
|
-
|
77
77
|
segment.master_project = master
|
78
78
|
segment.save
|
79
79
|
|
@@ -87,6 +87,19 @@ module GoodData
|
|
87
87
|
"Details: #{sync_result['links']['details']}")
|
88
88
|
end
|
89
89
|
|
90
|
+
if keep_only_previous_masters_count >= 0
|
91
|
+
number_of_deleted_projects = master_projects.count - (keep_only_previous_masters_count + 1)
|
92
|
+
|
93
|
+
if number_of_deleted_projects.positive?
|
94
|
+
begin
|
95
|
+
removal_master_project_ids = remove_multiple_workspace(params, segment.segment_id, master_projects, number_of_deleted_projects)
|
96
|
+
remove_old_workspaces_from_release_table(params, domain_name, data_product.data_product_id, segment.segment_id, master_projects, removal_master_project_ids)
|
97
|
+
rescue Exception => e # rubocop:disable RescueException
|
98
|
+
GoodData.logger.error "Problem occurs when removing old master workspace, reason: #{e.message}"
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
90
103
|
{
|
91
104
|
segment: segment.id,
|
92
105
|
master_pid: master.pid,
|
@@ -98,6 +111,42 @@ module GoodData
|
|
98
111
|
# Return results
|
99
112
|
results
|
100
113
|
end
|
114
|
+
|
115
|
+
def remove_multiple_workspace(params, segment_id, master_projects, number_of_deleted_projects)
|
116
|
+
removal_master_project_ids = []
|
117
|
+
need_to_delete_projects = master_projects.take(number_of_deleted_projects)
|
118
|
+
|
119
|
+
need_to_delete_projects.each do |project_wrapper|
|
120
|
+
master_project_id = project_wrapper[:master_project_id]
|
121
|
+
next if master_project_id.to_s.empty?
|
122
|
+
|
123
|
+
begin
|
124
|
+
project = params.gdc_gd_client.projects(master_project_id)
|
125
|
+
if project && !%w[deleted archived].include?(project.state.to_s)
|
126
|
+
GoodData.logger.info "Segment #{segment_id}: Deleting old master workspace, project: '#{project.title}', PID: (#{project.pid})."
|
127
|
+
project.delete
|
128
|
+
end
|
129
|
+
removal_master_project_ids << master_project_id
|
130
|
+
master_projects.delete_if { |p| p[:master_project_id] == master_project_id }
|
131
|
+
rescue Exception => ex # rubocop:disable RescueException
|
132
|
+
GoodData.logger.error "Unable to remove master workspace: '#{master_project_id}', Error: #{ex.message}"
|
133
|
+
end
|
134
|
+
end
|
135
|
+
removal_master_project_ids
|
136
|
+
end
|
137
|
+
|
138
|
+
# rubocop:disable Metrics/ParameterLists
|
139
|
+
def remove_old_workspaces_from_release_table(params, domain_id, data_product_id, segment_id, master_projects, removal_master_project_ids)
|
140
|
+
unless removal_master_project_ids.empty?
|
141
|
+
if params.ads_client
|
142
|
+
GoodData::LCM2::Helpers.delete_master_project_from_ads(params.release_table_name, params.ads_client, segment_id, removal_master_project_ids)
|
143
|
+
else
|
144
|
+
data = master_projects.sort_by { |master| master[:version] }
|
145
|
+
GoodData::LCM2::Helpers.update_master_project_to_nfs(domain_id, data_product_id, segment_id, data)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
# rubocop:enable Metrics/ParameterLists
|
101
150
|
end
|
102
151
|
end
|
103
152
|
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require_relative 'base_action'
|
9
|
+
|
10
|
+
module GoodData
|
11
|
+
module LCM2
|
12
|
+
class SynchronizeDataSetMapping < BaseAction
|
13
|
+
DESCRIPTION = 'Synchronize Dataset Mappings'
|
14
|
+
|
15
|
+
PARAMS = define_params(self) do
|
16
|
+
description 'Client Used for Connecting to GD'
|
17
|
+
param :gdc_gd_client, instance_of(Type::GdClientType), required: true
|
18
|
+
|
19
|
+
description 'Client used to connecting to development domain'
|
20
|
+
param :development_client, instance_of(Type::GdClientType), required: true
|
21
|
+
|
22
|
+
description 'Synchronization Info'
|
23
|
+
param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
|
24
|
+
|
25
|
+
description 'Logger'
|
26
|
+
param :gdc_logger, instance_of(Type::GdLogger), required: true
|
27
|
+
end
|
28
|
+
|
29
|
+
RESULT_HEADER = %i[from to count status]
|
30
|
+
|
31
|
+
class << self
|
32
|
+
def call(params)
|
33
|
+
results = []
|
34
|
+
|
35
|
+
client = params.gdc_gd_client
|
36
|
+
development_client = params.development_client
|
37
|
+
|
38
|
+
params.synchronize.peach do |info|
|
39
|
+
from_project = info.from
|
40
|
+
to_projects = info.to
|
41
|
+
|
42
|
+
from = development_client.projects(from_project) || fail("Invalid 'from' project specified - '#{from_project}'")
|
43
|
+
dataset_mapping = from.dataset_mapping
|
44
|
+
if dataset_mapping&.dig('datasetMappings', 'items').nil? || dataset_mapping['datasetMappings']['items'].empty?
|
45
|
+
params.gdc_logger.info "Project: '#{from.title}', PID: '#{from.pid}' has no model mapping, skip synchronizing model mapping."
|
46
|
+
else
|
47
|
+
to_projects.peach do |to|
|
48
|
+
pid = to[:pid]
|
49
|
+
to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
|
50
|
+
|
51
|
+
params.gdc_logger.info "Transferring model mapping, from project: '#{from.title}', PID: '#{from.pid}', to project: '#{to_project.title}', PID: '#{to_project.pid}'"
|
52
|
+
res = to_project.update_dataset_mapping(dataset_mapping)
|
53
|
+
res[:from] = from.pid
|
54
|
+
results << res
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
# Return results
|
59
|
+
results.flatten
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
@@ -83,18 +83,29 @@ module GoodData
|
|
83
83
|
segment_info[:from_blueprint] = blueprint
|
84
84
|
maql_diff = nil
|
85
85
|
previous_master = segment_info[:previous_master]
|
86
|
+
synchronize_ldm_mode = params[:synchronize_ldm].downcase
|
86
87
|
diff_against_master = %w(diff_against_master_with_fallback diff_against_master)
|
87
|
-
.include?(
|
88
|
-
GoodData.logger.info "Synchronize LDM mode: '#{
|
89
|
-
if previous_master && diff_against_master
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
88
|
+
.include?(synchronize_ldm_mode)
|
89
|
+
GoodData.logger.info "Synchronize LDM mode: '#{synchronize_ldm_mode}'"
|
90
|
+
if segment_info.key?(:previous_master) && diff_against_master
|
91
|
+
if previous_master
|
92
|
+
maql_diff_params = [:includeGrain]
|
93
|
+
maql_diff_params << :excludeFactRule if exclude_fact_rule
|
94
|
+
maql_diff_params << :includeDeprecated if include_deprecated
|
95
|
+
maql_diff = previous_master.maql_diff(blueprint: blueprint, params: maql_diff_params)
|
96
|
+
else
|
97
|
+
maql_diff = {
|
98
|
+
"projectModelDiff" =>
|
99
|
+
{
|
100
|
+
"updateOperations" => [],
|
101
|
+
"updateScripts" => []
|
102
|
+
}
|
103
|
+
}
|
104
|
+
end
|
94
105
|
chunks = maql_diff['projectModelDiff']['updateScripts']
|
95
106
|
if chunks.empty?
|
96
107
|
GoodData.logger.info "Synchronize LDM to clients will not proceed in mode \
|
97
|
-
'#{
|
108
|
+
'#{synchronize_ldm_mode}' due to no LDM changes in the segment master project. \
|
98
109
|
If you had changed LDM of clients manually, please use mode 'diff_against_clients' \
|
99
110
|
to force synchronize LDM to clients"
|
100
111
|
end
|
@@ -124,6 +124,7 @@ module GoodData
|
|
124
124
|
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, number_of_clients=#{all_clients.size}, data_rows=#{user_filters.size}")
|
125
125
|
|
126
126
|
GoodData.logger.info("Synchronizing in mode \"#{mode}\"")
|
127
|
+
results = []
|
127
128
|
case mode
|
128
129
|
when 'sync_project', 'sync_one_project_based_on_pid', 'sync_one_project_based_on_custom_id'
|
129
130
|
if mode == 'sync_one_project_based_on_pid'
|
@@ -134,7 +135,9 @@ module GoodData
|
|
134
135
|
user_filters = user_filters.select { |f| f[:pid] == filter } if filter
|
135
136
|
|
136
137
|
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{user_filters.size}")
|
137
|
-
sync_user_filters(project, user_filters, run_params, symbolized_config)
|
138
|
+
current_results = sync_user_filters(project, user_filters, run_params, symbolized_config)
|
139
|
+
|
140
|
+
results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
|
138
141
|
when 'sync_multiple_projects_based_on_pid', 'sync_multiple_projects_based_on_custom_id'
|
139
142
|
users_by_project = run_params[:users_brick_input].group_by { |u| u[:pid] }
|
140
143
|
user_filters.group_by { |u| u[:pid] }.flat_map.pmap do |id, new_filters|
|
@@ -149,7 +152,9 @@ module GoodData
|
|
149
152
|
end
|
150
153
|
|
151
154
|
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{id}, data_rows=#{new_filters.size}")
|
152
|
-
sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
|
155
|
+
current_results = sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
|
156
|
+
|
157
|
+
results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
|
153
158
|
end
|
154
159
|
when 'sync_domain_client_workspaces'
|
155
160
|
domain_clients = all_clients
|
@@ -161,12 +166,15 @@ module GoodData
|
|
161
166
|
working_client_ids = []
|
162
167
|
|
163
168
|
users_by_project = run_params[:users_brick_input].group_by { |u| u[:pid] }
|
164
|
-
results = []
|
165
169
|
user_filters.group_by { |u| u[multiple_projects_column] }.flat_map.pmap do |client_id, new_filters|
|
166
170
|
users = users_by_project[client_id]
|
167
171
|
fail "Client id cannot be empty" if client_id.blank?
|
168
172
|
|
169
173
|
c = all_clients.detect { |specific_client| specific_client.id == client_id }
|
174
|
+
if c.nil?
|
175
|
+
params.gdc_logger.warn "Client #{client_id} is not found"
|
176
|
+
next
|
177
|
+
end
|
170
178
|
if params.segments && !segment_uris.include?(c.segment_uri)
|
171
179
|
params.gdc_logger.warn "Client #{client_id} is outside segments_filter #{params.segments}"
|
172
180
|
next
|
@@ -178,7 +186,7 @@ module GoodData
|
|
178
186
|
|
179
187
|
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, client_id=#{client_id}, data_rows=#{new_filters.size}")
|
180
188
|
partial_results = sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
|
181
|
-
results.concat(partial_results[:results])
|
189
|
+
results.concat(partial_results[:results]) unless partial_results.nil? || partial_results[:results].empty?
|
182
190
|
end
|
183
191
|
|
184
192
|
unless run_params[:do_not_touch_filters_that_are_not_mentioned]
|
@@ -193,17 +201,16 @@ module GoodData
|
|
193
201
|
GoodData.gd_logger.info("Delete all filters in project_id=#{current_project.pid}, client_id=#{c.client_id}")
|
194
202
|
current_results = sync_user_filters(current_project, [], run_params.merge(users_brick_input: users), symbolized_config)
|
195
203
|
|
196
|
-
results.concat(current_results[:results])
|
204
|
+
results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
|
197
205
|
rescue StandardError => e
|
198
206
|
params.gdc_logger.error "Failed to clear filters of #{c.client_id} due to: #{e.inspect}"
|
199
207
|
end
|
200
208
|
end
|
201
209
|
end
|
202
|
-
|
203
|
-
{
|
204
|
-
results: results
|
205
|
-
}
|
206
210
|
end
|
211
|
+
{
|
212
|
+
results: results
|
213
|
+
}
|
207
214
|
end
|
208
215
|
|
209
216
|
def sync_user_filters(project, filters, params, filters_config)
|
@@ -182,11 +182,11 @@ module GoodData
|
|
182
182
|
# value of a project id in the data since he does not know it upfront
|
183
183
|
# and we cannot influence its value.
|
184
184
|
common_params = {
|
185
|
-
domain: domain,
|
186
|
-
whitelists: whitelists,
|
187
|
-
ignore_failures: ignore_failures,
|
188
|
-
remove_users_from_project: remove_users_from_project,
|
189
|
-
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
185
|
+
domain: domain,
|
186
|
+
whitelists: whitelists,
|
187
|
+
ignore_failures: ignore_failures,
|
188
|
+
remove_users_from_project: remove_users_from_project,
|
189
|
+
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
190
190
|
create_non_existing_user_groups: create_non_existing_user_groups,
|
191
191
|
user_groups_cache: nil
|
192
192
|
}
|
@@ -198,7 +198,8 @@ module GoodData
|
|
198
198
|
domain.create_users(new_users.uniq { |u| u[:login] || u[:email] })
|
199
199
|
when 'remove_from_organization'
|
200
200
|
user_ids = new_users.uniq { |u| u[:login] || u[:email] }.map { |u| u[:login] || u[:email] }
|
201
|
-
users = user_ids.map { |u| domain.users(u, client: client) }
|
201
|
+
users = user_ids.map { |u| domain.users(u, client: client) }.reject(&:nil?)
|
202
|
+
params.gdc_logger.info "#{user_ids.count - users.count} users were not found (or were deleted) in domain #{domain_name}" if user_ids.count > users.count
|
202
203
|
params.gdc_logger.warn "Deleting #{users.count} users from domain #{domain_name}"
|
203
204
|
|
204
205
|
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, domain=#{domain_name}, data_rows=#{users.count}")
|
@@ -0,0 +1,185 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
|
4
|
+
# This source code is licensed under the BSD-style license found in the
|
5
|
+
# LICENSE file in the root directory of this source tree.
|
6
|
+
|
7
|
+
require_relative 'base_action'
|
8
|
+
|
9
|
+
module GoodData
|
10
|
+
module LCM2
|
11
|
+
class UpdateMetricFormats < BaseAction
|
12
|
+
DESCRIPTION = 'Localize Metric Formats'
|
13
|
+
|
14
|
+
PARAMS = define_params(self) do
|
15
|
+
description 'Synchronization Info'
|
16
|
+
param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
|
17
|
+
|
18
|
+
description 'Client Used for Connecting to GD'
|
19
|
+
param :gdc_gd_client, instance_of(Type::GdClientType), required: true
|
20
|
+
|
21
|
+
description 'Organization Name'
|
22
|
+
param :organization, instance_of(Type::StringType), required: false
|
23
|
+
|
24
|
+
description 'DataProduct to manage'
|
25
|
+
param :data_product, instance_of(Type::GDDataProductType), required: false
|
26
|
+
|
27
|
+
description 'Logger'
|
28
|
+
param :gdc_logger, instance_of(Type::GdLogger), required: true
|
29
|
+
|
30
|
+
description 'ADS Client'
|
31
|
+
param :ads_client, instance_of(Type::AdsClientType), required: false
|
32
|
+
|
33
|
+
description 'Input Source'
|
34
|
+
param :input_source, instance_of(Type::HashType), required: false
|
35
|
+
|
36
|
+
description 'Localization query'
|
37
|
+
param :localization_query, instance_of(Type::StringType), required: false
|
38
|
+
end
|
39
|
+
|
40
|
+
RESULT_HEADER = %i[action ok_clients error_clients]
|
41
|
+
|
42
|
+
class << self
|
43
|
+
def load_metric_data(params)
|
44
|
+
if params&.dig(:input_source, :metric_format) && params[:input_source][:metric_format].present?
|
45
|
+
metric_input_source = validate_input_source(params[:input_source])
|
46
|
+
else
|
47
|
+
return nil
|
48
|
+
end
|
49
|
+
|
50
|
+
metric_data_source = GoodData::Helpers::DataSource.new(metric_input_source)
|
51
|
+
begin
|
52
|
+
temp_csv = without_check(PARAMS, params) do
|
53
|
+
File.open(metric_data_source.realize(params), 'r:UTF-8')
|
54
|
+
end
|
55
|
+
rescue StandardError => e
|
56
|
+
GoodData.logger.warn("Unable to get metric input source, skip updating metric formats. Error: #{e.message} - #{e}")
|
57
|
+
return nil
|
58
|
+
end
|
59
|
+
|
60
|
+
metrics_hash = GoodData::Helpers::Csv.read_as_hash temp_csv
|
61
|
+
return nil if metrics_hash.empty?
|
62
|
+
|
63
|
+
expected_keys = %w[tag client_id format]
|
64
|
+
unless expected_keys.map(&:to_sym).all? { |s| metrics_hash.first.key? s }
|
65
|
+
GoodData.logger.warn("The input metric data is incorrect, expecting the following fields: #{expected_keys}")
|
66
|
+
return nil
|
67
|
+
end
|
68
|
+
metrics_hash
|
69
|
+
end
|
70
|
+
|
71
|
+
def validate_input_source(input_source)
|
72
|
+
type = input_source[:type] if input_source&.dig(:type)
|
73
|
+
metric_format = input_source[:metric_format]
|
74
|
+
raise "Incorrect configuration: 'type' of 'input_source' is required" if type.blank?
|
75
|
+
|
76
|
+
modified_input_source = input_source
|
77
|
+
case type
|
78
|
+
when 'ads', 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql'
|
79
|
+
if metric_format[:query].blank?
|
80
|
+
GoodData.logger.warn("The metric input_source '#{type}' is missing property 'query'")
|
81
|
+
return nil
|
82
|
+
end
|
83
|
+
|
84
|
+
modified_input_source[:query] = metric_format[:query]
|
85
|
+
return modified_input_source
|
86
|
+
when 's3'
|
87
|
+
if metric_format[:file].blank?
|
88
|
+
GoodData.logger.warn("The metric input_source '#{type}' is missing property 'file'")
|
89
|
+
return nil
|
90
|
+
end
|
91
|
+
|
92
|
+
if modified_input_source.key?(:key)
|
93
|
+
modified_input_source[:key] = metric_format[:file]
|
94
|
+
else
|
95
|
+
modified_input_source[:file] = metric_format[:file]
|
96
|
+
end
|
97
|
+
return modified_input_source
|
98
|
+
when 'blobStorage'
|
99
|
+
if metric_format[:file].blank?
|
100
|
+
GoodData.logger.warn("The metric input_source '#{type}' is missing property 'file'")
|
101
|
+
return nil
|
102
|
+
end
|
103
|
+
|
104
|
+
modified_input_source[:file] = metric_format[:file]
|
105
|
+
return modified_input_source
|
106
|
+
when 'staging'
|
107
|
+
if metric_format[:file].blank?
|
108
|
+
GoodData.logger.warn("The metric input_source '#{type}' is missing property 'file'")
|
109
|
+
return nil
|
110
|
+
end
|
111
|
+
|
112
|
+
modified_input_source[:path] = metric_format[:file]
|
113
|
+
return modified_input_source
|
114
|
+
when 'web'
|
115
|
+
if metric_format[:url].blank?
|
116
|
+
GoodData.logger.warn("The metric input_source '#{type}' is missing property 'url'")
|
117
|
+
return nil
|
118
|
+
end
|
119
|
+
|
120
|
+
modified_input_source[:url] = metric_format[:url]
|
121
|
+
return modified_input_source
|
122
|
+
else
|
123
|
+
return nil
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def get_clients_metrics(metric_data)
|
128
|
+
return {} if metric_data.nil?
|
129
|
+
|
130
|
+
metric_groups = {}
|
131
|
+
clients = metric_data.map { |row| row[:client_id] }.uniq
|
132
|
+
clients.each do |client|
|
133
|
+
next if client.blank?
|
134
|
+
|
135
|
+
formats = {}
|
136
|
+
metric_data.select { |row| row[:client_id] == client && row[:tag].present? && row[:format].present? }.each { |row| formats[row[:tag]] = row[:format] }
|
137
|
+
metric_groups[client.to_s] ||= formats
|
138
|
+
end
|
139
|
+
metric_groups
|
140
|
+
end
|
141
|
+
|
142
|
+
def call(params)
|
143
|
+
data = load_metric_data(params)
|
144
|
+
result = []
|
145
|
+
return result if data.nil?
|
146
|
+
|
147
|
+
metric_group = get_clients_metrics(data)
|
148
|
+
return result if metric_group.empty?
|
149
|
+
|
150
|
+
GoodData.logger.debug("Clients have metrics which will be modified: #{metric_group.keys}")
|
151
|
+
updated_clients = params.synchronize.map { |segment| segment.to.map { |client| client[:client_id] } }.flatten.uniq
|
152
|
+
GoodData.logger.debug("Updating clients: #{updated_clients}")
|
153
|
+
data_product = params.data_product
|
154
|
+
data_product_clients = data_product.clients
|
155
|
+
number_client_ok = 0
|
156
|
+
number_client_error = 0
|
157
|
+
metric_group.each do |client_id, formats|
|
158
|
+
next unless updated_clients.include?(client_id)
|
159
|
+
|
160
|
+
client = data_product_clients.find { |c| c.id == client_id }
|
161
|
+
begin
|
162
|
+
GoodData.logger.info("Start updating metric format for client: '#{client_id}'")
|
163
|
+
metrics = client.project.metrics.to_a
|
164
|
+
formats.each do |tag, format|
|
165
|
+
next if tag.blank? || format.blank?
|
166
|
+
|
167
|
+
metrics_to_be_updated = metrics.select { |metric| metric.tags.include?(tag) }
|
168
|
+
metrics_to_be_updated.each do |metric|
|
169
|
+
metric.format = format
|
170
|
+
metric.save
|
171
|
+
end
|
172
|
+
end
|
173
|
+
number_client_ok += 1
|
174
|
+
GoodData.logger.info("Finished updating metric format for client: '#{client_id}'")
|
175
|
+
rescue StandardError => e
|
176
|
+
number_client_error += 1
|
177
|
+
GoodData.logger.warn("Failed to update metric format for client: '#{client_id}'. Error: #{e.message} - #{e}")
|
178
|
+
end
|
179
|
+
end
|
180
|
+
[{ :action => 'Update metric format', :ok_clients => number_client_ok, :error_clients => number_client_error }]
|
181
|
+
end
|
182
|
+
end
|
183
|
+
end
|
184
|
+
end
|
185
|
+
end
|