gooddata 2.2.0-java → 2.3.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gdc-ii-config.yaml +42 -1
- data/.github/workflows/build.yml +14 -13
- data/.github/workflows/pre-merge.yml +13 -13
- data/.pronto.yml +1 -0
- data/.rubocop.yml +2 -14
- data/CHANGELOG.md +9 -0
- data/Dockerfile +13 -7
- data/Dockerfile.jruby +5 -5
- data/Dockerfile.ruby +5 -7
- data/Gemfile +4 -2
- data/README.md +5 -4
- data/Rakefile +1 -1
- data/SDK_VERSION +1 -1
- data/VERSION +1 -1
- data/bin/run_brick.rb +7 -0
- data/ci/mysql/pom.xml +6 -1
- data/ci/redshift/pom.xml +3 -4
- data/docker-compose.lcm.yml +42 -1
- data/docker-compose.yml +42 -0
- data/gooddata.gemspec +21 -22
- data/lcm.rake +9 -0
- data/lib/gooddata/bricks/base_pipeline.rb +26 -0
- data/lib/gooddata/bricks/brick.rb +0 -1
- data/lib/gooddata/bricks/middleware/execution_result_middleware.rb +3 -3
- data/lib/gooddata/bricks/pipeline.rb +2 -14
- data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +18 -8
- data/lib/gooddata/cloud_resources/redshift/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/redshift/redshift_client.rb +0 -2
- data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +1 -1
- data/lib/gooddata/lcm/actions/base_action.rb +157 -0
- data/lib/gooddata/lcm/actions/collect_data_product.rb +2 -1
- data/lib/gooddata/lcm/actions/collect_projects_warning_status.rb +53 -0
- data/lib/gooddata/lcm/actions/collect_segment_clients.rb +14 -0
- data/lib/gooddata/lcm/actions/initialize_continue_on_error_option.rb +87 -0
- data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +28 -2
- data/lib/gooddata/lcm/actions/provision_clients.rb +34 -5
- data/lib/gooddata/lcm/actions/synchronize_cas.rb +24 -4
- data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -4
- data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +28 -3
- data/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb +48 -11
- data/lib/gooddata/lcm/actions/synchronize_kd_dashboard_permission.rb +103 -0
- data/lib/gooddata/lcm/actions/synchronize_ldm.rb +60 -15
- data/lib/gooddata/lcm/actions/synchronize_ldm_layout.rb +98 -0
- data/lib/gooddata/lcm/actions/synchronize_pp_dashboard_permission.rb +108 -0
- data/lib/gooddata/lcm/actions/synchronize_schedules.rb +31 -1
- data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +14 -9
- data/lib/gooddata/lcm/actions/synchronize_user_groups.rb +30 -4
- data/lib/gooddata/lcm/actions/synchronize_users.rb +11 -10
- data/lib/gooddata/lcm/actions/update_metric_formats.rb +21 -4
- data/lib/gooddata/lcm/exceptions/lcm_execution_warning.rb +15 -0
- data/lib/gooddata/lcm/helpers/check_helper.rb +19 -0
- data/lib/gooddata/lcm/lcm2.rb +45 -4
- data/lib/gooddata/lcm/user_bricks_helper.rb +9 -0
- data/lib/gooddata/mixins/inspector.rb +1 -1
- data/lib/gooddata/models/ldm_layout.rb +38 -0
- data/lib/gooddata/models/project.rb +197 -22
- data/lib/gooddata/models/project_creator.rb +83 -6
- data/lib/gooddata/models/segment.rb +2 -1
- data/lib/gooddata/models/user_filters/user_filter_builder.rb +104 -15
- data/lib/gooddata/rest/connection.rb +5 -3
- data/lib/gooddata/rest/phmap.rb +1 -0
- data/lib/gooddata.rb +1 -0
- data/lib/gooddata_brick_base.rb +35 -0
- data/sonar-project.properties +6 -0
- metadata +60 -55
- data/lib/gooddata/cloud_resources/redshift/drivers/log4j.properties +0 -15
@@ -5,6 +5,7 @@
|
|
5
5
|
# LICENSE file in the root directory of this source tree.
|
6
6
|
|
7
7
|
require_relative 'base_action'
|
8
|
+
require_relative '../helpers/helpers'
|
8
9
|
|
9
10
|
using TrueExtensions
|
10
11
|
using FalseExtensions
|
@@ -47,6 +48,15 @@ module GoodData
|
|
47
48
|
|
48
49
|
description 'Enables handling of deprecated objects in the logical data model.'
|
49
50
|
param :include_deprecated, instance_of(Type::BooleanType), required: false, default: false
|
51
|
+
|
52
|
+
description 'Abort on error'
|
53
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
54
|
+
|
55
|
+
description 'Collect synced status'
|
56
|
+
param :collect_synced_status, instance_of(Type::BooleanType), required: false
|
57
|
+
|
58
|
+
description 'Sync failed list'
|
59
|
+
param :sync_failed_list, instance_of(Type::HashType), required: false
|
50
60
|
end
|
51
61
|
|
52
62
|
RESULT_HEADER = %i[from to status]
|
@@ -56,6 +66,8 @@ module GoodData
|
|
56
66
|
results = []
|
57
67
|
synchronize = []
|
58
68
|
params.synchronize.map do |segment_info|
|
69
|
+
next if sync_failed_segment(segment_info[:segment_id], params)
|
70
|
+
|
59
71
|
new_segment_info, segment_results = sync_segment_ldm(params, segment_info)
|
60
72
|
results.concat(segment_results)
|
61
73
|
synchronize << new_segment_info
|
@@ -72,12 +84,27 @@ module GoodData
|
|
72
84
|
private
|
73
85
|
|
74
86
|
def sync_segment_ldm(params, segment_info)
|
75
|
-
|
87
|
+
collect_synced_status = collect_synced_status(params)
|
88
|
+
failed_projects = ThreadSafe::Array.new
|
89
|
+
results = ThreadSafe::Array.new
|
76
90
|
client = params.gdc_gd_client
|
77
91
|
exclude_fact_rule = params.exclude_fact_rule.to_b
|
78
92
|
include_deprecated = params.include_deprecated.to_b
|
93
|
+
update_preference = params[:update_preference]
|
94
|
+
exist_fallback_to_hard_sync_config = !update_preference.nil? && !update_preference[:fallback_to_hard_sync].nil?
|
95
|
+
include_maql_fallback_hard_sync = exist_fallback_to_hard_sync_config && Helpers.to_bool('fallback_to_hard_sync', update_preference[:fallback_to_hard_sync])
|
96
|
+
|
79
97
|
from_pid = segment_info[:from]
|
80
|
-
from = params.development_client.projects(from_pid)
|
98
|
+
from = params.development_client.projects(from_pid)
|
99
|
+
unless from
|
100
|
+
segment_id = segment_info[:segment_id]
|
101
|
+
error_message = "Failed to sync LDM for segment #{segment_id}. Error: Invalid 'from' project specified - '#{from_pid}'"
|
102
|
+
fail(error_message) unless collect_synced_status
|
103
|
+
|
104
|
+
add_failed_segment(segment_id, error_message, short_name, params)
|
105
|
+
return [segment_info, results]
|
106
|
+
end
|
107
|
+
|
81
108
|
GoodData.logger.info "Creating Blueprint, project: '#{from.title}', PID: #{from_pid}"
|
82
109
|
blueprint = from.blueprint(include_ca: params.include_computed_attributes.to_b)
|
83
110
|
segment_info[:from_blueprint] = blueprint
|
@@ -92,6 +119,8 @@ module GoodData
|
|
92
119
|
maql_diff_params = [:includeGrain]
|
93
120
|
maql_diff_params << :excludeFactRule if exclude_fact_rule
|
94
121
|
maql_diff_params << :includeDeprecated if include_deprecated
|
122
|
+
maql_diff_params << :includeMaqlFallbackHardSync if include_maql_fallback_hard_sync
|
123
|
+
|
95
124
|
maql_diff = previous_master.maql_diff(blueprint: blueprint, params: maql_diff_params)
|
96
125
|
else
|
97
126
|
maql_diff = {
|
@@ -102,6 +131,7 @@ module GoodData
|
|
102
131
|
}
|
103
132
|
}
|
104
133
|
end
|
134
|
+
|
105
135
|
chunks = maql_diff['projectModelDiff']['updateScripts']
|
106
136
|
if chunks.empty?
|
107
137
|
GoodData.logger.info "Synchronize LDM to clients will not proceed in mode \
|
@@ -112,40 +142,55 @@ to force synchronize LDM to clients"
|
|
112
142
|
end
|
113
143
|
|
114
144
|
segment_info[:to] = segment_info[:to].pmap do |entry|
|
145
|
+
update_status = true
|
115
146
|
pid = entry[:pid]
|
116
|
-
|
147
|
+
next if sync_failed_project(pid, params)
|
148
|
+
|
149
|
+
to_project = client.projects(pid)
|
150
|
+
unless to_project
|
151
|
+
process_failed_project(pid, "Invalid 'to' project specified - '#{pid}'", failed_projects, collect_synced_status)
|
152
|
+
next
|
153
|
+
end
|
117
154
|
|
118
155
|
GoodData.logger.info "Updating from Blueprint, project: '#{to_project.title}', PID: #{pid}"
|
119
156
|
begin
|
120
157
|
entry[:ca_scripts] = to_project.update_from_blueprint(
|
121
158
|
blueprint,
|
122
|
-
update_preference:
|
159
|
+
update_preference: update_preference,
|
123
160
|
exclude_fact_rule: exclude_fact_rule,
|
124
161
|
execute_ca_scripts: false,
|
125
162
|
maql_diff: maql_diff,
|
126
163
|
include_deprecated: include_deprecated
|
127
164
|
)
|
128
165
|
rescue MaqlExecutionError => e
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
166
|
+
if collect_synced_status
|
167
|
+
update_status = false
|
168
|
+
failed_message = "Applying MAQL to project #{to_project.title} - #{pid} failed. Reason: #{e}"
|
169
|
+
process_failed_project(pid, failed_message, failed_projects, collect_synced_status)
|
170
|
+
else
|
171
|
+
fail e unless previous_master && params[:synchronize_ldm] == 'diff_against_master_with_fallback'
|
172
|
+
|
173
|
+
GoodData.logger.info("Restoring the client project #{to_project.title} from master.")
|
174
|
+
entry[:ca_scripts] = to_project.update_from_blueprint(
|
175
|
+
blueprint,
|
176
|
+
update_preference: update_preference,
|
177
|
+
exclude_fact_rule: exclude_fact_rule,
|
178
|
+
execute_ca_scripts: false,
|
179
|
+
include_deprecated: include_deprecated
|
180
|
+
)
|
181
|
+
end
|
139
182
|
end
|
140
183
|
|
141
184
|
results << {
|
142
185
|
from: from_pid,
|
143
186
|
to: pid,
|
144
187
|
status: 'ok'
|
145
|
-
}
|
188
|
+
} if update_status
|
189
|
+
|
146
190
|
entry
|
147
191
|
end
|
148
192
|
|
193
|
+
process_failed_projects(failed_projects, short_name, params) if collect_synced_status
|
149
194
|
[segment_info, results]
|
150
195
|
end
|
151
196
|
end
|
@@ -0,0 +1,98 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2022 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require_relative 'base_action'
|
9
|
+
|
10
|
+
module GoodData
|
11
|
+
module LCM2
|
12
|
+
class SynchronizeLdmLayout < BaseAction
|
13
|
+
DESCRIPTION = 'Synchronize LDM Layout'
|
14
|
+
|
15
|
+
PARAMS = define_params(self) do
|
16
|
+
description 'Logger'
|
17
|
+
param :gdc_logger, instance_of(Type::GdLogger), required: true
|
18
|
+
|
19
|
+
description 'Client Used for Connecting to GD'
|
20
|
+
param :gdc_gd_client, instance_of(Type::GdClientType), required: true
|
21
|
+
|
22
|
+
description 'Client used to connecting to development domain'
|
23
|
+
param :development_client, instance_of(Type::GdClientType), required: true
|
24
|
+
|
25
|
+
description 'Synchronization Info'
|
26
|
+
param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
|
27
|
+
|
28
|
+
description 'Abort on error'
|
29
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
30
|
+
|
31
|
+
description 'Collect synced status'
|
32
|
+
param :collect_synced_status, instance_of(Type::BooleanType), required: false
|
33
|
+
|
34
|
+
description 'Sync failed list'
|
35
|
+
param :sync_failed_list, instance_of(Type::HashType), required: false
|
36
|
+
end
|
37
|
+
|
38
|
+
RESULT_HEADER = %i[from to status]
|
39
|
+
|
40
|
+
class << self
|
41
|
+
def call(params)
|
42
|
+
results = []
|
43
|
+
|
44
|
+
client = params.gdc_gd_client
|
45
|
+
development_client = params.development_client
|
46
|
+
gdc_logger = params.gdc_logger
|
47
|
+
collect_synced_status = collect_synced_status(params)
|
48
|
+
failed_projects = ThreadSafe::Array.new
|
49
|
+
|
50
|
+
params.synchronize.peach do |info|
|
51
|
+
from_project = info.from
|
52
|
+
to_projects = info.to
|
53
|
+
|
54
|
+
from = development_client.projects(from_project)
|
55
|
+
unless from
|
56
|
+
process_failed_project(from_project, "Invalid 'from' project specified - '#{from_project}'", failed_projects, collect_synced_status)
|
57
|
+
next
|
58
|
+
end
|
59
|
+
|
60
|
+
from_pid = from.pid
|
61
|
+
from_title = from.title
|
62
|
+
from_ldm_layout = from.ldm_layout
|
63
|
+
|
64
|
+
if from_ldm_layout&.dig('ldmLayout', 'layout').nil? || from_ldm_layout['ldmLayout']['layout'].empty?
|
65
|
+
gdc_logger.info "Project: '#{from_title}', PID: '#{from_pid}' has no ldm layout, skip synchronizing ldm layout."
|
66
|
+
else
|
67
|
+
to_projects.peach do |to|
|
68
|
+
pid = to[:pid]
|
69
|
+
to_project = client.projects(pid)
|
70
|
+
unless to_project
|
71
|
+
process_failed_project(pid, "Invalid 'from' project specified - '#{pid}'", failed_projects, collect_synced_status)
|
72
|
+
next
|
73
|
+
end
|
74
|
+
|
75
|
+
next if sync_failed_project(pid, params)
|
76
|
+
|
77
|
+
gdc_logger.info "Transferring ldm layout, from project: '#{from_title}', PID: '#{from_pid}', to project: '#{to_project.title}', PID: '#{to_project.pid}'"
|
78
|
+
res = to_project.save_ldm_layout(from_ldm_layout)
|
79
|
+
|
80
|
+
if res[:status] == 'OK' || !collect_synced_status
|
81
|
+
res[:from] = from_pid
|
82
|
+
results << res
|
83
|
+
else
|
84
|
+
warning_message = "Failed to transfer ldm layout from project: '#{from_pid}' to project: '#{to_project.title}'"
|
85
|
+
process_failed_project(pid, warning_message, failed_projects, collect_synced_status)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
process_failed_projects(failed_projects, short_name, params) if collect_synced_status
|
92
|
+
# Return results
|
93
|
+
results.flatten
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
@@ -0,0 +1,108 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require_relative 'base_action'
|
9
|
+
|
10
|
+
require 'thread_safe'
|
11
|
+
require 'json'
|
12
|
+
|
13
|
+
module GoodData
|
14
|
+
module LCM2
|
15
|
+
class SynchronizePPDashboardPermissions < BaseAction
|
16
|
+
DESCRIPTION = 'Synchronize Pixel Perfect Dashboard Permission'
|
17
|
+
|
18
|
+
PARAMS = define_params(self) do
|
19
|
+
description 'Client Used for Connecting to GD'
|
20
|
+
param :gdc_gd_client, instance_of(Type::GdClientType), required: true
|
21
|
+
|
22
|
+
description 'Client used to connecting to development domain'
|
23
|
+
param :development_client, instance_of(Type::GdClientType), required: true
|
24
|
+
|
25
|
+
description 'Synchronization Info'
|
26
|
+
param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
|
27
|
+
|
28
|
+
description 'Logger'
|
29
|
+
param :gdc_logger, instance_of(Type::GdLogger), required: true
|
30
|
+
|
31
|
+
description 'Additional Hidden Parameters'
|
32
|
+
param :additional_hidden_params, instance_of(Type::HashType), required: false
|
33
|
+
|
34
|
+
description 'Disable synchronizing dashboard permissions for Pixel Perfect dashboards'
|
35
|
+
param :disable_pp_dashboard_permission, instance_of(Type::BooleanType), required: false, default: false
|
36
|
+
|
37
|
+
description 'Abort on error'
|
38
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
39
|
+
|
40
|
+
description 'Collect synced status'
|
41
|
+
param :collect_synced_status, instance_of(Type::BooleanType), required: false
|
42
|
+
|
43
|
+
description 'Sync failed list'
|
44
|
+
param :sync_failed_list, instance_of(Type::HashType), required: false
|
45
|
+
end
|
46
|
+
|
47
|
+
class << self
|
48
|
+
def call(params)
|
49
|
+
results = []
|
50
|
+
disable_pp_dashboard_permission = GoodData::Helpers.to_boolean(params.disable_pp_dashboard_permission)
|
51
|
+
collect_synced_status = collect_synced_status(params)
|
52
|
+
failed_projects = ThreadSafe::Array.new
|
53
|
+
|
54
|
+
if disable_pp_dashboard_permission
|
55
|
+
params.gdc_logger.info "Skip synchronize Pixel Perfect dashboard permission."
|
56
|
+
else
|
57
|
+
client = params.gdc_gd_client
|
58
|
+
development_client = params.development_client
|
59
|
+
failed_projects = ThreadSafe::Array.new
|
60
|
+
|
61
|
+
params.synchronize.peach do |info|
|
62
|
+
from_project = info.from
|
63
|
+
to_projects = info.to
|
64
|
+
sync_success = false
|
65
|
+
|
66
|
+
from = development_client.projects(from_project)
|
67
|
+
unless from
|
68
|
+
process_failed_project(from_project, "Invalid 'from' project specified - '#{from_project}'", failed_projects, collect_synced_status)
|
69
|
+
next
|
70
|
+
end
|
71
|
+
|
72
|
+
source_dashboards = from.dashboards
|
73
|
+
|
74
|
+
params.gdc_logger.info "Transferring Pixel Perfect Dashboard permission, from project: '#{from.title}', PID: '#{from.pid}' for dashboard(s): #{source_dashboards.map { |d| "#{d.title.inspect}" }.join(', ')}" # rubocop:disable Metrics/LineLength
|
75
|
+
to_projects.peach do |entry|
|
76
|
+
pid = entry[:pid]
|
77
|
+
next if sync_failed_project(pid, params)
|
78
|
+
|
79
|
+
to_project = client.projects(pid)
|
80
|
+
unless to_project
|
81
|
+
process_failed_project(pid, "Invalid 'to' project specified - '#{pid}'", failed_projects, collect_synced_status)
|
82
|
+
next
|
83
|
+
end
|
84
|
+
|
85
|
+
target_dashboards = to_project.dashboards
|
86
|
+
begin
|
87
|
+
GoodData::Project.transfer_dashboard_permission(from, to_project, source_dashboards, target_dashboards)
|
88
|
+
sync_success = true
|
89
|
+
rescue StandardError => err
|
90
|
+
process_failed_project(pid, err.message, failed_projects, collect_synced_status)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
results << {
|
95
|
+
from_project_name: from.title,
|
96
|
+
from_project_pid: from.pid,
|
97
|
+
status: 'ok'
|
98
|
+
} if sync_success
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
process_failed_projects(failed_projects, short_name, params) if collect_synced_status
|
103
|
+
results
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
@@ -61,12 +61,13 @@ module GoodData
|
|
61
61
|
to_projects = info.to
|
62
62
|
|
63
63
|
from = development_client.projects(from_project) || fail("Invalid 'from' project specified - '#{from_project}'")
|
64
|
+
has_cycle_trigger = exist_cycle_trigger(from)
|
64
65
|
to_projects.peach do |entry|
|
65
66
|
pid = entry[:pid]
|
66
67
|
to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
|
67
68
|
|
68
69
|
params.gdc_logger.info "Transferring Schedules, from project: '#{from.title}', PID: '#{from.pid}', to project: '#{to_project.title}', PID: '#{to_project.pid}'"
|
69
|
-
res = GoodData::Project.transfer_schedules(from, to_project).sort_by do |item|
|
70
|
+
res = GoodData::Project.transfer_schedules(from, to_project, has_cycle_trigger).sort_by do |item|
|
70
71
|
item[:status]
|
71
72
|
end
|
72
73
|
|
@@ -97,6 +98,35 @@ module GoodData
|
|
97
98
|
# Return results
|
98
99
|
results
|
99
100
|
end
|
101
|
+
|
102
|
+
private
|
103
|
+
|
104
|
+
def exist_cycle_trigger(project)
|
105
|
+
schedules = project.schedules
|
106
|
+
triggers = {}
|
107
|
+
schedules.each do |schedule|
|
108
|
+
triggers[schedule.obj_id] = schedule.trigger_id if schedule.trigger_id
|
109
|
+
end
|
110
|
+
|
111
|
+
triggers.each do |schedule_id, trigger_id|
|
112
|
+
checking_id = trigger_id
|
113
|
+
if checking_id == schedule_id
|
114
|
+
return true
|
115
|
+
else
|
116
|
+
max_step = triggers.length
|
117
|
+
count = 1
|
118
|
+
loop do
|
119
|
+
checking_id = triggers[checking_id]
|
120
|
+
count += 1
|
121
|
+
break if !checking_id || count > max_step
|
122
|
+
|
123
|
+
return true if checking_id == schedule_id
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
false
|
129
|
+
end
|
100
130
|
end
|
101
131
|
end
|
102
132
|
end
|
@@ -121,7 +121,7 @@ module GoodData
|
|
121
121
|
users_brick_input: params.users_brick_users
|
122
122
|
}
|
123
123
|
all_clients = domain.clients(:all, data_product).to_a
|
124
|
-
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, number_of_clients=#{all_clients.size}, data_rows=#{user_filters.size}")
|
124
|
+
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, number_of_clients=#{all_clients.size}, data_rows=#{user_filters.size} ,")
|
125
125
|
|
126
126
|
GoodData.logger.info("Synchronizing in mode \"#{mode}\"")
|
127
127
|
results = []
|
@@ -134,7 +134,7 @@ module GoodData
|
|
134
134
|
end
|
135
135
|
user_filters = user_filters.select { |f| f[:pid] == filter } if filter
|
136
136
|
|
137
|
-
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{user_filters.size}")
|
137
|
+
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{user_filters.size} ,")
|
138
138
|
current_results = sync_user_filters(project, user_filters, run_params, symbolized_config)
|
139
139
|
|
140
140
|
results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
|
@@ -151,7 +151,7 @@ module GoodData
|
|
151
151
|
current_project = client.projects(id)
|
152
152
|
end
|
153
153
|
|
154
|
-
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{id}, data_rows=#{new_filters.size}")
|
154
|
+
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{id}, data_rows=#{new_filters.size} ,")
|
155
155
|
current_results = sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
|
156
156
|
|
157
157
|
results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
|
@@ -164,6 +164,7 @@ module GoodData
|
|
164
164
|
end
|
165
165
|
|
166
166
|
working_client_ids = []
|
167
|
+
semaphore = Mutex.new
|
167
168
|
|
168
169
|
users_by_project = run_params[:users_brick_input].group_by { |u| u[:pid] }
|
169
170
|
user_filters.group_by { |u| u[multiple_projects_column] }.flat_map.pmap do |client_id, new_filters|
|
@@ -182,23 +183,26 @@ module GoodData
|
|
182
183
|
current_project = c.project
|
183
184
|
fail "Client #{client_id} does not have project." unless current_project
|
184
185
|
|
185
|
-
|
186
|
+
semaphore.synchronize do
|
187
|
+
working_client_ids << client_id.to_s
|
188
|
+
end
|
186
189
|
|
187
|
-
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, client_id=#{client_id}, data_rows=#{new_filters.size}")
|
190
|
+
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, client_id=#{client_id}, data_rows=#{new_filters.size} ,")
|
188
191
|
partial_results = sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
|
189
192
|
results.concat(partial_results[:results]) unless partial_results.nil? || partial_results[:results].empty?
|
190
193
|
end
|
191
194
|
|
192
|
-
|
193
|
-
domain_clients.peach do |c|
|
194
|
-
next if working_client_ids.include?(c.client_id)
|
195
|
+
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, working_client_ids=#{working_client_ids.join(', ')} ,") if working_client_ids.size < 50
|
195
196
|
|
197
|
+
unless run_params[:do_not_touch_filters_that_are_not_mentioned]
|
198
|
+
to_be_deleted_clients = UserBricksHelper.non_working_clients(domain_clients, working_client_ids)
|
199
|
+
to_be_deleted_clients.peach do |c|
|
196
200
|
begin
|
197
201
|
current_project = c.project
|
198
202
|
users = users_by_project[c.client_id]
|
199
203
|
params.gdc_logger.info "Delete all filters in project #{current_project.pid} of client #{c.client_id}"
|
200
204
|
|
201
|
-
GoodData.gd_logger.info("Delete all filters in project_id=#{current_project.pid}, client_id=#{c.client_id}")
|
205
|
+
GoodData.gd_logger.info("Delete all filters in project_id=#{current_project.pid}, client_id=#{c.client_id} ,")
|
202
206
|
current_results = sync_user_filters(current_project, [], run_params.merge(users_brick_input: users), symbolized_config)
|
203
207
|
|
204
208
|
results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
|
@@ -208,6 +212,7 @@ module GoodData
|
|
208
212
|
end
|
209
213
|
end
|
210
214
|
end
|
215
|
+
|
211
216
|
{
|
212
217
|
results: results
|
213
218
|
}
|
@@ -27,11 +27,22 @@ module GoodData
|
|
27
27
|
|
28
28
|
description 'Additional Hidden Parameters'
|
29
29
|
param :additional_hidden_params, instance_of(Type::HashType), required: false
|
30
|
+
|
31
|
+
description 'Abort on error'
|
32
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
33
|
+
|
34
|
+
description 'Collect synced status'
|
35
|
+
param :collect_synced_status, instance_of(Type::BooleanType), required: false
|
36
|
+
|
37
|
+
description 'Sync failed list'
|
38
|
+
param :sync_failed_list, instance_of(Type::HashType), required: false
|
30
39
|
end
|
31
40
|
|
32
41
|
class << self
|
33
42
|
def call(params)
|
34
43
|
results = ThreadSafe::Array.new
|
44
|
+
collect_synced_status = collect_synced_status(params)
|
45
|
+
failed_projects = ThreadSafe::Array.new
|
35
46
|
|
36
47
|
client = params.gdc_gd_client
|
37
48
|
development_client = params.development_client
|
@@ -40,17 +51,32 @@ module GoodData
|
|
40
51
|
from_project = info.from
|
41
52
|
to_projects = info.to
|
42
53
|
|
43
|
-
from = development_client.projects(from_project)
|
54
|
+
from = development_client.projects(from_project)
|
55
|
+
unless from
|
56
|
+
process_failed_project(from_project, "Invalid 'from' project specified - '#{from_project}'", failed_projects, collect_synced_status)
|
57
|
+
next
|
58
|
+
end
|
44
59
|
|
45
60
|
to_projects.peach do |entry|
|
46
61
|
pid = entry[:pid]
|
47
|
-
|
62
|
+
next if sync_failed_project(pid, params)
|
63
|
+
|
64
|
+
to_project = client.projects(pid)
|
65
|
+
unless to_project
|
66
|
+
process_failed_project(pid, "Invalid 'to' project specified - '#{pid}'", failed_projects, collect_synced_status)
|
67
|
+
next
|
68
|
+
end
|
48
69
|
|
49
|
-
|
50
|
-
|
70
|
+
begin
|
71
|
+
params.gdc_logger.info "Transferring User Groups, from project: '#{from.title}', PID: '#{from.pid}', to project: '#{to_project.title}', PID: '#{to_project.pid}'"
|
72
|
+
results += GoodData::Project.transfer_user_groups(from, to_project)
|
73
|
+
rescue StandardError => err
|
74
|
+
process_failed_project(pid, err.message, failed_projects, collect_synced_status)
|
75
|
+
end
|
51
76
|
end
|
52
77
|
end
|
53
78
|
|
79
|
+
process_failed_projects(failed_projects, short_name, params) if collect_synced_status
|
54
80
|
results.uniq
|
55
81
|
end
|
56
82
|
end
|
@@ -190,7 +190,7 @@ module GoodData
|
|
190
190
|
create_non_existing_user_groups: create_non_existing_user_groups,
|
191
191
|
user_groups_cache: nil
|
192
192
|
}
|
193
|
-
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, data_rows=#{new_users.size}")
|
193
|
+
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, data_rows=#{new_users.size} ,")
|
194
194
|
|
195
195
|
GoodData.logger.info("Synchronizing in mode \"#{mode}\"")
|
196
196
|
results = case mode
|
@@ -202,7 +202,7 @@ module GoodData
|
|
202
202
|
params.gdc_logger.info "#{user_ids.count - users.count} users were not found (or were deleted) in domain #{domain_name}" if user_ids.count > users.count
|
203
203
|
params.gdc_logger.warn "Deleting #{users.count} users from domain #{domain_name}"
|
204
204
|
|
205
|
-
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, domain=#{domain_name}, data_rows=#{users.count}")
|
205
|
+
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, domain=#{domain_name}, data_rows=#{users.count} ,")
|
206
206
|
users.map(&:delete)
|
207
207
|
when 'sync_project'
|
208
208
|
project.import_users(new_users, common_params)
|
@@ -211,7 +211,7 @@ module GoodData
|
|
211
211
|
begin
|
212
212
|
project = client.projects(project_id)
|
213
213
|
|
214
|
-
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project_id}, data_rows=#{users.count}")
|
214
|
+
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project_id}, data_rows=#{users.count} ,")
|
215
215
|
project.import_users(users, common_params)
|
216
216
|
rescue RestClient::ResourceNotFound
|
217
217
|
fail "Project \"#{project_id}\" was not found. Please check your project ids in the source file"
|
@@ -224,7 +224,7 @@ module GoodData
|
|
224
224
|
when 'sync_one_project_based_on_pid'
|
225
225
|
filtered_users = new_users.select { |u| u[:pid] == project.pid }
|
226
226
|
|
227
|
-
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, data_rows=#{filtered_users.count}")
|
227
|
+
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, data_rows=#{filtered_users.count} ,")
|
228
228
|
project.import_users(filtered_users, common_params)
|
229
229
|
when 'sync_one_project_based_on_custom_id'
|
230
230
|
filter_value = UserBricksHelper.resolve_client_id(domain, project, data_product)
|
@@ -245,7 +245,7 @@ module GoodData
|
|
245
245
|
end
|
246
246
|
|
247
247
|
GoodData.logger.info("Project #{project.pid} will receive #{filtered_users.count} from #{new_users.count} users")
|
248
|
-
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, filtered_users=#{filtered_users.count}, data_rows=#{new_users.count}")
|
248
|
+
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, filtered_users=#{filtered_users.count}, data_rows=#{new_users.count} ,")
|
249
249
|
project.import_users(filtered_users, common_params)
|
250
250
|
when 'sync_multiple_projects_based_on_custom_id'
|
251
251
|
all_clients = domain.clients(:all, data_product).to_a
|
@@ -260,7 +260,7 @@ module GoodData
|
|
260
260
|
|
261
261
|
GoodData.logger.info("Project #{project.pid} of client #{client_id} will receive #{users.count} users")
|
262
262
|
|
263
|
-
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{users.count}")
|
263
|
+
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{users.count} ,")
|
264
264
|
project.import_users(users, common_params)
|
265
265
|
end
|
266
266
|
when 'sync_domain_client_workspaces'
|
@@ -294,7 +294,7 @@ module GoodData
|
|
294
294
|
working_client_ids << client_id.to_s
|
295
295
|
GoodData.logger.info("Project #{project.pid} of client #{client_id} will receive #{users.count} users")
|
296
296
|
|
297
|
-
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{users.count}")
|
297
|
+
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{users.count} ,")
|
298
298
|
project.import_users(users, common_params)
|
299
299
|
end
|
300
300
|
|
@@ -319,17 +319,17 @@ module GoodData
|
|
319
319
|
end
|
320
320
|
GoodData.logger.info("Synchronizing all users in project #{project.pid} of client #{c.client_id}")
|
321
321
|
|
322
|
-
GoodData.gd_logger.info("Synchronizing all users in project_id=#{project.pid}, client_id=#{c.client_id}")
|
322
|
+
GoodData.gd_logger.info("Synchronizing all users in project_id=#{project.pid}, client_id=#{c.client_id} ,")
|
323
323
|
res += project.import_users([], common_params)
|
324
324
|
end
|
325
325
|
end
|
326
326
|
|
327
327
|
res
|
328
328
|
when 'sync_domain_and_project'
|
329
|
-
GoodData.gd_logger.info("Create users in mode=#{mode}, data_rows=#{new_users.count}")
|
329
|
+
GoodData.gd_logger.info("Create users in mode=#{mode}, data_rows=#{new_users.count} ,")
|
330
330
|
domain.create_users(new_users, ignore_failures: ignore_failures)
|
331
331
|
|
332
|
-
GoodData.gd_logger.info("Import users in mode=#{mode}, data_rows=#{new_users.count}")
|
332
|
+
GoodData.gd_logger.info("Import users in mode=#{mode}, data_rows=#{new_users.count} ,")
|
333
333
|
project.import_users(new_users, common_params)
|
334
334
|
end
|
335
335
|
|
@@ -390,6 +390,7 @@ module GoodData
|
|
390
390
|
|
391
391
|
user_group = row[user_groups_column] || row[user_groups_column.to_sym]
|
392
392
|
user_group = user_group.split(',').map(&:strip) if user_group
|
393
|
+
user_group = [] if row.headers.include?(user_groups_column) && !user_group
|
393
394
|
|
394
395
|
ip_whitelist = row[ip_whitelist_column] || row[ip_whitelist_column.to_sym]
|
395
396
|
ip_whitelist = ip_whitelist.split(',').map(&:strip) if ip_whitelist
|