gooddata 2.1.19-java → 2.3.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gdc-ii-config.yaml +42 -1
- data/.github/workflows/build.yml +67 -0
- data/.github/workflows/pre-merge.yml +72 -0
- data/.pronto.yml +1 -0
- data/.rubocop.yml +2 -14
- data/CHANGELOG.md +47 -0
- data/Dockerfile +27 -14
- data/Dockerfile.jruby +5 -15
- data/Dockerfile.ruby +5 -7
- data/Gemfile +4 -2
- data/README.md +6 -6
- data/Rakefile +1 -1
- data/SDK_VERSION +1 -1
- data/VERSION +1 -1
- data/bin/run_brick.rb +7 -0
- data/ci/mssql/pom.xml +62 -0
- data/ci/mysql/pom.xml +62 -0
- data/ci/redshift/pom.xml +4 -5
- data/docker-compose.lcm.yml +42 -4
- data/docker-compose.yml +42 -0
- data/gooddata.gemspec +21 -21
- data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
- data/lcm.rake +11 -8
- data/lib/gooddata/bricks/base_pipeline.rb +26 -0
- data/lib/gooddata/bricks/brick.rb +0 -1
- data/lib/gooddata/bricks/middleware/aws_middleware.rb +35 -9
- data/lib/gooddata/bricks/middleware/execution_result_middleware.rb +3 -3
- data/lib/gooddata/bricks/pipeline.rb +2 -14
- data/lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb +98 -0
- data/lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mssql/mssql_client.rb +122 -0
- data/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +121 -0
- data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +0 -1
- data/lib/gooddata/cloud_resources/redshift/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/redshift/redshift_client.rb +0 -2
- data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +18 -1
- data/lib/gooddata/helpers/data_helper.rb +9 -4
- data/lib/gooddata/lcm/actions/base_action.rb +157 -0
- data/lib/gooddata/lcm/actions/collect_data_product.rb +2 -1
- data/lib/gooddata/lcm/actions/collect_meta.rb +3 -1
- data/lib/gooddata/lcm/actions/collect_projects_warning_status.rb +53 -0
- data/lib/gooddata/lcm/actions/collect_segment_clients.rb +14 -0
- data/lib/gooddata/lcm/actions/initialize_continue_on_error_option.rb +87 -0
- data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +31 -4
- data/lib/gooddata/lcm/actions/provision_clients.rb +34 -5
- data/lib/gooddata/lcm/actions/synchronize_cas.rb +24 -4
- data/lib/gooddata/lcm/actions/synchronize_clients.rb +112 -11
- data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +89 -0
- data/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb +48 -11
- data/lib/gooddata/lcm/actions/synchronize_kd_dashboard_permission.rb +103 -0
- data/lib/gooddata/lcm/actions/synchronize_ldm.rb +79 -23
- data/lib/gooddata/lcm/actions/synchronize_ldm_layout.rb +98 -0
- data/lib/gooddata/lcm/actions/synchronize_pp_dashboard_permission.rb +108 -0
- data/lib/gooddata/lcm/actions/synchronize_schedules.rb +31 -1
- data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +26 -18
- data/lib/gooddata/lcm/actions/synchronize_user_groups.rb +30 -4
- data/lib/gooddata/lcm/actions/synchronize_users.rb +11 -10
- data/lib/gooddata/lcm/actions/update_metric_formats.rb +202 -0
- data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb +5 -0
- data/lib/gooddata/lcm/exceptions/lcm_execution_warning.rb +15 -0
- data/lib/gooddata/lcm/helpers/check_helper.rb +19 -0
- data/lib/gooddata/lcm/helpers/release_table_helper.rb +42 -8
- data/lib/gooddata/lcm/lcm2.rb +50 -4
- data/lib/gooddata/lcm/user_bricks_helper.rb +9 -0
- data/lib/gooddata/mixins/inspector.rb +1 -1
- data/lib/gooddata/mixins/md_object_query.rb +1 -0
- data/lib/gooddata/models/data_source.rb +5 -1
- data/lib/gooddata/models/dataset_mapping.rb +36 -0
- data/lib/gooddata/models/ldm_layout.rb +38 -0
- data/lib/gooddata/models/metadata/label.rb +26 -27
- data/lib/gooddata/models/project.rb +230 -30
- data/lib/gooddata/models/project_creator.rb +83 -6
- data/lib/gooddata/models/schedule.rb +13 -1
- data/lib/gooddata/models/segment.rb +2 -1
- data/lib/gooddata/models/user_filters/user_filter_builder.rb +162 -68
- data/lib/gooddata/rest/connection.rb +5 -3
- data/lib/gooddata/rest/phmap.rb +2 -0
- data/lib/gooddata.rb +1 -0
- data/lib/gooddata_brick_base.rb +35 -0
- data/sonar-project.properties +6 -0
- metadata +96 -65
- data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb +0 -37
- data/lib/gooddata/cloud_resources/redshift/drivers/log4j.properties +0 -15
@@ -0,0 +1,87 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# (C) 2019-2022 GoodData Corporation
|
3
|
+
require_relative 'base_action'
|
4
|
+
|
5
|
+
module GoodData
|
6
|
+
module LCM2
|
7
|
+
class InitializeContinueOnErrorOption < BaseAction
|
8
|
+
DESCRIPTION = 'Initialize continue on error option'
|
9
|
+
|
10
|
+
PARAMS = define_params(self) do
|
11
|
+
description 'Organization Name'
|
12
|
+
param :organization, instance_of(Type::StringType), required: false
|
13
|
+
|
14
|
+
description 'Domain'
|
15
|
+
param :domain, instance_of(Type::StringType), required: false
|
16
|
+
|
17
|
+
description 'DataProduct'
|
18
|
+
param :data_product, instance_of(Type::GDDataProductType), required: false
|
19
|
+
|
20
|
+
description 'Restricts synchronization to specified segments'
|
21
|
+
param :segments_filter, array_of(instance_of(Type::StringType)), required: false
|
22
|
+
|
23
|
+
description 'Abort on error'
|
24
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
25
|
+
|
26
|
+
description 'Client Used for Connecting to GD'
|
27
|
+
param :gdc_gd_client, instance_of(Type::GdClientType), required: true
|
28
|
+
end
|
29
|
+
|
30
|
+
class << self
|
31
|
+
def call(params)
|
32
|
+
project_mappings = ThreadSafe::Hash.new
|
33
|
+
client_mappings = ThreadSafe::Hash.new
|
34
|
+
continue_on_error = continue_on_error(params)
|
35
|
+
|
36
|
+
if continue_on_error
|
37
|
+
client = params.gdc_gd_client
|
38
|
+
domain_name = params.organization || params.domain
|
39
|
+
domain = client.domain(domain_name)
|
40
|
+
data_product = params.data_product
|
41
|
+
data_product_segments = domain.segments(:all, data_product)
|
42
|
+
|
43
|
+
if params.segments_filter
|
44
|
+
data_product_segments.select! do |segment|
|
45
|
+
params.segments_filter.include?(segment.segment_id)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
data_product_segments.pmap do |segment|
|
50
|
+
segment.clients.map do |segment_client|
|
51
|
+
project = segment_client.project
|
52
|
+
next unless project
|
53
|
+
|
54
|
+
project_mappings[project.pid.to_sym] = {
|
55
|
+
client_id: segment_client.client_id,
|
56
|
+
segment_id: segment.segment_id
|
57
|
+
}
|
58
|
+
client_mappings[segment_client.client_id.to_sym] = {
|
59
|
+
project_id: project.pid,
|
60
|
+
segment_id: segment.segment_id
|
61
|
+
}
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
{
|
67
|
+
params: {
|
68
|
+
collect_synced_status: continue_on_error,
|
69
|
+
sync_failed_list: {
|
70
|
+
project_client_mappings: project_mappings,
|
71
|
+
client_project_mappings: client_mappings,
|
72
|
+
failed_detailed_projects: [],
|
73
|
+
failed_projects: [],
|
74
|
+
failed_clients: [],
|
75
|
+
failed_segments: []
|
76
|
+
}
|
77
|
+
}
|
78
|
+
}
|
79
|
+
end
|
80
|
+
|
81
|
+
def print_result(_params)
|
82
|
+
false
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
@@ -19,6 +19,18 @@ module GoodData
|
|
19
19
|
|
20
20
|
description 'Synchronization Info'
|
21
21
|
param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
|
22
|
+
|
23
|
+
description 'Abort on error'
|
24
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
25
|
+
|
26
|
+
description 'Logger'
|
27
|
+
param :gdc_logger, instance_of(Type::GdLogger), required: false
|
28
|
+
|
29
|
+
description 'Collect synced status'
|
30
|
+
param :collect_synced_status, instance_of(Type::BooleanType), required: false
|
31
|
+
|
32
|
+
description 'Sync failed list'
|
33
|
+
param :sync_failed_list, instance_of(Type::HashType), required: false
|
22
34
|
end
|
23
35
|
|
24
36
|
RESULT_HEADER = %i[from to status]
|
@@ -27,6 +39,8 @@ module GoodData
|
|
27
39
|
def call(params)
|
28
40
|
results = []
|
29
41
|
params.synchronize.map do |segment_info|
|
42
|
+
next if sync_failed_segment(segment_info[:segment_id], params)
|
43
|
+
|
30
44
|
result = migrate_date_dimension(params, segment_info)
|
31
45
|
results.concat(result)
|
32
46
|
end
|
@@ -47,21 +61,32 @@ module GoodData
|
|
47
61
|
# check latest master and previous master
|
48
62
|
master_upgrade_datasets = get_upgrade_dates(latest_blueprint, previous_blueprint) if params[:synchronize_ldm].downcase == 'diff_against_master' && previous_blueprint
|
49
63
|
unless master_upgrade_datasets&.empty?
|
64
|
+
collect_synced_status = collect_synced_status(params)
|
65
|
+
failed_projects = ThreadSafe::Array.new
|
66
|
+
|
50
67
|
segment_info[:to].pmap do |entry|
|
51
68
|
pid = entry[:pid]
|
69
|
+
next if sync_failed_project(pid, params)
|
70
|
+
|
52
71
|
to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
|
72
|
+
GoodData.logger.info "Migrating date dimension, project: '#{to_project.title}', PID: #{pid}"
|
53
73
|
to_blueprint = to_project.blueprint
|
54
74
|
upgrade_datasets = get_upgrade_dates(latest_blueprint, to_blueprint)
|
55
75
|
next if upgrade_datasets.empty?
|
56
76
|
|
57
77
|
message = get_upgrade_message(upgrade_datasets)
|
78
|
+
failed_message = "Failed to migrate date dimension for project #{pid}"
|
79
|
+
update_status = to_project.upgrade_custom_v2(message)
|
80
|
+
process_failed_project(pid, failed_message, failed_projects, collect_synced_status) if collect_synced_status && update_status != 'OK'
|
58
81
|
|
59
82
|
results << {
|
60
83
|
from: segment_info[:from],
|
61
84
|
to: pid,
|
62
|
-
status:
|
85
|
+
status: update_status
|
63
86
|
}
|
64
87
|
end
|
88
|
+
|
89
|
+
process_failed_projects(failed_projects, short_name, params) if collect_synced_status
|
65
90
|
end
|
66
91
|
|
67
92
|
results
|
@@ -71,9 +96,9 @@ module GoodData
|
|
71
96
|
dest_dates = get_date_dimensions(dest_blueprint) if dest_blueprint
|
72
97
|
src_dates = get_date_dimensions(src_blueprint) if src_blueprint
|
73
98
|
|
74
|
-
return false if dest_dates.empty? || src_dates.empty?
|
75
|
-
|
76
99
|
upgrade_datasets = []
|
100
|
+
return upgrade_datasets if dest_dates.empty? || src_dates.empty?
|
101
|
+
|
77
102
|
dest_dates.each do |dest|
|
78
103
|
src_dim = get_date_dimension(src_blueprint, dest[:id])
|
79
104
|
next unless src_dim
|
@@ -103,8 +128,10 @@ module GoodData
|
|
103
128
|
get_date_dimensions(blueprint).any? { |e| e[:urn] == DATE_DIMENSION_CUSTOM_V2 }
|
104
129
|
end
|
105
130
|
|
131
|
+
# Get date dimension from blue print. Return nil if date dimension not existing
|
106
132
|
def get_date_dimension(blueprint, id)
|
107
|
-
|
133
|
+
date_dimensions = get_date_dimensions(blueprint)
|
134
|
+
date_dimensions.find { |d| d[:id] == id }
|
108
135
|
end
|
109
136
|
|
110
137
|
def get_date_dimensions(blueprint)
|
@@ -29,7 +29,16 @@ module GoodData
|
|
29
29
|
param :data_product, instance_of(Type::GDDataProductType), required: false
|
30
30
|
|
31
31
|
description 'Logger'
|
32
|
-
param :gdc_logger, instance_of(Type::GdLogger), required:
|
32
|
+
param :gdc_logger, instance_of(Type::GdLogger), required: false
|
33
|
+
|
34
|
+
description 'Abort on error'
|
35
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
36
|
+
|
37
|
+
description 'Collect synced status'
|
38
|
+
param :collect_synced_status, instance_of(Type::BooleanType), required: false
|
39
|
+
|
40
|
+
description 'Sync failed list'
|
41
|
+
param :sync_failed_list, instance_of(Type::HashType), required: false
|
33
42
|
end
|
34
43
|
|
35
44
|
RESULT_HEADER = [
|
@@ -45,6 +54,8 @@ module GoodData
|
|
45
54
|
synchronize_projects = []
|
46
55
|
data_product = params.data_product
|
47
56
|
client = params.gdc_gd_client
|
57
|
+
collect_synced_status = collect_synced_status(params)
|
58
|
+
continue_on_error = continue_on_error(params)
|
48
59
|
domain_name = params.organization || params.domain
|
49
60
|
fail "Either organisation or domain has to be specified in params" unless domain_name
|
50
61
|
domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
|
@@ -52,6 +63,8 @@ module GoodData
|
|
52
63
|
invalid_client_ids = []
|
53
64
|
begin
|
54
65
|
results = params.segments.map do |segment|
|
66
|
+
next if sync_failed_segment(segment.segment_id, params)
|
67
|
+
|
55
68
|
segment_object = domain.segments(segment.segment_id, data_product)
|
56
69
|
tmp = segment_object.provision_client_projects.map do |m|
|
57
70
|
Hash[m.each_pair.to_a].merge(type: :provision_result)
|
@@ -65,10 +78,21 @@ module GoodData
|
|
65
78
|
unless entry[:project_uri]
|
66
79
|
error_message = "There was error during provisioning clients: #{entry[:error]}" unless error_message
|
67
80
|
invalid_client_ids << entry[:id]
|
81
|
+
if collect_synced_status
|
82
|
+
failed_message = "Failed to provision client #{entry[:id]} in segment #{segment.segment_id}. Error: #{entry[:error]}"
|
83
|
+
add_failed_client(entry[:id], failed_message, short_name, params)
|
84
|
+
end
|
68
85
|
next
|
69
86
|
end
|
87
|
+
|
88
|
+
project_id = entry[:project_uri].split('/').last
|
89
|
+
if collect_synced_status && entry[:status] == 'CREATED' && entry[:id]
|
90
|
+
# Update project client mappings when there are create new clients during provision clients of the segment
|
91
|
+
add_new_clients_to_project_client_mapping(project_id, entry[:id], segment.segment_id, params)
|
92
|
+
end
|
93
|
+
|
70
94
|
{
|
71
|
-
pid:
|
95
|
+
pid: project_id,
|
72
96
|
client_id: entry[:id]
|
73
97
|
}
|
74
98
|
end.compact
|
@@ -93,15 +117,20 @@ module GoodData
|
|
93
117
|
end
|
94
118
|
|
95
119
|
params.gdc_logger.debug "Deleted clients: #{deleted_client_ids.join(', ')}"
|
96
|
-
|
120
|
+
unless error_message['TooManyProjectsCreatedException'] || error_message['Max number registered projects']
|
121
|
+
raise error_message unless continue_on_error
|
122
|
+
|
123
|
+
next
|
124
|
+
end
|
125
|
+
|
97
126
|
break tmp
|
98
127
|
end
|
99
128
|
|
100
129
|
tmp
|
101
130
|
end
|
102
131
|
rescue => e
|
103
|
-
params.gdc_logger.error "Problem occurs when provisioning clients."
|
104
|
-
raise e
|
132
|
+
params.gdc_logger.error "Problem occurs when provisioning clients. Error: #{e}"
|
133
|
+
raise e unless continue_on_error
|
105
134
|
end
|
106
135
|
|
107
136
|
results.flatten! if results
|
@@ -32,6 +32,15 @@ module GoodData
|
|
32
32
|
|
33
33
|
description 'Additional Hidden Parameters'
|
34
34
|
param :additional_hidden_params, instance_of(Type::HashType), required: false
|
35
|
+
|
36
|
+
description 'Abort on error'
|
37
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
38
|
+
|
39
|
+
description 'Collect synced status'
|
40
|
+
param :collect_synced_status, instance_of(Type::BooleanType), required: false
|
41
|
+
|
42
|
+
description 'Sync failed list'
|
43
|
+
param :sync_failed_list, instance_of(Type::HashType), required: false
|
35
44
|
end
|
36
45
|
|
37
46
|
class << self
|
@@ -46,6 +55,8 @@ module GoodData
|
|
46
55
|
return results unless include_ca
|
47
56
|
|
48
57
|
client = params.gdc_gd_client
|
58
|
+
collect_synced_status = collect_synced_status(params)
|
59
|
+
failed_projects = ThreadSafe::Array.new
|
49
60
|
|
50
61
|
params.synchronize.each do |info|
|
51
62
|
from = info.from
|
@@ -58,24 +69,33 @@ module GoodData
|
|
58
69
|
next unless ca_scripts
|
59
70
|
|
60
71
|
pid = entry[:pid]
|
72
|
+
next if sync_failed_project(pid, params)
|
73
|
+
|
61
74
|
ca_chunks = ca_scripts[:maqlDdlChunks]
|
62
|
-
to_project = client.projects(pid)
|
63
|
-
|
75
|
+
to_project = client.projects(pid)
|
76
|
+
unless to_project
|
77
|
+
process_failed_project(pid, "Invalid 'to' project specified - '#{pid}'", failed_projects, collect_synced_status)
|
78
|
+
next
|
79
|
+
end
|
64
80
|
|
81
|
+
params.gdc_logger.info "Synchronizing Computed Attributes to project: '#{to_project.title}', PID: #{pid}"
|
82
|
+
error_message = nil
|
65
83
|
begin
|
66
84
|
ca_chunks.each { |chunk| to_project.execute_maql(chunk) }
|
67
85
|
rescue => e
|
68
|
-
|
86
|
+
error_message = "Error occurred when executing MAQL for project: \"#{to_project.title}\" reason: \"#{e.message}\", chunks: #{ca_chunks.inspect}"
|
87
|
+
process_failed_project(pid, error_message, failed_projects, collect_synced_status)
|
69
88
|
end
|
70
89
|
|
71
90
|
results << {
|
72
91
|
from: from,
|
73
92
|
to: pid,
|
74
|
-
status: 'ok'
|
93
|
+
status: error_message.nil? ? 'ok' : 'failed'
|
75
94
|
}
|
76
95
|
end
|
77
96
|
end
|
78
97
|
|
98
|
+
process_failed_projects(failed_projects, short_name, params) if collect_synced_status
|
79
99
|
results
|
80
100
|
end
|
81
101
|
end
|
@@ -33,8 +33,23 @@ module GoodData
|
|
33
33
|
description 'ADS Client'
|
34
34
|
param :ads_client, instance_of(Type::AdsClientType), required: false
|
35
35
|
|
36
|
+
description 'Keep number of old master workspace excluding the latest one'
|
37
|
+
param :keep_only_previous_masters_count, instance_of(Type::StringType), required: false, default: '-1'
|
38
|
+
|
36
39
|
description 'Additional Hidden Parameters'
|
37
40
|
param :additional_hidden_params, instance_of(Type::HashType), required: false
|
41
|
+
|
42
|
+
description 'Abort on error'
|
43
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
44
|
+
|
45
|
+
description 'Logger'
|
46
|
+
param :gdc_logger, instance_of(Type::GdLogger), required: false
|
47
|
+
|
48
|
+
description 'Collect synced status'
|
49
|
+
param :collect_synced_status, instance_of(Type::BooleanType), required: false
|
50
|
+
|
51
|
+
description 'Sync failed list'
|
52
|
+
param :sync_failed_list, instance_of(Type::HashType), required: false
|
38
53
|
end
|
39
54
|
|
40
55
|
RESULT_HEADER = [
|
@@ -53,6 +68,7 @@ module GoodData
|
|
53
68
|
domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
|
54
69
|
data_product = params.data_product
|
55
70
|
domain_segments = domain.segments(:all, data_product)
|
71
|
+
keep_only_previous_masters_count = Integer(params.keep_only_previous_masters_count || "-1")
|
56
72
|
|
57
73
|
segments = params.segments.map do |seg|
|
58
74
|
domain_segments.find do |s|
|
@@ -61,19 +77,17 @@ module GoodData
|
|
61
77
|
end
|
62
78
|
|
63
79
|
results = segments.map do |segment|
|
80
|
+
next if sync_failed_segment(segment.segment_id, params)
|
81
|
+
|
64
82
|
if params.ads_client
|
65
|
-
|
66
|
-
params.release_table_name,
|
67
|
-
params.ads_client,
|
68
|
-
segment.segment_id
|
69
|
-
)
|
83
|
+
master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_ads(params.release_table_name, params.ads_client, segment.segment_id)
|
70
84
|
else
|
71
|
-
|
85
|
+
master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
|
72
86
|
end
|
73
87
|
|
88
|
+
current_master = master_projects.last
|
74
89
|
# TODO: Check res.first.nil? || res.first[:master_project_id].nil?
|
75
90
|
master = client.projects(current_master[:master_project_id])
|
76
|
-
|
77
91
|
segment.master_project = master
|
78
92
|
segment.save
|
79
93
|
|
@@ -83,14 +97,26 @@ module GoodData
|
|
83
97
|
failed_count = sync_result['failedClients']['count']
|
84
98
|
|
85
99
|
if failed_count.to_i > 0
|
86
|
-
|
87
|
-
|
100
|
+
error_handle(segment, res, collect_synced_status(params), params)
|
101
|
+
end
|
102
|
+
|
103
|
+
if keep_only_previous_masters_count >= 0
|
104
|
+
number_of_deleted_projects = master_projects.count - (keep_only_previous_masters_count + 1)
|
105
|
+
|
106
|
+
if number_of_deleted_projects.positive?
|
107
|
+
begin
|
108
|
+
removal_master_project_ids = remove_multiple_workspace(params, segment.segment_id, master_projects, number_of_deleted_projects)
|
109
|
+
remove_old_workspaces_from_release_table(params, domain_name, data_product.data_product_id, segment.segment_id, master_projects, removal_master_project_ids)
|
110
|
+
rescue Exception => e # rubocop:disable RescueException
|
111
|
+
GoodData.logger.error "Problem occurs when removing old master workspace, reason: #{e.message}"
|
112
|
+
end
|
113
|
+
end
|
88
114
|
end
|
89
115
|
|
90
116
|
{
|
91
117
|
segment: segment.id,
|
92
|
-
master_pid: master.pid,
|
93
|
-
master_name: master.title,
|
118
|
+
master_pid: master.nil? ? '' : master.pid,
|
119
|
+
master_name: master.nil? ? '' : master.title,
|
94
120
|
successful_count: sync_result['successfulClients']['count']
|
95
121
|
}
|
96
122
|
end
|
@@ -98,6 +124,81 @@ module GoodData
|
|
98
124
|
# Return results
|
99
125
|
results
|
100
126
|
end
|
127
|
+
|
128
|
+
def remove_multiple_workspace(params, segment_id, master_projects, number_of_deleted_projects)
|
129
|
+
removal_master_project_ids = []
|
130
|
+
need_to_delete_projects = master_projects.take(number_of_deleted_projects)
|
131
|
+
|
132
|
+
need_to_delete_projects.each do |project_wrapper|
|
133
|
+
master_project_id = project_wrapper[:master_project_id]
|
134
|
+
next if master_project_id.to_s.empty?
|
135
|
+
|
136
|
+
begin
|
137
|
+
project = params.gdc_gd_client.projects(master_project_id)
|
138
|
+
if project && !%w[deleted archived].include?(project.state.to_s)
|
139
|
+
GoodData.logger.info "Segment #{segment_id}: Deleting old master workspace, project: '#{project.title}', PID: (#{project.pid})."
|
140
|
+
project.delete
|
141
|
+
end
|
142
|
+
removal_master_project_ids << master_project_id
|
143
|
+
master_projects.delete_if { |p| p[:master_project_id] == master_project_id }
|
144
|
+
rescue Exception => ex # rubocop:disable RescueException
|
145
|
+
GoodData.logger.error "Unable to remove master workspace: '#{master_project_id}', Error: #{ex.message}"
|
146
|
+
end
|
147
|
+
end
|
148
|
+
removal_master_project_ids
|
149
|
+
end
|
150
|
+
|
151
|
+
# rubocop:disable Metrics/ParameterLists
|
152
|
+
def remove_old_workspaces_from_release_table(params, domain_id, data_product_id, segment_id, master_projects, removal_master_project_ids)
|
153
|
+
unless removal_master_project_ids.empty?
|
154
|
+
if params.ads_client
|
155
|
+
GoodData::LCM2::Helpers.delete_master_project_from_ads(params.release_table_name, params.ads_client, segment_id, removal_master_project_ids)
|
156
|
+
else
|
157
|
+
data = master_projects.sort_by { |master| master[:version] }
|
158
|
+
GoodData::LCM2::Helpers.update_master_project_to_nfs(domain_id, data_product_id, segment_id, data)
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
162
|
+
# rubocop:enable Metrics/ParameterLists
|
163
|
+
|
164
|
+
private
|
165
|
+
|
166
|
+
def error_handle(segment, error_result, continue_on_error, params)
|
167
|
+
sync_result = error_result.json['synchronizationResult']
|
168
|
+
success_count = sync_result['successfulClients']['count'].to_i
|
169
|
+
failed_count = sync_result['failedClients']['count'].to_i
|
170
|
+
|
171
|
+
# Synchronize failure for all clients in segment
|
172
|
+
if continue_on_error && success_count.zero? && failed_count.positive?
|
173
|
+
segment_warning_message = "Failed to synchronize clients for #{segment.segment_id} segment. Details: #{sync_result['links']['details']}"
|
174
|
+
add_failed_segment(segment.segment_id, segment_warning_message, short_name, params)
|
175
|
+
return
|
176
|
+
end
|
177
|
+
|
178
|
+
summary_error_message = "#{failed_count} clients failed to synchronize. Details: #{sync_result['links']['details']}"
|
179
|
+
fail(summary_error_message) unless continue_on_error
|
180
|
+
|
181
|
+
GoodData.logger.warn summary_error_message
|
182
|
+
if error_result.details # rubocop:disable Style/SafeNavigation
|
183
|
+
error_result.details.items.each do |item|
|
184
|
+
next if item['status'] == 'OK'
|
185
|
+
|
186
|
+
error_client_id = item['id']
|
187
|
+
error_message = error_message(item, segment)
|
188
|
+
add_failed_client(error_client_id, error_message, short_name, params)
|
189
|
+
end
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
def error_message(error_item, segment)
|
194
|
+
error_client_id = error_item['id']
|
195
|
+
error_message = "Failed to synchronize #{error_client_id} client in #{segment.segment_id} segment."
|
196
|
+
error_message = "#{error_message}. Detail: #{error_item['error']['message']}" if error_item['error'] && error_item['error']['message']
|
197
|
+
|
198
|
+
error_message = "#{error_message}. Error items: #{error_item['error']['parameters']}" if error_item['error'] && error_item['error']['parameters']
|
199
|
+
|
200
|
+
error_message
|
201
|
+
end
|
101
202
|
end
|
102
203
|
end
|
103
204
|
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require_relative 'base_action'
|
9
|
+
|
10
|
+
module GoodData
|
11
|
+
module LCM2
|
12
|
+
class SynchronizeDataSetMapping < BaseAction
|
13
|
+
DESCRIPTION = 'Synchronize Dataset Mappings'
|
14
|
+
|
15
|
+
PARAMS = define_params(self) do
|
16
|
+
description 'Client Used for Connecting to GD'
|
17
|
+
param :gdc_gd_client, instance_of(Type::GdClientType), required: true
|
18
|
+
|
19
|
+
description 'Client used to connecting to development domain'
|
20
|
+
param :development_client, instance_of(Type::GdClientType), required: true
|
21
|
+
|
22
|
+
description 'Synchronization Info'
|
23
|
+
param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
|
24
|
+
|
25
|
+
description 'Logger'
|
26
|
+
param :gdc_logger, instance_of(Type::GdLogger), required: true
|
27
|
+
|
28
|
+
description 'Abort on error'
|
29
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
30
|
+
|
31
|
+
description 'Collect synced status'
|
32
|
+
param :collect_synced_status, instance_of(Type::BooleanType), required: false
|
33
|
+
|
34
|
+
description 'Sync failed list'
|
35
|
+
param :sync_failed_list, instance_of(Type::HashType), required: false
|
36
|
+
end
|
37
|
+
|
38
|
+
RESULT_HEADER = %i[from to count status]
|
39
|
+
|
40
|
+
class << self
|
41
|
+
def call(params)
|
42
|
+
results = []
|
43
|
+
collect_synced_status = collect_synced_status(params)
|
44
|
+
failed_projects = ThreadSafe::Array.new
|
45
|
+
|
46
|
+
client = params.gdc_gd_client
|
47
|
+
development_client = params.development_client
|
48
|
+
|
49
|
+
params.synchronize.peach do |info|
|
50
|
+
from_project = info.from
|
51
|
+
to_projects = info.to
|
52
|
+
|
53
|
+
from = development_client.projects(from_project)
|
54
|
+
unless from
|
55
|
+
process_failed_project(from_project, "Invalid 'from' project specified - '#{from_project}'", failed_projects, collect_synced_status)
|
56
|
+
next
|
57
|
+
end
|
58
|
+
|
59
|
+
dataset_mapping = from.dataset_mapping
|
60
|
+
if dataset_mapping&.dig('datasetMappings', 'items').nil? || dataset_mapping['datasetMappings']['items'].empty?
|
61
|
+
params.gdc_logger.info "Project: '#{from.title}', PID: '#{from.pid}' has no model mapping, skip synchronizing model mapping."
|
62
|
+
else
|
63
|
+
to_projects.peach do |to|
|
64
|
+
pid = to[:pid]
|
65
|
+
next if sync_failed_project(pid, params)
|
66
|
+
|
67
|
+
to_project = client.projects(pid)
|
68
|
+
process_failed_project(pid, "Invalid 'to' project specified - '#{pid}'", failed_projects, collect_synced_status) unless to_project
|
69
|
+
|
70
|
+
message_to_project = "to project: '#{to_project.title}', PID: '#{to_project.pid}'"
|
71
|
+
params.gdc_logger.info "Transferring model mapping, from project: '#{from.title}', PID: '#{from.pid}', #{message_to_project}"
|
72
|
+
res = to_project.update_dataset_mapping(dataset_mapping)
|
73
|
+
res[:from] = from.pid
|
74
|
+
results << res
|
75
|
+
|
76
|
+
failed_message = "Failed to transfer model mapping from project '#{from.pid}' to project '#{to_project.pid}'"
|
77
|
+
process_failed_project(pid, failed_message, failed_projects, collect_synced_status) if collect_synced_status && res[:status] != 'OK'
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
process_failed_projects(failed_projects, short_name, params) if collect_synced_status
|
83
|
+
# Return results
|
84
|
+
results.flatten
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|