gooddata 2.1.19-java → 2.3.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gdc-ii-config.yaml +42 -1
- data/.github/workflows/build.yml +67 -0
- data/.github/workflows/pre-merge.yml +72 -0
- data/.pronto.yml +1 -0
- data/.rubocop.yml +2 -14
- data/CHANGELOG.md +47 -0
- data/Dockerfile +27 -14
- data/Dockerfile.jruby +5 -15
- data/Dockerfile.ruby +5 -7
- data/Gemfile +4 -2
- data/README.md +6 -6
- data/Rakefile +1 -1
- data/SDK_VERSION +1 -1
- data/VERSION +1 -1
- data/bin/run_brick.rb +7 -0
- data/ci/mssql/pom.xml +62 -0
- data/ci/mysql/pom.xml +62 -0
- data/ci/redshift/pom.xml +4 -5
- data/docker-compose.lcm.yml +42 -4
- data/docker-compose.yml +42 -0
- data/gooddata.gemspec +21 -21
- data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
- data/lcm.rake +11 -8
- data/lib/gooddata/bricks/base_pipeline.rb +26 -0
- data/lib/gooddata/bricks/brick.rb +0 -1
- data/lib/gooddata/bricks/middleware/aws_middleware.rb +35 -9
- data/lib/gooddata/bricks/middleware/execution_result_middleware.rb +3 -3
- data/lib/gooddata/bricks/pipeline.rb +2 -14
- data/lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb +98 -0
- data/lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mssql/mssql_client.rb +122 -0
- data/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +121 -0
- data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +0 -1
- data/lib/gooddata/cloud_resources/redshift/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/redshift/redshift_client.rb +0 -2
- data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +18 -1
- data/lib/gooddata/helpers/data_helper.rb +9 -4
- data/lib/gooddata/lcm/actions/base_action.rb +157 -0
- data/lib/gooddata/lcm/actions/collect_data_product.rb +2 -1
- data/lib/gooddata/lcm/actions/collect_meta.rb +3 -1
- data/lib/gooddata/lcm/actions/collect_projects_warning_status.rb +53 -0
- data/lib/gooddata/lcm/actions/collect_segment_clients.rb +14 -0
- data/lib/gooddata/lcm/actions/initialize_continue_on_error_option.rb +87 -0
- data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +31 -4
- data/lib/gooddata/lcm/actions/provision_clients.rb +34 -5
- data/lib/gooddata/lcm/actions/synchronize_cas.rb +24 -4
- data/lib/gooddata/lcm/actions/synchronize_clients.rb +112 -11
- data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +89 -0
- data/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb +48 -11
- data/lib/gooddata/lcm/actions/synchronize_kd_dashboard_permission.rb +103 -0
- data/lib/gooddata/lcm/actions/synchronize_ldm.rb +79 -23
- data/lib/gooddata/lcm/actions/synchronize_ldm_layout.rb +98 -0
- data/lib/gooddata/lcm/actions/synchronize_pp_dashboard_permission.rb +108 -0
- data/lib/gooddata/lcm/actions/synchronize_schedules.rb +31 -1
- data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +26 -18
- data/lib/gooddata/lcm/actions/synchronize_user_groups.rb +30 -4
- data/lib/gooddata/lcm/actions/synchronize_users.rb +11 -10
- data/lib/gooddata/lcm/actions/update_metric_formats.rb +202 -0
- data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb +5 -0
- data/lib/gooddata/lcm/exceptions/lcm_execution_warning.rb +15 -0
- data/lib/gooddata/lcm/helpers/check_helper.rb +19 -0
- data/lib/gooddata/lcm/helpers/release_table_helper.rb +42 -8
- data/lib/gooddata/lcm/lcm2.rb +50 -4
- data/lib/gooddata/lcm/user_bricks_helper.rb +9 -0
- data/lib/gooddata/mixins/inspector.rb +1 -1
- data/lib/gooddata/mixins/md_object_query.rb +1 -0
- data/lib/gooddata/models/data_source.rb +5 -1
- data/lib/gooddata/models/dataset_mapping.rb +36 -0
- data/lib/gooddata/models/ldm_layout.rb +38 -0
- data/lib/gooddata/models/metadata/label.rb +26 -27
- data/lib/gooddata/models/project.rb +230 -30
- data/lib/gooddata/models/project_creator.rb +83 -6
- data/lib/gooddata/models/schedule.rb +13 -1
- data/lib/gooddata/models/segment.rb +2 -1
- data/lib/gooddata/models/user_filters/user_filter_builder.rb +162 -68
- data/lib/gooddata/rest/connection.rb +5 -3
- data/lib/gooddata/rest/phmap.rb +2 -0
- data/lib/gooddata.rb +1 -0
- data/lib/gooddata_brick_base.rb +35 -0
- data/sonar-project.properties +6 -0
- metadata +96 -65
- data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb +0 -37
- data/lib/gooddata/cloud_resources/redshift/drivers/log4j.properties +0 -15
@@ -0,0 +1,87 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# (C) 2019-2022 GoodData Corporation
|
3
|
+
require_relative 'base_action'
|
4
|
+
|
5
|
+
module GoodData
|
6
|
+
module LCM2
|
7
|
+
class InitializeContinueOnErrorOption < BaseAction
|
8
|
+
DESCRIPTION = 'Initialize continue on error option'
|
9
|
+
|
10
|
+
PARAMS = define_params(self) do
|
11
|
+
description 'Organization Name'
|
12
|
+
param :organization, instance_of(Type::StringType), required: false
|
13
|
+
|
14
|
+
description 'Domain'
|
15
|
+
param :domain, instance_of(Type::StringType), required: false
|
16
|
+
|
17
|
+
description 'DataProduct'
|
18
|
+
param :data_product, instance_of(Type::GDDataProductType), required: false
|
19
|
+
|
20
|
+
description 'Restricts synchronization to specified segments'
|
21
|
+
param :segments_filter, array_of(instance_of(Type::StringType)), required: false
|
22
|
+
|
23
|
+
description 'Abort on error'
|
24
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
25
|
+
|
26
|
+
description 'Client Used for Connecting to GD'
|
27
|
+
param :gdc_gd_client, instance_of(Type::GdClientType), required: true
|
28
|
+
end
|
29
|
+
|
30
|
+
class << self
|
31
|
+
def call(params)
|
32
|
+
project_mappings = ThreadSafe::Hash.new
|
33
|
+
client_mappings = ThreadSafe::Hash.new
|
34
|
+
continue_on_error = continue_on_error(params)
|
35
|
+
|
36
|
+
if continue_on_error
|
37
|
+
client = params.gdc_gd_client
|
38
|
+
domain_name = params.organization || params.domain
|
39
|
+
domain = client.domain(domain_name)
|
40
|
+
data_product = params.data_product
|
41
|
+
data_product_segments = domain.segments(:all, data_product)
|
42
|
+
|
43
|
+
if params.segments_filter
|
44
|
+
data_product_segments.select! do |segment|
|
45
|
+
params.segments_filter.include?(segment.segment_id)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
data_product_segments.pmap do |segment|
|
50
|
+
segment.clients.map do |segment_client|
|
51
|
+
project = segment_client.project
|
52
|
+
next unless project
|
53
|
+
|
54
|
+
project_mappings[project.pid.to_sym] = {
|
55
|
+
client_id: segment_client.client_id,
|
56
|
+
segment_id: segment.segment_id
|
57
|
+
}
|
58
|
+
client_mappings[segment_client.client_id.to_sym] = {
|
59
|
+
project_id: project.pid,
|
60
|
+
segment_id: segment.segment_id
|
61
|
+
}
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
{
|
67
|
+
params: {
|
68
|
+
collect_synced_status: continue_on_error,
|
69
|
+
sync_failed_list: {
|
70
|
+
project_client_mappings: project_mappings,
|
71
|
+
client_project_mappings: client_mappings,
|
72
|
+
failed_detailed_projects: [],
|
73
|
+
failed_projects: [],
|
74
|
+
failed_clients: [],
|
75
|
+
failed_segments: []
|
76
|
+
}
|
77
|
+
}
|
78
|
+
}
|
79
|
+
end
|
80
|
+
|
81
|
+
def print_result(_params)
|
82
|
+
false
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
@@ -19,6 +19,18 @@ module GoodData
|
|
19
19
|
|
20
20
|
description 'Synchronization Info'
|
21
21
|
param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
|
22
|
+
|
23
|
+
description 'Abort on error'
|
24
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
25
|
+
|
26
|
+
description 'Logger'
|
27
|
+
param :gdc_logger, instance_of(Type::GdLogger), required: false
|
28
|
+
|
29
|
+
description 'Collect synced status'
|
30
|
+
param :collect_synced_status, instance_of(Type::BooleanType), required: false
|
31
|
+
|
32
|
+
description 'Sync failed list'
|
33
|
+
param :sync_failed_list, instance_of(Type::HashType), required: false
|
22
34
|
end
|
23
35
|
|
24
36
|
RESULT_HEADER = %i[from to status]
|
@@ -27,6 +39,8 @@ module GoodData
|
|
27
39
|
def call(params)
|
28
40
|
results = []
|
29
41
|
params.synchronize.map do |segment_info|
|
42
|
+
next if sync_failed_segment(segment_info[:segment_id], params)
|
43
|
+
|
30
44
|
result = migrate_date_dimension(params, segment_info)
|
31
45
|
results.concat(result)
|
32
46
|
end
|
@@ -47,21 +61,32 @@ module GoodData
|
|
47
61
|
# check latest master and previous master
|
48
62
|
master_upgrade_datasets = get_upgrade_dates(latest_blueprint, previous_blueprint) if params[:synchronize_ldm].downcase == 'diff_against_master' && previous_blueprint
|
49
63
|
unless master_upgrade_datasets&.empty?
|
64
|
+
collect_synced_status = collect_synced_status(params)
|
65
|
+
failed_projects = ThreadSafe::Array.new
|
66
|
+
|
50
67
|
segment_info[:to].pmap do |entry|
|
51
68
|
pid = entry[:pid]
|
69
|
+
next if sync_failed_project(pid, params)
|
70
|
+
|
52
71
|
to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
|
72
|
+
GoodData.logger.info "Migrating date dimension, project: '#{to_project.title}', PID: #{pid}"
|
53
73
|
to_blueprint = to_project.blueprint
|
54
74
|
upgrade_datasets = get_upgrade_dates(latest_blueprint, to_blueprint)
|
55
75
|
next if upgrade_datasets.empty?
|
56
76
|
|
57
77
|
message = get_upgrade_message(upgrade_datasets)
|
78
|
+
failed_message = "Failed to migrate date dimension for project #{pid}"
|
79
|
+
update_status = to_project.upgrade_custom_v2(message)
|
80
|
+
process_failed_project(pid, failed_message, failed_projects, collect_synced_status) if collect_synced_status && update_status != 'OK'
|
58
81
|
|
59
82
|
results << {
|
60
83
|
from: segment_info[:from],
|
61
84
|
to: pid,
|
62
|
-
status:
|
85
|
+
status: update_status
|
63
86
|
}
|
64
87
|
end
|
88
|
+
|
89
|
+
process_failed_projects(failed_projects, short_name, params) if collect_synced_status
|
65
90
|
end
|
66
91
|
|
67
92
|
results
|
@@ -71,9 +96,9 @@ module GoodData
|
|
71
96
|
dest_dates = get_date_dimensions(dest_blueprint) if dest_blueprint
|
72
97
|
src_dates = get_date_dimensions(src_blueprint) if src_blueprint
|
73
98
|
|
74
|
-
return false if dest_dates.empty? || src_dates.empty?
|
75
|
-
|
76
99
|
upgrade_datasets = []
|
100
|
+
return upgrade_datasets if dest_dates.empty? || src_dates.empty?
|
101
|
+
|
77
102
|
dest_dates.each do |dest|
|
78
103
|
src_dim = get_date_dimension(src_blueprint, dest[:id])
|
79
104
|
next unless src_dim
|
@@ -103,8 +128,10 @@ module GoodData
|
|
103
128
|
get_date_dimensions(blueprint).any? { |e| e[:urn] == DATE_DIMENSION_CUSTOM_V2 }
|
104
129
|
end
|
105
130
|
|
131
|
+
# Get date dimension from blue print. Return nil if date dimension not existing
|
106
132
|
def get_date_dimension(blueprint, id)
|
107
|
-
|
133
|
+
date_dimensions = get_date_dimensions(blueprint)
|
134
|
+
date_dimensions.find { |d| d[:id] == id }
|
108
135
|
end
|
109
136
|
|
110
137
|
def get_date_dimensions(blueprint)
|
@@ -29,7 +29,16 @@ module GoodData
|
|
29
29
|
param :data_product, instance_of(Type::GDDataProductType), required: false
|
30
30
|
|
31
31
|
description 'Logger'
|
32
|
-
param :gdc_logger, instance_of(Type::GdLogger), required:
|
32
|
+
param :gdc_logger, instance_of(Type::GdLogger), required: false
|
33
|
+
|
34
|
+
description 'Abort on error'
|
35
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
36
|
+
|
37
|
+
description 'Collect synced status'
|
38
|
+
param :collect_synced_status, instance_of(Type::BooleanType), required: false
|
39
|
+
|
40
|
+
description 'Sync failed list'
|
41
|
+
param :sync_failed_list, instance_of(Type::HashType), required: false
|
33
42
|
end
|
34
43
|
|
35
44
|
RESULT_HEADER = [
|
@@ -45,6 +54,8 @@ module GoodData
|
|
45
54
|
synchronize_projects = []
|
46
55
|
data_product = params.data_product
|
47
56
|
client = params.gdc_gd_client
|
57
|
+
collect_synced_status = collect_synced_status(params)
|
58
|
+
continue_on_error = continue_on_error(params)
|
48
59
|
domain_name = params.organization || params.domain
|
49
60
|
fail "Either organisation or domain has to be specified in params" unless domain_name
|
50
61
|
domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
|
@@ -52,6 +63,8 @@ module GoodData
|
|
52
63
|
invalid_client_ids = []
|
53
64
|
begin
|
54
65
|
results = params.segments.map do |segment|
|
66
|
+
next if sync_failed_segment(segment.segment_id, params)
|
67
|
+
|
55
68
|
segment_object = domain.segments(segment.segment_id, data_product)
|
56
69
|
tmp = segment_object.provision_client_projects.map do |m|
|
57
70
|
Hash[m.each_pair.to_a].merge(type: :provision_result)
|
@@ -65,10 +78,21 @@ module GoodData
|
|
65
78
|
unless entry[:project_uri]
|
66
79
|
error_message = "There was error during provisioning clients: #{entry[:error]}" unless error_message
|
67
80
|
invalid_client_ids << entry[:id]
|
81
|
+
if collect_synced_status
|
82
|
+
failed_message = "Failed to provision client #{entry[:id]} in segment #{segment.segment_id}. Error: #{entry[:error]}"
|
83
|
+
add_failed_client(entry[:id], failed_message, short_name, params)
|
84
|
+
end
|
68
85
|
next
|
69
86
|
end
|
87
|
+
|
88
|
+
project_id = entry[:project_uri].split('/').last
|
89
|
+
if collect_synced_status && entry[:status] == 'CREATED' && entry[:id]
|
90
|
+
# Update project client mappings when there are create new clients during provision clients of the segment
|
91
|
+
add_new_clients_to_project_client_mapping(project_id, entry[:id], segment.segment_id, params)
|
92
|
+
end
|
93
|
+
|
70
94
|
{
|
71
|
-
pid:
|
95
|
+
pid: project_id,
|
72
96
|
client_id: entry[:id]
|
73
97
|
}
|
74
98
|
end.compact
|
@@ -93,15 +117,20 @@ module GoodData
|
|
93
117
|
end
|
94
118
|
|
95
119
|
params.gdc_logger.debug "Deleted clients: #{deleted_client_ids.join(', ')}"
|
96
|
-
|
120
|
+
unless error_message['TooManyProjectsCreatedException'] || error_message['Max number registered projects']
|
121
|
+
raise error_message unless continue_on_error
|
122
|
+
|
123
|
+
next
|
124
|
+
end
|
125
|
+
|
97
126
|
break tmp
|
98
127
|
end
|
99
128
|
|
100
129
|
tmp
|
101
130
|
end
|
102
131
|
rescue => e
|
103
|
-
params.gdc_logger.error "Problem occurs when provisioning clients."
|
104
|
-
raise e
|
132
|
+
params.gdc_logger.error "Problem occurs when provisioning clients. Error: #{e}"
|
133
|
+
raise e unless continue_on_error
|
105
134
|
end
|
106
135
|
|
107
136
|
results.flatten! if results
|
@@ -32,6 +32,15 @@ module GoodData
|
|
32
32
|
|
33
33
|
description 'Additional Hidden Parameters'
|
34
34
|
param :additional_hidden_params, instance_of(Type::HashType), required: false
|
35
|
+
|
36
|
+
description 'Abort on error'
|
37
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
38
|
+
|
39
|
+
description 'Collect synced status'
|
40
|
+
param :collect_synced_status, instance_of(Type::BooleanType), required: false
|
41
|
+
|
42
|
+
description 'Sync failed list'
|
43
|
+
param :sync_failed_list, instance_of(Type::HashType), required: false
|
35
44
|
end
|
36
45
|
|
37
46
|
class << self
|
@@ -46,6 +55,8 @@ module GoodData
|
|
46
55
|
return results unless include_ca
|
47
56
|
|
48
57
|
client = params.gdc_gd_client
|
58
|
+
collect_synced_status = collect_synced_status(params)
|
59
|
+
failed_projects = ThreadSafe::Array.new
|
49
60
|
|
50
61
|
params.synchronize.each do |info|
|
51
62
|
from = info.from
|
@@ -58,24 +69,33 @@ module GoodData
|
|
58
69
|
next unless ca_scripts
|
59
70
|
|
60
71
|
pid = entry[:pid]
|
72
|
+
next if sync_failed_project(pid, params)
|
73
|
+
|
61
74
|
ca_chunks = ca_scripts[:maqlDdlChunks]
|
62
|
-
to_project = client.projects(pid)
|
63
|
-
|
75
|
+
to_project = client.projects(pid)
|
76
|
+
unless to_project
|
77
|
+
process_failed_project(pid, "Invalid 'to' project specified - '#{pid}'", failed_projects, collect_synced_status)
|
78
|
+
next
|
79
|
+
end
|
64
80
|
|
81
|
+
params.gdc_logger.info "Synchronizing Computed Attributes to project: '#{to_project.title}', PID: #{pid}"
|
82
|
+
error_message = nil
|
65
83
|
begin
|
66
84
|
ca_chunks.each { |chunk| to_project.execute_maql(chunk) }
|
67
85
|
rescue => e
|
68
|
-
|
86
|
+
error_message = "Error occurred when executing MAQL for project: \"#{to_project.title}\" reason: \"#{e.message}\", chunks: #{ca_chunks.inspect}"
|
87
|
+
process_failed_project(pid, error_message, failed_projects, collect_synced_status)
|
69
88
|
end
|
70
89
|
|
71
90
|
results << {
|
72
91
|
from: from,
|
73
92
|
to: pid,
|
74
|
-
status: 'ok'
|
93
|
+
status: error_message.nil? ? 'ok' : 'failed'
|
75
94
|
}
|
76
95
|
end
|
77
96
|
end
|
78
97
|
|
98
|
+
process_failed_projects(failed_projects, short_name, params) if collect_synced_status
|
79
99
|
results
|
80
100
|
end
|
81
101
|
end
|
@@ -33,8 +33,23 @@ module GoodData
|
|
33
33
|
description 'ADS Client'
|
34
34
|
param :ads_client, instance_of(Type::AdsClientType), required: false
|
35
35
|
|
36
|
+
description 'Keep number of old master workspace excluding the latest one'
|
37
|
+
param :keep_only_previous_masters_count, instance_of(Type::StringType), required: false, default: '-1'
|
38
|
+
|
36
39
|
description 'Additional Hidden Parameters'
|
37
40
|
param :additional_hidden_params, instance_of(Type::HashType), required: false
|
41
|
+
|
42
|
+
description 'Abort on error'
|
43
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
44
|
+
|
45
|
+
description 'Logger'
|
46
|
+
param :gdc_logger, instance_of(Type::GdLogger), required: false
|
47
|
+
|
48
|
+
description 'Collect synced status'
|
49
|
+
param :collect_synced_status, instance_of(Type::BooleanType), required: false
|
50
|
+
|
51
|
+
description 'Sync failed list'
|
52
|
+
param :sync_failed_list, instance_of(Type::HashType), required: false
|
38
53
|
end
|
39
54
|
|
40
55
|
RESULT_HEADER = [
|
@@ -53,6 +68,7 @@ module GoodData
|
|
53
68
|
domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
|
54
69
|
data_product = params.data_product
|
55
70
|
domain_segments = domain.segments(:all, data_product)
|
71
|
+
keep_only_previous_masters_count = Integer(params.keep_only_previous_masters_count || "-1")
|
56
72
|
|
57
73
|
segments = params.segments.map do |seg|
|
58
74
|
domain_segments.find do |s|
|
@@ -61,19 +77,17 @@ module GoodData
|
|
61
77
|
end
|
62
78
|
|
63
79
|
results = segments.map do |segment|
|
80
|
+
next if sync_failed_segment(segment.segment_id, params)
|
81
|
+
|
64
82
|
if params.ads_client
|
65
|
-
|
66
|
-
params.release_table_name,
|
67
|
-
params.ads_client,
|
68
|
-
segment.segment_id
|
69
|
-
)
|
83
|
+
master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_ads(params.release_table_name, params.ads_client, segment.segment_id)
|
70
84
|
else
|
71
|
-
|
85
|
+
master_projects = GoodData::LCM2::Helpers.get_master_project_list_from_nfs(domain_name, data_product.data_product_id, segment.segment_id)
|
72
86
|
end
|
73
87
|
|
88
|
+
current_master = master_projects.last
|
74
89
|
# TODO: Check res.first.nil? || res.first[:master_project_id].nil?
|
75
90
|
master = client.projects(current_master[:master_project_id])
|
76
|
-
|
77
91
|
segment.master_project = master
|
78
92
|
segment.save
|
79
93
|
|
@@ -83,14 +97,26 @@ module GoodData
|
|
83
97
|
failed_count = sync_result['failedClients']['count']
|
84
98
|
|
85
99
|
if failed_count.to_i > 0
|
86
|
-
|
87
|
-
|
100
|
+
error_handle(segment, res, collect_synced_status(params), params)
|
101
|
+
end
|
102
|
+
|
103
|
+
if keep_only_previous_masters_count >= 0
|
104
|
+
number_of_deleted_projects = master_projects.count - (keep_only_previous_masters_count + 1)
|
105
|
+
|
106
|
+
if number_of_deleted_projects.positive?
|
107
|
+
begin
|
108
|
+
removal_master_project_ids = remove_multiple_workspace(params, segment.segment_id, master_projects, number_of_deleted_projects)
|
109
|
+
remove_old_workspaces_from_release_table(params, domain_name, data_product.data_product_id, segment.segment_id, master_projects, removal_master_project_ids)
|
110
|
+
rescue Exception => e # rubocop:disable RescueException
|
111
|
+
GoodData.logger.error "Problem occurs when removing old master workspace, reason: #{e.message}"
|
112
|
+
end
|
113
|
+
end
|
88
114
|
end
|
89
115
|
|
90
116
|
{
|
91
117
|
segment: segment.id,
|
92
|
-
master_pid: master.pid,
|
93
|
-
master_name: master.title,
|
118
|
+
master_pid: master.nil? ? '' : master.pid,
|
119
|
+
master_name: master.nil? ? '' : master.title,
|
94
120
|
successful_count: sync_result['successfulClients']['count']
|
95
121
|
}
|
96
122
|
end
|
@@ -98,6 +124,81 @@ module GoodData
|
|
98
124
|
# Return results
|
99
125
|
results
|
100
126
|
end
|
127
|
+
|
128
|
+
def remove_multiple_workspace(params, segment_id, master_projects, number_of_deleted_projects)
|
129
|
+
removal_master_project_ids = []
|
130
|
+
need_to_delete_projects = master_projects.take(number_of_deleted_projects)
|
131
|
+
|
132
|
+
need_to_delete_projects.each do |project_wrapper|
|
133
|
+
master_project_id = project_wrapper[:master_project_id]
|
134
|
+
next if master_project_id.to_s.empty?
|
135
|
+
|
136
|
+
begin
|
137
|
+
project = params.gdc_gd_client.projects(master_project_id)
|
138
|
+
if project && !%w[deleted archived].include?(project.state.to_s)
|
139
|
+
GoodData.logger.info "Segment #{segment_id}: Deleting old master workspace, project: '#{project.title}', PID: (#{project.pid})."
|
140
|
+
project.delete
|
141
|
+
end
|
142
|
+
removal_master_project_ids << master_project_id
|
143
|
+
master_projects.delete_if { |p| p[:master_project_id] == master_project_id }
|
144
|
+
rescue Exception => ex # rubocop:disable RescueException
|
145
|
+
GoodData.logger.error "Unable to remove master workspace: '#{master_project_id}', Error: #{ex.message}"
|
146
|
+
end
|
147
|
+
end
|
148
|
+
removal_master_project_ids
|
149
|
+
end
|
150
|
+
|
151
|
+
# rubocop:disable Metrics/ParameterLists
|
152
|
+
def remove_old_workspaces_from_release_table(params, domain_id, data_product_id, segment_id, master_projects, removal_master_project_ids)
|
153
|
+
unless removal_master_project_ids.empty?
|
154
|
+
if params.ads_client
|
155
|
+
GoodData::LCM2::Helpers.delete_master_project_from_ads(params.release_table_name, params.ads_client, segment_id, removal_master_project_ids)
|
156
|
+
else
|
157
|
+
data = master_projects.sort_by { |master| master[:version] }
|
158
|
+
GoodData::LCM2::Helpers.update_master_project_to_nfs(domain_id, data_product_id, segment_id, data)
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
162
|
+
# rubocop:enable Metrics/ParameterLists
|
163
|
+
|
164
|
+
private
|
165
|
+
|
166
|
+
def error_handle(segment, error_result, continue_on_error, params)
|
167
|
+
sync_result = error_result.json['synchronizationResult']
|
168
|
+
success_count = sync_result['successfulClients']['count'].to_i
|
169
|
+
failed_count = sync_result['failedClients']['count'].to_i
|
170
|
+
|
171
|
+
# Synchronize failure for all clients in segment
|
172
|
+
if continue_on_error && success_count.zero? && failed_count.positive?
|
173
|
+
segment_warning_message = "Failed to synchronize clients for #{segment.segment_id} segment. Details: #{sync_result['links']['details']}"
|
174
|
+
add_failed_segment(segment.segment_id, segment_warning_message, short_name, params)
|
175
|
+
return
|
176
|
+
end
|
177
|
+
|
178
|
+
summary_error_message = "#{failed_count} clients failed to synchronize. Details: #{sync_result['links']['details']}"
|
179
|
+
fail(summary_error_message) unless continue_on_error
|
180
|
+
|
181
|
+
GoodData.logger.warn summary_error_message
|
182
|
+
if error_result.details # rubocop:disable Style/SafeNavigation
|
183
|
+
error_result.details.items.each do |item|
|
184
|
+
next if item['status'] == 'OK'
|
185
|
+
|
186
|
+
error_client_id = item['id']
|
187
|
+
error_message = error_message(item, segment)
|
188
|
+
add_failed_client(error_client_id, error_message, short_name, params)
|
189
|
+
end
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
def error_message(error_item, segment)
|
194
|
+
error_client_id = error_item['id']
|
195
|
+
error_message = "Failed to synchronize #{error_client_id} client in #{segment.segment_id} segment."
|
196
|
+
error_message = "#{error_message}. Detail: #{error_item['error']['message']}" if error_item['error'] && error_item['error']['message']
|
197
|
+
|
198
|
+
error_message = "#{error_message}. Error items: #{error_item['error']['parameters']}" if error_item['error'] && error_item['error']['parameters']
|
199
|
+
|
200
|
+
error_message
|
201
|
+
end
|
101
202
|
end
|
102
203
|
end
|
103
204
|
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
require_relative 'base_action'
|
9
|
+
|
10
|
+
module GoodData
|
11
|
+
module LCM2
|
12
|
+
class SynchronizeDataSetMapping < BaseAction
|
13
|
+
DESCRIPTION = 'Synchronize Dataset Mappings'
|
14
|
+
|
15
|
+
PARAMS = define_params(self) do
|
16
|
+
description 'Client Used for Connecting to GD'
|
17
|
+
param :gdc_gd_client, instance_of(Type::GdClientType), required: true
|
18
|
+
|
19
|
+
description 'Client used to connecting to development domain'
|
20
|
+
param :development_client, instance_of(Type::GdClientType), required: true
|
21
|
+
|
22
|
+
description 'Synchronization Info'
|
23
|
+
param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
|
24
|
+
|
25
|
+
description 'Logger'
|
26
|
+
param :gdc_logger, instance_of(Type::GdLogger), required: true
|
27
|
+
|
28
|
+
description 'Abort on error'
|
29
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
30
|
+
|
31
|
+
description 'Collect synced status'
|
32
|
+
param :collect_synced_status, instance_of(Type::BooleanType), required: false
|
33
|
+
|
34
|
+
description 'Sync failed list'
|
35
|
+
param :sync_failed_list, instance_of(Type::HashType), required: false
|
36
|
+
end
|
37
|
+
|
38
|
+
RESULT_HEADER = %i[from to count status]
|
39
|
+
|
40
|
+
class << self
|
41
|
+
def call(params)
|
42
|
+
results = []
|
43
|
+
collect_synced_status = collect_synced_status(params)
|
44
|
+
failed_projects = ThreadSafe::Array.new
|
45
|
+
|
46
|
+
client = params.gdc_gd_client
|
47
|
+
development_client = params.development_client
|
48
|
+
|
49
|
+
params.synchronize.peach do |info|
|
50
|
+
from_project = info.from
|
51
|
+
to_projects = info.to
|
52
|
+
|
53
|
+
from = development_client.projects(from_project)
|
54
|
+
unless from
|
55
|
+
process_failed_project(from_project, "Invalid 'from' project specified - '#{from_project}'", failed_projects, collect_synced_status)
|
56
|
+
next
|
57
|
+
end
|
58
|
+
|
59
|
+
dataset_mapping = from.dataset_mapping
|
60
|
+
if dataset_mapping&.dig('datasetMappings', 'items').nil? || dataset_mapping['datasetMappings']['items'].empty?
|
61
|
+
params.gdc_logger.info "Project: '#{from.title}', PID: '#{from.pid}' has no model mapping, skip synchronizing model mapping."
|
62
|
+
else
|
63
|
+
to_projects.peach do |to|
|
64
|
+
pid = to[:pid]
|
65
|
+
next if sync_failed_project(pid, params)
|
66
|
+
|
67
|
+
to_project = client.projects(pid)
|
68
|
+
process_failed_project(pid, "Invalid 'to' project specified - '#{pid}'", failed_projects, collect_synced_status) unless to_project
|
69
|
+
|
70
|
+
message_to_project = "to project: '#{to_project.title}', PID: '#{to_project.pid}'"
|
71
|
+
params.gdc_logger.info "Transferring model mapping, from project: '#{from.title}', PID: '#{from.pid}', #{message_to_project}"
|
72
|
+
res = to_project.update_dataset_mapping(dataset_mapping)
|
73
|
+
res[:from] = from.pid
|
74
|
+
results << res
|
75
|
+
|
76
|
+
failed_message = "Failed to transfer model mapping from project '#{from.pid}' to project '#{to_project.pid}'"
|
77
|
+
process_failed_project(pid, failed_message, failed_projects, collect_synced_status) if collect_synced_status && res[:status] != 'OK'
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
process_failed_projects(failed_projects, short_name, params) if collect_synced_status
|
83
|
+
# Return results
|
84
|
+
results.flatten
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|