gooddata 2.2.0-java → 2.3.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gdc-ii-config.yaml +42 -1
- data/.github/workflows/build.yml +14 -13
- data/.github/workflows/pre-merge.yml +13 -13
- data/.pronto.yml +1 -0
- data/.rubocop.yml +2 -14
- data/CHANGELOG.md +9 -0
- data/Dockerfile +13 -7
- data/Dockerfile.jruby +5 -5
- data/Dockerfile.ruby +5 -7
- data/Gemfile +4 -2
- data/README.md +5 -4
- data/Rakefile +1 -1
- data/SDK_VERSION +1 -1
- data/VERSION +1 -1
- data/bin/run_brick.rb +7 -0
- data/ci/mysql/pom.xml +6 -1
- data/ci/redshift/pom.xml +3 -4
- data/docker-compose.lcm.yml +42 -1
- data/docker-compose.yml +42 -0
- data/gooddata.gemspec +21 -22
- data/lcm.rake +9 -0
- data/lib/gooddata/bricks/base_pipeline.rb +26 -0
- data/lib/gooddata/bricks/brick.rb +0 -1
- data/lib/gooddata/bricks/middleware/execution_result_middleware.rb +3 -3
- data/lib/gooddata/bricks/pipeline.rb +2 -14
- data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +18 -8
- data/lib/gooddata/cloud_resources/redshift/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/redshift/redshift_client.rb +0 -2
- data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +1 -1
- data/lib/gooddata/lcm/actions/base_action.rb +157 -0
- data/lib/gooddata/lcm/actions/collect_data_product.rb +2 -1
- data/lib/gooddata/lcm/actions/collect_projects_warning_status.rb +53 -0
- data/lib/gooddata/lcm/actions/collect_segment_clients.rb +14 -0
- data/lib/gooddata/lcm/actions/initialize_continue_on_error_option.rb +87 -0
- data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +28 -2
- data/lib/gooddata/lcm/actions/provision_clients.rb +34 -5
- data/lib/gooddata/lcm/actions/synchronize_cas.rb +24 -4
- data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -4
- data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +28 -3
- data/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb +48 -11
- data/lib/gooddata/lcm/actions/synchronize_kd_dashboard_permission.rb +103 -0
- data/lib/gooddata/lcm/actions/synchronize_ldm.rb +60 -15
- data/lib/gooddata/lcm/actions/synchronize_ldm_layout.rb +98 -0
- data/lib/gooddata/lcm/actions/synchronize_pp_dashboard_permission.rb +108 -0
- data/lib/gooddata/lcm/actions/synchronize_schedules.rb +31 -1
- data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +14 -9
- data/lib/gooddata/lcm/actions/synchronize_user_groups.rb +30 -4
- data/lib/gooddata/lcm/actions/synchronize_users.rb +11 -10
- data/lib/gooddata/lcm/actions/update_metric_formats.rb +21 -4
- data/lib/gooddata/lcm/exceptions/lcm_execution_warning.rb +15 -0
- data/lib/gooddata/lcm/helpers/check_helper.rb +19 -0
- data/lib/gooddata/lcm/lcm2.rb +45 -4
- data/lib/gooddata/lcm/user_bricks_helper.rb +9 -0
- data/lib/gooddata/mixins/inspector.rb +1 -1
- data/lib/gooddata/models/ldm_layout.rb +38 -0
- data/lib/gooddata/models/project.rb +197 -22
- data/lib/gooddata/models/project_creator.rb +83 -6
- data/lib/gooddata/models/segment.rb +2 -1
- data/lib/gooddata/models/user_filters/user_filter_builder.rb +104 -15
- data/lib/gooddata/rest/connection.rb +5 -3
- data/lib/gooddata/rest/phmap.rb +1 -0
- data/lib/gooddata.rb +1 -0
- data/lib/gooddata_brick_base.rb +35 -0
- data/sonar-project.properties +6 -0
- metadata +60 -55
- data/lib/gooddata/cloud_resources/redshift/drivers/log4j.properties +0 -15
data/gooddata.gemspec
CHANGED
@@ -1,13 +1,11 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
$LOAD_PATH.push File.expand_path('../lib/', __FILE__)
|
3
3
|
require 'gooddata/version'
|
4
|
-
|
5
4
|
Gem::Specification.new do |s|
|
6
5
|
s.name = 'gooddata'
|
7
6
|
s.version = GoodData::VERSION
|
8
7
|
s.licenses = ['BSD-3-Clause']
|
9
8
|
s.platform = 'java' if RUBY_PLATFORM =~ /java/
|
10
|
-
|
11
9
|
s.required_rubygems_version = Gem::Requirement.new('>= 0') if s.respond_to? :required_rubygems_version=
|
12
10
|
s.authors = [
|
13
11
|
'Pavel Kolesnikov',
|
@@ -18,27 +16,27 @@ Gem::Specification.new do |s|
|
|
18
16
|
'Petr Gadorek',
|
19
17
|
'Jakub Mahnert'
|
20
18
|
]
|
21
|
-
|
22
19
|
s.summary = 'A convenient Ruby wrapper around the GoodData RESTful API'
|
23
20
|
s.description = 'Use the GoodData::Client class to integrate GoodData into your own application or use the CLI to work with GoodData directly from the command line.'
|
24
21
|
s.email = 'lcm@gooddata.com'
|
25
22
|
s.extra_rdoc_files = %w(LICENSE README.md)
|
26
|
-
|
27
23
|
s.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR).map { |f| f unless %w(NOTICES.txt LICENSE_FOR_RUBY_SDK_COMPONENT.txt).include?(f) }
|
28
24
|
s.files.reject! { |fn| fn.start_with? 'spec/' }
|
29
25
|
s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
|
30
|
-
|
31
26
|
s.homepage = 'http://github.com/gooddata/gooddata-ruby'
|
32
27
|
s.require_paths = ['lib']
|
33
|
-
|
34
28
|
# s.add_development_dependency 'bundler', "~> 1.14"
|
35
29
|
s.add_development_dependency 'license_finder', '~> 2.0'
|
36
|
-
s.add_development_dependency 'rake', '~>
|
30
|
+
s.add_development_dependency 'rake', '~> 13.0'
|
37
31
|
s.add_development_dependency 'redcarpet', '~> 3.1' if RUBY_PLATFORM != 'java'
|
38
|
-
s.add_development_dependency 'rspec', '~> 3.
|
39
|
-
s.add_development_dependency 'rspec-expectations', '~> 3.
|
40
|
-
s.add_development_dependency 'rspec_junit_formatter', '~> 0.
|
41
|
-
|
32
|
+
s.add_development_dependency 'rspec', '~> 3.12.0'
|
33
|
+
s.add_development_dependency 'rspec-expectations', '~> 3.12'
|
34
|
+
s.add_development_dependency 'rspec_junit_formatter', '~> 0.6.0'
|
35
|
+
if RUBY_VERSION >= '2.6'
|
36
|
+
s.add_development_dependency 'rubocop', '>= 1.28'
|
37
|
+
else
|
38
|
+
s.add_development_dependency 'rubocop', '~> 0.81'
|
39
|
+
end
|
42
40
|
s.add_development_dependency 'simplecov', '~> 0.12'
|
43
41
|
s.add_development_dependency 'webmock', '~> 2.3.1'
|
44
42
|
s.add_development_dependency 'yard', '~> 0.9.11'
|
@@ -46,16 +44,16 @@ Gem::Specification.new do |s|
|
|
46
44
|
s.add_development_dependency 'pry'
|
47
45
|
s.add_development_dependency 'pry-byebug', '~> 3.6' if RUBY_PLATFORM != 'java'
|
48
46
|
|
49
|
-
s.add_development_dependency 'pronto', '
|
50
|
-
s.add_development_dependency 'pronto-rubocop', '
|
51
|
-
s.add_development_dependency 'pronto-reek', '
|
47
|
+
s.add_development_dependency 'pronto', '>= 0.10' if RUBY_PLATFORM != 'java'
|
48
|
+
s.add_development_dependency 'pronto-rubocop', '>= 0.9' if RUBY_PLATFORM != 'java'
|
49
|
+
s.add_development_dependency 'pronto-reek', '>= 0.9' if RUBY_PLATFORM != 'java'
|
52
50
|
s.add_development_dependency 'vcr', '5.0.0'
|
53
51
|
s.add_development_dependency 'hashdiff', '~> 0.4'
|
54
52
|
|
55
53
|
s.add_development_dependency 'sqlite3' if RUBY_PLATFORM != 'java'
|
56
54
|
|
57
|
-
|
58
|
-
|
55
|
+
if RUBY_VERSION >= '2.5'
|
56
|
+
s.add_dependency 'activesupport', '< 7.0.0'
|
59
57
|
else
|
60
58
|
s.add_dependency 'activesupport', '>= 5.2.4.3', '< 6.0'
|
61
59
|
end
|
@@ -66,19 +64,20 @@ Gem::Specification.new do |s|
|
|
66
64
|
else
|
67
65
|
s.add_dependency 'docile', '> 1.1', '< 1.4.0'
|
68
66
|
end
|
69
|
-
s.add_dependency 'azure-storage-blob', '~>
|
70
|
-
s.add_dependency 'nokogiri', '~> 1.10.
|
67
|
+
s.add_dependency 'azure-storage-blob', '~> 2.0'
|
68
|
+
s.add_dependency 'nokogiri', '~> 1', '>= 1.10.8'
|
71
69
|
s.add_dependency 'gli', '~> 2.15'
|
72
|
-
s.add_dependency 'gooddata_datawarehouse', '~> 0.0.
|
70
|
+
s.add_dependency 'gooddata_datawarehouse', '~> 0.0.11' if RUBY_PLATFORM == 'java'
|
73
71
|
s.add_dependency 'highline', '= 2.0.0.pre.develop.14'
|
74
|
-
s.add_dependency 'json_pure', '~>
|
72
|
+
s.add_dependency 'json_pure', '~> 2.6'
|
75
73
|
s.add_dependency 'multi_json', '~> 1.12'
|
76
74
|
s.add_dependency 'parseconfig', '~> 1.0'
|
77
75
|
s.add_dependency 'pmap', '~> 1.1'
|
76
|
+
s.add_dependency 'sequel', '< 5.72.0'
|
78
77
|
s.add_dependency 'remote_syslog_logger', '~> 1.0.3'
|
79
|
-
s.add_dependency 'restforce', '>= 2.4'
|
78
|
+
s.add_dependency 'restforce', '>= 2.4'
|
80
79
|
s.add_dependency 'rest-client', '~> 2.0'
|
81
|
-
s.add_dependency 'rubyzip'
|
80
|
+
s.add_dependency 'rubyzip'
|
82
81
|
s.add_dependency 'terminal-table', '~> 1.7'
|
83
82
|
s.add_dependency 'thread_safe'
|
84
83
|
s.add_dependency 'backports'
|
data/lcm.rake
CHANGED
@@ -138,6 +138,15 @@ namespace :maven do
|
|
138
138
|
|
139
139
|
system('mvn -f ci/redshift/pom.xml clean install -P binary-packaging')
|
140
140
|
system('cp -rf ci/redshift/target/*.jar lib/gooddata/cloud_resources/redshift/drivers/')
|
141
|
+
|
142
|
+
system('mvn -f ci/postgresql/pom.xml clean install -P binary-packaging')
|
143
|
+
system('cp -rf ci/postgresql/target/*.jar lib/gooddata/cloud_resources/postgresql/drivers/')
|
144
|
+
|
145
|
+
system('mvn -f ci/mssql/pom.xml clean install -P binary-packaging')
|
146
|
+
system('cp -rf ci/mssql/target/*.jar lib/gooddata/cloud_resources/mssql/drivers/')
|
147
|
+
|
148
|
+
system('mvn -f ci/mysql/pom.xml clean install -P binary-packaging')
|
149
|
+
system('cp -rf ci/mysql/target/*.jar lib/gooddata/cloud_resources/mysql/drivers/')
|
141
150
|
end
|
142
151
|
end
|
143
152
|
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2010-2022 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
module GoodData
|
9
|
+
module Bricks
|
10
|
+
class BasePipeline
|
11
|
+
# Pipeline preparation code
|
12
|
+
def self.prepare(pipeline)
|
13
|
+
pipeline.reverse.reduce(nil) do |memo, app|
|
14
|
+
if memo.nil?
|
15
|
+
app.respond_to?(:new) ? app.new : app
|
16
|
+
elsif app.respond_to?(:new)
|
17
|
+
app.new(app: memo)
|
18
|
+
else
|
19
|
+
app.app = memo
|
20
|
+
app
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -39,7 +39,7 @@ module GoodData
|
|
39
39
|
message: message
|
40
40
|
}
|
41
41
|
}
|
42
|
-
update_result(result)
|
42
|
+
update_result(result, status)
|
43
43
|
end
|
44
44
|
|
45
45
|
private
|
@@ -52,9 +52,9 @@ module GoodData
|
|
52
52
|
@@result_log_path = result_log_path
|
53
53
|
end
|
54
54
|
|
55
|
-
def self.update_result(result)
|
55
|
+
def self.update_result(result, status = ExecutionStatus::ERROR)
|
56
56
|
if @@result_log_path.nil?
|
57
|
-
GoodData.gd_logger
|
57
|
+
GoodData.gd_logger&.warn("action=update_execution_result status=#{status} Not found execution result logger file.")
|
58
58
|
return
|
59
59
|
end
|
60
60
|
|
@@ -10,23 +10,11 @@ require_relative 'release_brick'
|
|
10
10
|
require_relative 'provisioning_brick'
|
11
11
|
require_relative 'rollout_brick'
|
12
12
|
require_relative 'hello_world_brick'
|
13
|
+
require_relative 'base_pipeline'
|
13
14
|
|
14
15
|
module GoodData
|
15
16
|
module Bricks
|
16
|
-
class Pipeline
|
17
|
-
# Pipeline preparation code
|
18
|
-
def self.prepare(pipeline)
|
19
|
-
pipeline.reverse.reduce(nil) do |memo, app|
|
20
|
-
if memo.nil?
|
21
|
-
app.respond_to?(:new) ? app.new : app
|
22
|
-
elsif app.respond_to?(:new)
|
23
|
-
app.new(app: memo)
|
24
|
-
else
|
25
|
-
app.app = memo
|
26
|
-
app
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
17
|
+
class Pipeline < GoodData::Bricks::BasePipeline
|
30
18
|
|
31
19
|
def self.users_brick_pipeline
|
32
20
|
prepare([
|
@@ -21,10 +21,12 @@ module GoodData
|
|
21
21
|
JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
|
22
22
|
MYSQL_DEFAULT_PORT = 3306
|
23
23
|
JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://'
|
24
|
-
VERIFY_FULL = '
|
25
|
-
PREFER = '
|
26
|
-
REQUIRE = '
|
24
|
+
VERIFY_FULL = '&useSSL=true&verifyServerCertificate=true'
|
25
|
+
PREFER = '&useSSL=true&requireSSL=false&verifyServerCertificate=false'
|
26
|
+
REQUIRE = '&useSSL=true&requireSSL=true&verifyServerCertificate=false'
|
27
27
|
MYSQL_FETCH_SIZE = 1000
|
28
|
+
MONGO_BI_FETCH_SIZE = 1
|
29
|
+
MONGO_BI_TYPE = 'MongoDBConnector'
|
28
30
|
|
29
31
|
class << self
|
30
32
|
def accept?(type)
|
@@ -39,6 +41,7 @@ module GoodData
|
|
39
41
|
@database = options['mysql_client']['connection']['database']
|
40
42
|
@authentication = options['mysql_client']['connection']['authentication']
|
41
43
|
@ssl_mode = options['mysql_client']['connection']['sslMode']
|
44
|
+
@database_type = options['mysql_client']['connection']['databaseType']
|
42
45
|
raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
|
43
46
|
|
44
47
|
@url = build_url(options['mysql_client']['connection']['url'])
|
@@ -46,7 +49,7 @@ module GoodData
|
|
46
49
|
raise('Missing connection info for Mysql client')
|
47
50
|
end
|
48
51
|
|
49
|
-
Java.com.mysql.
|
52
|
+
Java.com.mysql.jdbc.Driver
|
50
53
|
end
|
51
54
|
|
52
55
|
def realize_query(query, _params)
|
@@ -56,7 +59,7 @@ module GoodData
|
|
56
59
|
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
57
60
|
measure = Benchmark.measure do
|
58
61
|
statement = @connection.create_statement
|
59
|
-
statement.set_fetch_size(
|
62
|
+
statement.set_fetch_size(fetch_size)
|
60
63
|
has_result = statement.execute(query)
|
61
64
|
if has_result
|
62
65
|
result = statement.get_result_set
|
@@ -76,12 +79,11 @@ module GoodData
|
|
76
79
|
end
|
77
80
|
|
78
81
|
def connect
|
79
|
-
GoodData.logger.info "Setting up connection to Mysql #{@url}"
|
82
|
+
GoodData.logger.info "Setting up connection to Mysql #{@database_type} #{@url} "
|
80
83
|
|
81
84
|
prop = java.util.Properties.new
|
82
85
|
prop.setProperty('user', @authentication['basic']['userName'])
|
83
86
|
prop.setProperty('password', @authentication['basic']['password'])
|
84
|
-
|
85
87
|
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
86
88
|
@connection.set_auto_commit(false)
|
87
89
|
end
|
@@ -93,7 +95,11 @@ module GoodData
|
|
93
95
|
host = matches[0][0]
|
94
96
|
port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT
|
95
97
|
|
96
|
-
"#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}
|
98
|
+
"#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?#{get_ssl_mode(@ssl_mode)}#{add_extended}&useCursorFetch=true&enabledTLSProtocols=TLSv1.2"
|
99
|
+
end
|
100
|
+
|
101
|
+
def fetch_size
|
102
|
+
@database_type == MONGO_BI_TYPE ? MONGO_BI_FETCH_SIZE : MYSQL_FETCH_SIZE
|
97
103
|
end
|
98
104
|
|
99
105
|
def get_ssl_mode(ssl_mode)
|
@@ -106,6 +112,10 @@ module GoodData
|
|
106
112
|
|
107
113
|
mode
|
108
114
|
end
|
115
|
+
|
116
|
+
def add_extended
|
117
|
+
@database_type == MONGO_BI_TYPE ? '&authenticationPlugins=org.mongodb.mongosql.auth.plugin.MongoSqlAuthenticationPlugin&useLocalTransactionState=true' : ''
|
118
|
+
end
|
109
119
|
end
|
110
120
|
end
|
111
121
|
end
|
File without changes
|
@@ -39,8 +39,6 @@ module GoodData
|
|
39
39
|
@debug = options['debug'] == true || options['debug'] == 'true'
|
40
40
|
|
41
41
|
Java.com.amazon.redshift.jdbc42.Driver
|
42
|
-
base = Pathname(__FILE__).dirname
|
43
|
-
org.apache.log4j.PropertyConfigurator.configure("#{base}/drivers/log4j.properties")
|
44
42
|
end
|
45
43
|
|
46
44
|
def realize_query(query, _params)
|
@@ -18,7 +18,7 @@ end
|
|
18
18
|
module GoodData
|
19
19
|
module CloudResources
|
20
20
|
class SnowflakeClient < CloudResourceClient
|
21
|
-
SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=
|
21
|
+
SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=gooddata_platform'
|
22
22
|
SNOWFLAKE_SEPARATOR_PARAM = '?'
|
23
23
|
|
24
24
|
class << self
|
@@ -18,6 +18,11 @@ module GoodData
|
|
18
18
|
class << self
|
19
19
|
include Dsl::Dsl
|
20
20
|
|
21
|
+
SYNC_FAILED_LIST = 'sync_failed_list'.to_sym
|
22
|
+
FAILED_PROJECTS = 'failed_projects'.to_sym
|
23
|
+
FAILED_CLIENTS = 'failed_clients'.to_sym
|
24
|
+
FAILED_SEGMENTS = 'failed_segments'.to_sym
|
25
|
+
|
21
26
|
def check_params(specification, params)
|
22
27
|
Helpers.check_params(specification, params)
|
23
28
|
end
|
@@ -31,6 +36,158 @@ module GoodData
|
|
31
36
|
params.setup_filters(specification) # enables params validation
|
32
37
|
result
|
33
38
|
end
|
39
|
+
|
40
|
+
def print_result(_params)
|
41
|
+
true
|
42
|
+
end
|
43
|
+
|
44
|
+
def continue_on_error(params)
|
45
|
+
Helpers.continue_on_error(params)
|
46
|
+
end
|
47
|
+
|
48
|
+
def collect_synced_status(params)
|
49
|
+
Helpers.collect_synced_status(params)
|
50
|
+
end
|
51
|
+
|
52
|
+
def add_failed_project(project_id, message, failed_action, params)
|
53
|
+
if collect_synced_status(params) && !sync_failed_project(project_id, params)
|
54
|
+
sync_failed_list = sync_failed_list(params)
|
55
|
+
project_client_mappings = sync_failed_list[:project_client_mappings]
|
56
|
+
project_client_mapping = project_client_mappings ? project_client_mappings[project_id.to_sym] : nil
|
57
|
+
client_id = project_client_mapping ? project_client_mapping[:client_id] : nil
|
58
|
+
segment_id = project_client_mapping ? project_client_mapping[:segment_id] : nil
|
59
|
+
|
60
|
+
failed_detailed_project = {
|
61
|
+
project_id: project_id,
|
62
|
+
client_id: client_id,
|
63
|
+
segment: segment_id,
|
64
|
+
message: message,
|
65
|
+
action: failed_action
|
66
|
+
}
|
67
|
+
add_failed_detail(params, failed_detailed_project, sync_failed_list)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def add_failed_client(client_id, message, error_action, params)
|
72
|
+
if collect_synced_status(params) && !sync_failed_client(client_id, params)
|
73
|
+
sync_failed_list = sync_failed_list(params)
|
74
|
+
client_project_mappings = sync_failed_list[:client_project_mappings]
|
75
|
+
client_project_mapping = client_project_mappings ? client_project_mappings[client_id.to_sym] : nil
|
76
|
+
project_id = client_project_mapping ? client_project_mapping[:project_id] : nil
|
77
|
+
segment_id = client_project_mapping ? client_project_mapping[:segment_id] : nil
|
78
|
+
|
79
|
+
failed_detailed_client = {
|
80
|
+
project_id: project_id,
|
81
|
+
client_id: client_id,
|
82
|
+
segment: segment_id,
|
83
|
+
message: message,
|
84
|
+
action: error_action
|
85
|
+
}
|
86
|
+
add_failed_detail(params, failed_detailed_client, sync_failed_list)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def add_failed_segment(segment_id, message, error_action, params)
|
91
|
+
if collect_synced_status(params) && !sync_failed_segment(segment_id, params)
|
92
|
+
sync_failed_list = sync_failed_list(params)
|
93
|
+
failed_detailed_segment = {
|
94
|
+
project_id: nil,
|
95
|
+
client_id: nil,
|
96
|
+
segment: segment_id,
|
97
|
+
message: message,
|
98
|
+
action: error_action
|
99
|
+
}
|
100
|
+
add_failed_detail(params, failed_detailed_segment, sync_failed_list, true)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
# Add new clients to project client mapping
|
105
|
+
#
|
106
|
+
# @param [String] project_id project identify will be added to mapping
|
107
|
+
# @param [String] client_id client identify will be added to mapping
|
108
|
+
# @param [String] segment_id segment identify will be added to mapping
|
109
|
+
# @param [Hash] params the hash contains list of parameters and values
|
110
|
+
def add_new_clients_to_project_client_mapping(project_id, client_id, segment_id, params)
|
111
|
+
if collect_synced_status(params)
|
112
|
+
sync_failed_list = sync_failed_list(params)
|
113
|
+
client_project_mappings = sync_failed_list[:client_project_mappings]
|
114
|
+
project_client_mappings = sync_failed_list[:project_client_mappings]
|
115
|
+
client_project_mappings[client_id.to_sym] = {
|
116
|
+
project_id: project_id,
|
117
|
+
segment_id: segment_id
|
118
|
+
} if client_project_mappings
|
119
|
+
|
120
|
+
project_client_mappings[project_id.to_sym] = {
|
121
|
+
client_id: client_id,
|
122
|
+
segment_id: segment_id
|
123
|
+
} if project_client_mappings
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def process_failed_project(project_id, failed_message, failed_projects, continue_on_error)
|
128
|
+
fail(failed_message) unless continue_on_error
|
129
|
+
|
130
|
+
failed_projects << {
|
131
|
+
project_id: project_id,
|
132
|
+
message: failed_message
|
133
|
+
}
|
134
|
+
end
|
135
|
+
|
136
|
+
def process_failed_projects(failed_projects, failed_action, params)
|
137
|
+
failed_projects.each do |failed_project|
|
138
|
+
add_failed_project(failed_project[:project_id], failed_project[:message], failed_action, params)
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def sync_failed_project(project_id, params)
|
143
|
+
collect_synced_status(params) && params[SYNC_FAILED_LIST][FAILED_PROJECTS].include?(project_id)
|
144
|
+
end
|
145
|
+
|
146
|
+
def sync_failed_client(client_id, params)
|
147
|
+
collect_synced_status(params) && params[SYNC_FAILED_LIST][FAILED_CLIENTS].include?(client_id)
|
148
|
+
end
|
149
|
+
|
150
|
+
def sync_failed_segment(segment_id, params)
|
151
|
+
collect_synced_status(params) && params[SYNC_FAILED_LIST][FAILED_SEGMENTS].include?(segment_id)
|
152
|
+
end
|
153
|
+
|
154
|
+
private
|
155
|
+
|
156
|
+
def add_failed_detail(params, failed_detailed_project, sync_failed_list, ignore_segment = false)
|
157
|
+
params.gdc_logger&.warn failed_detailed_project[:message]
|
158
|
+
sync_failed_list[:failed_detailed_projects] << failed_detailed_project
|
159
|
+
|
160
|
+
if ignore_segment
|
161
|
+
add_failed_detail_segment(failed_detailed_project[:segment_id], sync_failed_list)
|
162
|
+
else
|
163
|
+
add_failed_detail_client(failed_detailed_project[:client_id], failed_detailed_project[:project_id], sync_failed_list)
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
def sync_failed_list(params)
|
168
|
+
if params.include?(SYNC_FAILED_LIST)
|
169
|
+
params[SYNC_FAILED_LIST]
|
170
|
+
else
|
171
|
+
nil
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
def add_failed_detail_client(client_id, project_id, sync_failed_list)
|
176
|
+
sync_failed_list[FAILED_CLIENTS] << client_id if client_id
|
177
|
+
|
178
|
+
sync_failed_list[FAILED_PROJECTS] << project_id if project_id
|
179
|
+
end
|
180
|
+
|
181
|
+
def add_failed_detail_segment(segment_id, sync_failed_list)
|
182
|
+
if segment_id
|
183
|
+
sync_failed_list[FAILED_SEGMENTS] << segment_id
|
184
|
+
|
185
|
+
client_project_mappings = sync_failed_list[:client_project_mappings]
|
186
|
+
client_project_mappings.each do |client_id, client_project_mapping|
|
187
|
+
add_failed_detail_client(client_id, client_project_mapping[:project_id], sync_failed_list) if client_project_mapping[:segment_id] == segment_id
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
34
191
|
end
|
35
192
|
end
|
36
193
|
end
|
@@ -47,8 +47,9 @@ module GoodData
|
|
47
47
|
end
|
48
48
|
|
49
49
|
begin
|
50
|
+
params.gdc_logger.info "Starting to find DataProduct ID: #{data_product_id}"
|
50
51
|
data_product = domain.data_products(data_product_id)
|
51
|
-
rescue RestClient::BadRequest
|
52
|
+
rescue RestClient::BadRequest, RestClient::NotFound
|
52
53
|
params.gdc_logger.info "Can not find DataProduct #{params.data_product}, creating it instead"
|
53
54
|
data_product = domain.create_data_product(id: params.data_product)
|
54
55
|
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# (C) 2019-2022 GoodData Corporation
|
3
|
+
require_relative 'base_action'
|
4
|
+
|
5
|
+
module GoodData
|
6
|
+
module LCM2
|
7
|
+
class CollectProjectsWarningStatus < BaseAction
|
8
|
+
DESCRIPTION = 'Collect synced projects status'
|
9
|
+
|
10
|
+
PARAMS = define_params(self) do
|
11
|
+
description 'Abort on error'
|
12
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
13
|
+
|
14
|
+
description 'Logger'
|
15
|
+
param :gdc_logger, instance_of(Type::GdLogger), required: false
|
16
|
+
|
17
|
+
description 'Collect synced status'
|
18
|
+
param :collect_synced_status, instance_of(Type::BooleanType), required: false
|
19
|
+
|
20
|
+
description 'Sync failed list'
|
21
|
+
param :sync_failed_list, instance_of(Type::HashType), required: false
|
22
|
+
end
|
23
|
+
|
24
|
+
RESULT_HEADER = %i[segment client project_pid failed_action]
|
25
|
+
|
26
|
+
class << self
|
27
|
+
def call(params)
|
28
|
+
results = []
|
29
|
+
return results unless collect_synced_status(params)
|
30
|
+
|
31
|
+
sync_failed_list = params[SYNC_FAILED_LIST]
|
32
|
+
if sync_failed_list
|
33
|
+
failed_detailed_projects = sync_failed_list[:failed_detailed_projects]
|
34
|
+
failed_detailed_projects.each do |failed_detailed_project|
|
35
|
+
results << {
|
36
|
+
segment: failed_detailed_project[:segment],
|
37
|
+
client: failed_detailed_project[:client_id],
|
38
|
+
project_pid: failed_detailed_project[:project_id],
|
39
|
+
failed_action: failed_detailed_project[:action]
|
40
|
+
}
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
results
|
45
|
+
end
|
46
|
+
|
47
|
+
def print_result(params)
|
48
|
+
collect_synced_status(params)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -32,9 +32,22 @@ module GoodData
|
|
32
32
|
|
33
33
|
description 'Domain'
|
34
34
|
param :domain, instance_of(Type::StringType), required: false
|
35
|
+
|
36
|
+
description 'Abort on error'
|
37
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
38
|
+
|
39
|
+
description 'Logger'
|
40
|
+
param :gdc_logger, instance_of(Type::GdLogger), required: false
|
41
|
+
|
42
|
+
description 'Collect synced status'
|
43
|
+
param :collect_synced_status, instance_of(Type::BooleanType), required: false
|
44
|
+
|
45
|
+
description 'Sync failed list'
|
46
|
+
param :sync_failed_list, instance_of(Type::HashType), required: false
|
35
47
|
end
|
36
48
|
|
37
49
|
RESULT_HEADER = [
|
50
|
+
:segment,
|
38
51
|
:from_name,
|
39
52
|
:from_pid,
|
40
53
|
:to_name,
|
@@ -89,6 +102,7 @@ module GoodData
|
|
89
102
|
client_project = segment_client.project
|
90
103
|
to_pid = client_project.pid
|
91
104
|
results << {
|
105
|
+
segment: segment.segment_id,
|
92
106
|
from_name: master_name,
|
93
107
|
from_pid: master_pid,
|
94
108
|
to_name: client_project.title,
|
@@ -0,0 +1,87 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# (C) 2019-2022 GoodData Corporation
|
3
|
+
require_relative 'base_action'
|
4
|
+
|
5
|
+
module GoodData
|
6
|
+
module LCM2
|
7
|
+
class InitializeContinueOnErrorOption < BaseAction
|
8
|
+
DESCRIPTION = 'Initialize continue on error option'
|
9
|
+
|
10
|
+
PARAMS = define_params(self) do
|
11
|
+
description 'Organization Name'
|
12
|
+
param :organization, instance_of(Type::StringType), required: false
|
13
|
+
|
14
|
+
description 'Domain'
|
15
|
+
param :domain, instance_of(Type::StringType), required: false
|
16
|
+
|
17
|
+
description 'DataProduct'
|
18
|
+
param :data_product, instance_of(Type::GDDataProductType), required: false
|
19
|
+
|
20
|
+
description 'Restricts synchronization to specified segments'
|
21
|
+
param :segments_filter, array_of(instance_of(Type::StringType)), required: false
|
22
|
+
|
23
|
+
description 'Abort on error'
|
24
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
25
|
+
|
26
|
+
description 'Client Used for Connecting to GD'
|
27
|
+
param :gdc_gd_client, instance_of(Type::GdClientType), required: true
|
28
|
+
end
|
29
|
+
|
30
|
+
class << self
|
31
|
+
def call(params)
|
32
|
+
project_mappings = ThreadSafe::Hash.new
|
33
|
+
client_mappings = ThreadSafe::Hash.new
|
34
|
+
continue_on_error = continue_on_error(params)
|
35
|
+
|
36
|
+
if continue_on_error
|
37
|
+
client = params.gdc_gd_client
|
38
|
+
domain_name = params.organization || params.domain
|
39
|
+
domain = client.domain(domain_name)
|
40
|
+
data_product = params.data_product
|
41
|
+
data_product_segments = domain.segments(:all, data_product)
|
42
|
+
|
43
|
+
if params.segments_filter
|
44
|
+
data_product_segments.select! do |segment|
|
45
|
+
params.segments_filter.include?(segment.segment_id)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
data_product_segments.pmap do |segment|
|
50
|
+
segment.clients.map do |segment_client|
|
51
|
+
project = segment_client.project
|
52
|
+
next unless project
|
53
|
+
|
54
|
+
project_mappings[project.pid.to_sym] = {
|
55
|
+
client_id: segment_client.client_id,
|
56
|
+
segment_id: segment.segment_id
|
57
|
+
}
|
58
|
+
client_mappings[segment_client.client_id.to_sym] = {
|
59
|
+
project_id: project.pid,
|
60
|
+
segment_id: segment.segment_id
|
61
|
+
}
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
{
|
67
|
+
params: {
|
68
|
+
collect_synced_status: continue_on_error,
|
69
|
+
sync_failed_list: {
|
70
|
+
project_client_mappings: project_mappings,
|
71
|
+
client_project_mappings: client_mappings,
|
72
|
+
failed_detailed_projects: [],
|
73
|
+
failed_projects: [],
|
74
|
+
failed_clients: [],
|
75
|
+
failed_segments: []
|
76
|
+
}
|
77
|
+
}
|
78
|
+
}
|
79
|
+
end
|
80
|
+
|
81
|
+
def print_result(_params)
|
82
|
+
false
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|