gooddata 2.2.0-java → 2.3.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gdc-ii-config.yaml +42 -1
- data/.github/workflows/build.yml +14 -13
- data/.github/workflows/pre-merge.yml +13 -13
- data/.pronto.yml +1 -0
- data/.rubocop.yml +2 -14
- data/CHANGELOG.md +9 -0
- data/Dockerfile +13 -7
- data/Dockerfile.jruby +5 -5
- data/Dockerfile.ruby +5 -7
- data/Gemfile +4 -2
- data/README.md +5 -4
- data/Rakefile +1 -1
- data/SDK_VERSION +1 -1
- data/VERSION +1 -1
- data/bin/run_brick.rb +7 -0
- data/ci/mysql/pom.xml +6 -1
- data/ci/redshift/pom.xml +3 -4
- data/docker-compose.lcm.yml +42 -1
- data/docker-compose.yml +42 -0
- data/gooddata.gemspec +21 -22
- data/lcm.rake +9 -0
- data/lib/gooddata/bricks/base_pipeline.rb +26 -0
- data/lib/gooddata/bricks/brick.rb +0 -1
- data/lib/gooddata/bricks/middleware/execution_result_middleware.rb +3 -3
- data/lib/gooddata/bricks/pipeline.rb +2 -14
- data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +18 -8
- data/lib/gooddata/cloud_resources/redshift/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/redshift/redshift_client.rb +0 -2
- data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +1 -1
- data/lib/gooddata/lcm/actions/base_action.rb +157 -0
- data/lib/gooddata/lcm/actions/collect_data_product.rb +2 -1
- data/lib/gooddata/lcm/actions/collect_projects_warning_status.rb +53 -0
- data/lib/gooddata/lcm/actions/collect_segment_clients.rb +14 -0
- data/lib/gooddata/lcm/actions/initialize_continue_on_error_option.rb +87 -0
- data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +28 -2
- data/lib/gooddata/lcm/actions/provision_clients.rb +34 -5
- data/lib/gooddata/lcm/actions/synchronize_cas.rb +24 -4
- data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -4
- data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +28 -3
- data/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb +48 -11
- data/lib/gooddata/lcm/actions/synchronize_kd_dashboard_permission.rb +103 -0
- data/lib/gooddata/lcm/actions/synchronize_ldm.rb +60 -15
- data/lib/gooddata/lcm/actions/synchronize_ldm_layout.rb +98 -0
- data/lib/gooddata/lcm/actions/synchronize_pp_dashboard_permission.rb +108 -0
- data/lib/gooddata/lcm/actions/synchronize_schedules.rb +31 -1
- data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +14 -9
- data/lib/gooddata/lcm/actions/synchronize_user_groups.rb +30 -4
- data/lib/gooddata/lcm/actions/synchronize_users.rb +11 -10
- data/lib/gooddata/lcm/actions/update_metric_formats.rb +21 -4
- data/lib/gooddata/lcm/exceptions/lcm_execution_warning.rb +15 -0
- data/lib/gooddata/lcm/helpers/check_helper.rb +19 -0
- data/lib/gooddata/lcm/lcm2.rb +45 -4
- data/lib/gooddata/lcm/user_bricks_helper.rb +9 -0
- data/lib/gooddata/mixins/inspector.rb +1 -1
- data/lib/gooddata/models/ldm_layout.rb +38 -0
- data/lib/gooddata/models/project.rb +197 -22
- data/lib/gooddata/models/project_creator.rb +83 -6
- data/lib/gooddata/models/segment.rb +2 -1
- data/lib/gooddata/models/user_filters/user_filter_builder.rb +104 -15
- data/lib/gooddata/rest/connection.rb +5 -3
- data/lib/gooddata/rest/phmap.rb +1 -0
- data/lib/gooddata.rb +1 -0
- data/lib/gooddata_brick_base.rb +35 -0
- data/sonar-project.properties +6 -0
- metadata +60 -55
- data/lib/gooddata/cloud_resources/redshift/drivers/log4j.properties +0 -15
data/gooddata.gemspec
CHANGED
@@ -1,13 +1,11 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
$LOAD_PATH.push File.expand_path('../lib/', __FILE__)
|
3
3
|
require 'gooddata/version'
|
4
|
-
|
5
4
|
Gem::Specification.new do |s|
|
6
5
|
s.name = 'gooddata'
|
7
6
|
s.version = GoodData::VERSION
|
8
7
|
s.licenses = ['BSD-3-Clause']
|
9
8
|
s.platform = 'java' if RUBY_PLATFORM =~ /java/
|
10
|
-
|
11
9
|
s.required_rubygems_version = Gem::Requirement.new('>= 0') if s.respond_to? :required_rubygems_version=
|
12
10
|
s.authors = [
|
13
11
|
'Pavel Kolesnikov',
|
@@ -18,27 +16,27 @@ Gem::Specification.new do |s|
|
|
18
16
|
'Petr Gadorek',
|
19
17
|
'Jakub Mahnert'
|
20
18
|
]
|
21
|
-
|
22
19
|
s.summary = 'A convenient Ruby wrapper around the GoodData RESTful API'
|
23
20
|
s.description = 'Use the GoodData::Client class to integrate GoodData into your own application or use the CLI to work with GoodData directly from the command line.'
|
24
21
|
s.email = 'lcm@gooddata.com'
|
25
22
|
s.extra_rdoc_files = %w(LICENSE README.md)
|
26
|
-
|
27
23
|
s.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR).map { |f| f unless %w(NOTICES.txt LICENSE_FOR_RUBY_SDK_COMPONENT.txt).include?(f) }
|
28
24
|
s.files.reject! { |fn| fn.start_with? 'spec/' }
|
29
25
|
s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
|
30
|
-
|
31
26
|
s.homepage = 'http://github.com/gooddata/gooddata-ruby'
|
32
27
|
s.require_paths = ['lib']
|
33
|
-
|
34
28
|
# s.add_development_dependency 'bundler', "~> 1.14"
|
35
29
|
s.add_development_dependency 'license_finder', '~> 2.0'
|
36
|
-
s.add_development_dependency 'rake', '~>
|
30
|
+
s.add_development_dependency 'rake', '~> 13.0'
|
37
31
|
s.add_development_dependency 'redcarpet', '~> 3.1' if RUBY_PLATFORM != 'java'
|
38
|
-
s.add_development_dependency 'rspec', '~> 3.
|
39
|
-
s.add_development_dependency 'rspec-expectations', '~> 3.
|
40
|
-
s.add_development_dependency 'rspec_junit_formatter', '~> 0.
|
41
|
-
|
32
|
+
s.add_development_dependency 'rspec', '~> 3.12.0'
|
33
|
+
s.add_development_dependency 'rspec-expectations', '~> 3.12'
|
34
|
+
s.add_development_dependency 'rspec_junit_formatter', '~> 0.6.0'
|
35
|
+
if RUBY_VERSION >= '2.6'
|
36
|
+
s.add_development_dependency 'rubocop', '>= 1.28'
|
37
|
+
else
|
38
|
+
s.add_development_dependency 'rubocop', '~> 0.81'
|
39
|
+
end
|
42
40
|
s.add_development_dependency 'simplecov', '~> 0.12'
|
43
41
|
s.add_development_dependency 'webmock', '~> 2.3.1'
|
44
42
|
s.add_development_dependency 'yard', '~> 0.9.11'
|
@@ -46,16 +44,16 @@ Gem::Specification.new do |s|
|
|
46
44
|
s.add_development_dependency 'pry'
|
47
45
|
s.add_development_dependency 'pry-byebug', '~> 3.6' if RUBY_PLATFORM != 'java'
|
48
46
|
|
49
|
-
s.add_development_dependency 'pronto', '
|
50
|
-
s.add_development_dependency 'pronto-rubocop', '
|
51
|
-
s.add_development_dependency 'pronto-reek', '
|
47
|
+
s.add_development_dependency 'pronto', '>= 0.10' if RUBY_PLATFORM != 'java'
|
48
|
+
s.add_development_dependency 'pronto-rubocop', '>= 0.9' if RUBY_PLATFORM != 'java'
|
49
|
+
s.add_development_dependency 'pronto-reek', '>= 0.9' if RUBY_PLATFORM != 'java'
|
52
50
|
s.add_development_dependency 'vcr', '5.0.0'
|
53
51
|
s.add_development_dependency 'hashdiff', '~> 0.4'
|
54
52
|
|
55
53
|
s.add_development_dependency 'sqlite3' if RUBY_PLATFORM != 'java'
|
56
54
|
|
57
|
-
|
58
|
-
|
55
|
+
if RUBY_VERSION >= '2.5'
|
56
|
+
s.add_dependency 'activesupport', '< 7.0.0'
|
59
57
|
else
|
60
58
|
s.add_dependency 'activesupport', '>= 5.2.4.3', '< 6.0'
|
61
59
|
end
|
@@ -66,19 +64,20 @@ Gem::Specification.new do |s|
|
|
66
64
|
else
|
67
65
|
s.add_dependency 'docile', '> 1.1', '< 1.4.0'
|
68
66
|
end
|
69
|
-
s.add_dependency 'azure-storage-blob', '~>
|
70
|
-
s.add_dependency 'nokogiri', '~> 1.10.
|
67
|
+
s.add_dependency 'azure-storage-blob', '~> 2.0'
|
68
|
+
s.add_dependency 'nokogiri', '~> 1', '>= 1.10.8'
|
71
69
|
s.add_dependency 'gli', '~> 2.15'
|
72
|
-
s.add_dependency 'gooddata_datawarehouse', '~> 0.0.
|
70
|
+
s.add_dependency 'gooddata_datawarehouse', '~> 0.0.11' if RUBY_PLATFORM == 'java'
|
73
71
|
s.add_dependency 'highline', '= 2.0.0.pre.develop.14'
|
74
|
-
s.add_dependency 'json_pure', '~>
|
72
|
+
s.add_dependency 'json_pure', '~> 2.6'
|
75
73
|
s.add_dependency 'multi_json', '~> 1.12'
|
76
74
|
s.add_dependency 'parseconfig', '~> 1.0'
|
77
75
|
s.add_dependency 'pmap', '~> 1.1'
|
76
|
+
s.add_dependency 'sequel', '< 5.72.0'
|
78
77
|
s.add_dependency 'remote_syslog_logger', '~> 1.0.3'
|
79
|
-
s.add_dependency 'restforce', '>= 2.4'
|
78
|
+
s.add_dependency 'restforce', '>= 2.4'
|
80
79
|
s.add_dependency 'rest-client', '~> 2.0'
|
81
|
-
s.add_dependency 'rubyzip'
|
80
|
+
s.add_dependency 'rubyzip'
|
82
81
|
s.add_dependency 'terminal-table', '~> 1.7'
|
83
82
|
s.add_dependency 'thread_safe'
|
84
83
|
s.add_dependency 'backports'
|
data/lcm.rake
CHANGED
@@ -138,6 +138,15 @@ namespace :maven do
|
|
138
138
|
|
139
139
|
system('mvn -f ci/redshift/pom.xml clean install -P binary-packaging')
|
140
140
|
system('cp -rf ci/redshift/target/*.jar lib/gooddata/cloud_resources/redshift/drivers/')
|
141
|
+
|
142
|
+
system('mvn -f ci/postgresql/pom.xml clean install -P binary-packaging')
|
143
|
+
system('cp -rf ci/postgresql/target/*.jar lib/gooddata/cloud_resources/postgresql/drivers/')
|
144
|
+
|
145
|
+
system('mvn -f ci/mssql/pom.xml clean install -P binary-packaging')
|
146
|
+
system('cp -rf ci/mssql/target/*.jar lib/gooddata/cloud_resources/mssql/drivers/')
|
147
|
+
|
148
|
+
system('mvn -f ci/mysql/pom.xml clean install -P binary-packaging')
|
149
|
+
system('cp -rf ci/mysql/target/*.jar lib/gooddata/cloud_resources/mysql/drivers/')
|
141
150
|
end
|
142
151
|
end
|
143
152
|
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2010-2022 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
module GoodData
|
9
|
+
module Bricks
|
10
|
+
class BasePipeline
|
11
|
+
# Pipeline preparation code
|
12
|
+
def self.prepare(pipeline)
|
13
|
+
pipeline.reverse.reduce(nil) do |memo, app|
|
14
|
+
if memo.nil?
|
15
|
+
app.respond_to?(:new) ? app.new : app
|
16
|
+
elsif app.respond_to?(:new)
|
17
|
+
app.new(app: memo)
|
18
|
+
else
|
19
|
+
app.app = memo
|
20
|
+
app
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -39,7 +39,7 @@ module GoodData
|
|
39
39
|
message: message
|
40
40
|
}
|
41
41
|
}
|
42
|
-
update_result(result)
|
42
|
+
update_result(result, status)
|
43
43
|
end
|
44
44
|
|
45
45
|
private
|
@@ -52,9 +52,9 @@ module GoodData
|
|
52
52
|
@@result_log_path = result_log_path
|
53
53
|
end
|
54
54
|
|
55
|
-
def self.update_result(result)
|
55
|
+
def self.update_result(result, status = ExecutionStatus::ERROR)
|
56
56
|
if @@result_log_path.nil?
|
57
|
-
GoodData.gd_logger
|
57
|
+
GoodData.gd_logger&.warn("action=update_execution_result status=#{status} Not found execution result logger file.")
|
58
58
|
return
|
59
59
|
end
|
60
60
|
|
@@ -10,23 +10,11 @@ require_relative 'release_brick'
|
|
10
10
|
require_relative 'provisioning_brick'
|
11
11
|
require_relative 'rollout_brick'
|
12
12
|
require_relative 'hello_world_brick'
|
13
|
+
require_relative 'base_pipeline'
|
13
14
|
|
14
15
|
module GoodData
|
15
16
|
module Bricks
|
16
|
-
class Pipeline
|
17
|
-
# Pipeline preparation code
|
18
|
-
def self.prepare(pipeline)
|
19
|
-
pipeline.reverse.reduce(nil) do |memo, app|
|
20
|
-
if memo.nil?
|
21
|
-
app.respond_to?(:new) ? app.new : app
|
22
|
-
elsif app.respond_to?(:new)
|
23
|
-
app.new(app: memo)
|
24
|
-
else
|
25
|
-
app.app = memo
|
26
|
-
app
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
17
|
+
class Pipeline < GoodData::Bricks::BasePipeline
|
30
18
|
|
31
19
|
def self.users_brick_pipeline
|
32
20
|
prepare([
|
@@ -21,10 +21,12 @@ module GoodData
|
|
21
21
|
JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
|
22
22
|
MYSQL_DEFAULT_PORT = 3306
|
23
23
|
JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://'
|
24
|
-
VERIFY_FULL = '
|
25
|
-
PREFER = '
|
26
|
-
REQUIRE = '
|
24
|
+
VERIFY_FULL = '&useSSL=true&verifyServerCertificate=true'
|
25
|
+
PREFER = '&useSSL=true&requireSSL=false&verifyServerCertificate=false'
|
26
|
+
REQUIRE = '&useSSL=true&requireSSL=true&verifyServerCertificate=false'
|
27
27
|
MYSQL_FETCH_SIZE = 1000
|
28
|
+
MONGO_BI_FETCH_SIZE = 1
|
29
|
+
MONGO_BI_TYPE = 'MongoDBConnector'
|
28
30
|
|
29
31
|
class << self
|
30
32
|
def accept?(type)
|
@@ -39,6 +41,7 @@ module GoodData
|
|
39
41
|
@database = options['mysql_client']['connection']['database']
|
40
42
|
@authentication = options['mysql_client']['connection']['authentication']
|
41
43
|
@ssl_mode = options['mysql_client']['connection']['sslMode']
|
44
|
+
@database_type = options['mysql_client']['connection']['databaseType']
|
42
45
|
raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
|
43
46
|
|
44
47
|
@url = build_url(options['mysql_client']['connection']['url'])
|
@@ -46,7 +49,7 @@ module GoodData
|
|
46
49
|
raise('Missing connection info for Mysql client')
|
47
50
|
end
|
48
51
|
|
49
|
-
Java.com.mysql.
|
52
|
+
Java.com.mysql.jdbc.Driver
|
50
53
|
end
|
51
54
|
|
52
55
|
def realize_query(query, _params)
|
@@ -56,7 +59,7 @@ module GoodData
|
|
56
59
|
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
|
57
60
|
measure = Benchmark.measure do
|
58
61
|
statement = @connection.create_statement
|
59
|
-
statement.set_fetch_size(
|
62
|
+
statement.set_fetch_size(fetch_size)
|
60
63
|
has_result = statement.execute(query)
|
61
64
|
if has_result
|
62
65
|
result = statement.get_result_set
|
@@ -76,12 +79,11 @@ module GoodData
|
|
76
79
|
end
|
77
80
|
|
78
81
|
def connect
|
79
|
-
GoodData.logger.info "Setting up connection to Mysql #{@url}"
|
82
|
+
GoodData.logger.info "Setting up connection to Mysql #{@database_type} #{@url} "
|
80
83
|
|
81
84
|
prop = java.util.Properties.new
|
82
85
|
prop.setProperty('user', @authentication['basic']['userName'])
|
83
86
|
prop.setProperty('password', @authentication['basic']['password'])
|
84
|
-
|
85
87
|
@connection = java.sql.DriverManager.getConnection(@url, prop)
|
86
88
|
@connection.set_auto_commit(false)
|
87
89
|
end
|
@@ -93,7 +95,11 @@ module GoodData
|
|
93
95
|
host = matches[0][0]
|
94
96
|
port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT
|
95
97
|
|
96
|
-
"#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}
|
98
|
+
"#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?#{get_ssl_mode(@ssl_mode)}#{add_extended}&useCursorFetch=true&enabledTLSProtocols=TLSv1.2"
|
99
|
+
end
|
100
|
+
|
101
|
+
def fetch_size
|
102
|
+
@database_type == MONGO_BI_TYPE ? MONGO_BI_FETCH_SIZE : MYSQL_FETCH_SIZE
|
97
103
|
end
|
98
104
|
|
99
105
|
def get_ssl_mode(ssl_mode)
|
@@ -106,6 +112,10 @@ module GoodData
|
|
106
112
|
|
107
113
|
mode
|
108
114
|
end
|
115
|
+
|
116
|
+
def add_extended
|
117
|
+
@database_type == MONGO_BI_TYPE ? '&authenticationPlugins=org.mongodb.mongosql.auth.plugin.MongoSqlAuthenticationPlugin&useLocalTransactionState=true' : ''
|
118
|
+
end
|
109
119
|
end
|
110
120
|
end
|
111
121
|
end
|
File without changes
|
@@ -39,8 +39,6 @@ module GoodData
|
|
39
39
|
@debug = options['debug'] == true || options['debug'] == 'true'
|
40
40
|
|
41
41
|
Java.com.amazon.redshift.jdbc42.Driver
|
42
|
-
base = Pathname(__FILE__).dirname
|
43
|
-
org.apache.log4j.PropertyConfigurator.configure("#{base}/drivers/log4j.properties")
|
44
42
|
end
|
45
43
|
|
46
44
|
def realize_query(query, _params)
|
@@ -18,7 +18,7 @@ end
|
|
18
18
|
module GoodData
|
19
19
|
module CloudResources
|
20
20
|
class SnowflakeClient < CloudResourceClient
|
21
|
-
SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=
|
21
|
+
SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=gooddata_platform'
|
22
22
|
SNOWFLAKE_SEPARATOR_PARAM = '?'
|
23
23
|
|
24
24
|
class << self
|
@@ -18,6 +18,11 @@ module GoodData
|
|
18
18
|
class << self
|
19
19
|
include Dsl::Dsl
|
20
20
|
|
21
|
+
SYNC_FAILED_LIST = 'sync_failed_list'.to_sym
|
22
|
+
FAILED_PROJECTS = 'failed_projects'.to_sym
|
23
|
+
FAILED_CLIENTS = 'failed_clients'.to_sym
|
24
|
+
FAILED_SEGMENTS = 'failed_segments'.to_sym
|
25
|
+
|
21
26
|
def check_params(specification, params)
|
22
27
|
Helpers.check_params(specification, params)
|
23
28
|
end
|
@@ -31,6 +36,158 @@ module GoodData
|
|
31
36
|
params.setup_filters(specification) # enables params validation
|
32
37
|
result
|
33
38
|
end
|
39
|
+
|
40
|
+
def print_result(_params)
|
41
|
+
true
|
42
|
+
end
|
43
|
+
|
44
|
+
def continue_on_error(params)
|
45
|
+
Helpers.continue_on_error(params)
|
46
|
+
end
|
47
|
+
|
48
|
+
def collect_synced_status(params)
|
49
|
+
Helpers.collect_synced_status(params)
|
50
|
+
end
|
51
|
+
|
52
|
+
def add_failed_project(project_id, message, failed_action, params)
|
53
|
+
if collect_synced_status(params) && !sync_failed_project(project_id, params)
|
54
|
+
sync_failed_list = sync_failed_list(params)
|
55
|
+
project_client_mappings = sync_failed_list[:project_client_mappings]
|
56
|
+
project_client_mapping = project_client_mappings ? project_client_mappings[project_id.to_sym] : nil
|
57
|
+
client_id = project_client_mapping ? project_client_mapping[:client_id] : nil
|
58
|
+
segment_id = project_client_mapping ? project_client_mapping[:segment_id] : nil
|
59
|
+
|
60
|
+
failed_detailed_project = {
|
61
|
+
project_id: project_id,
|
62
|
+
client_id: client_id,
|
63
|
+
segment: segment_id,
|
64
|
+
message: message,
|
65
|
+
action: failed_action
|
66
|
+
}
|
67
|
+
add_failed_detail(params, failed_detailed_project, sync_failed_list)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def add_failed_client(client_id, message, error_action, params)
|
72
|
+
if collect_synced_status(params) && !sync_failed_client(client_id, params)
|
73
|
+
sync_failed_list = sync_failed_list(params)
|
74
|
+
client_project_mappings = sync_failed_list[:client_project_mappings]
|
75
|
+
client_project_mapping = client_project_mappings ? client_project_mappings[client_id.to_sym] : nil
|
76
|
+
project_id = client_project_mapping ? client_project_mapping[:project_id] : nil
|
77
|
+
segment_id = client_project_mapping ? client_project_mapping[:segment_id] : nil
|
78
|
+
|
79
|
+
failed_detailed_client = {
|
80
|
+
project_id: project_id,
|
81
|
+
client_id: client_id,
|
82
|
+
segment: segment_id,
|
83
|
+
message: message,
|
84
|
+
action: error_action
|
85
|
+
}
|
86
|
+
add_failed_detail(params, failed_detailed_client, sync_failed_list)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def add_failed_segment(segment_id, message, error_action, params)
|
91
|
+
if collect_synced_status(params) && !sync_failed_segment(segment_id, params)
|
92
|
+
sync_failed_list = sync_failed_list(params)
|
93
|
+
failed_detailed_segment = {
|
94
|
+
project_id: nil,
|
95
|
+
client_id: nil,
|
96
|
+
segment: segment_id,
|
97
|
+
message: message,
|
98
|
+
action: error_action
|
99
|
+
}
|
100
|
+
add_failed_detail(params, failed_detailed_segment, sync_failed_list, true)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
# Add new clients to project client mapping
|
105
|
+
#
|
106
|
+
# @param [String] project_id project identify will be added to mapping
|
107
|
+
# @param [String] client_id client identify will be added to mapping
|
108
|
+
# @param [String] segment_id segment identify will be added to mapping
|
109
|
+
# @param [Hash] params the hash contains list of parameters and values
|
110
|
+
def add_new_clients_to_project_client_mapping(project_id, client_id, segment_id, params)
|
111
|
+
if collect_synced_status(params)
|
112
|
+
sync_failed_list = sync_failed_list(params)
|
113
|
+
client_project_mappings = sync_failed_list[:client_project_mappings]
|
114
|
+
project_client_mappings = sync_failed_list[:project_client_mappings]
|
115
|
+
client_project_mappings[client_id.to_sym] = {
|
116
|
+
project_id: project_id,
|
117
|
+
segment_id: segment_id
|
118
|
+
} if client_project_mappings
|
119
|
+
|
120
|
+
project_client_mappings[project_id.to_sym] = {
|
121
|
+
client_id: client_id,
|
122
|
+
segment_id: segment_id
|
123
|
+
} if project_client_mappings
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def process_failed_project(project_id, failed_message, failed_projects, continue_on_error)
|
128
|
+
fail(failed_message) unless continue_on_error
|
129
|
+
|
130
|
+
failed_projects << {
|
131
|
+
project_id: project_id,
|
132
|
+
message: failed_message
|
133
|
+
}
|
134
|
+
end
|
135
|
+
|
136
|
+
def process_failed_projects(failed_projects, failed_action, params)
|
137
|
+
failed_projects.each do |failed_project|
|
138
|
+
add_failed_project(failed_project[:project_id], failed_project[:message], failed_action, params)
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def sync_failed_project(project_id, params)
|
143
|
+
collect_synced_status(params) && params[SYNC_FAILED_LIST][FAILED_PROJECTS].include?(project_id)
|
144
|
+
end
|
145
|
+
|
146
|
+
def sync_failed_client(client_id, params)
|
147
|
+
collect_synced_status(params) && params[SYNC_FAILED_LIST][FAILED_CLIENTS].include?(client_id)
|
148
|
+
end
|
149
|
+
|
150
|
+
def sync_failed_segment(segment_id, params)
|
151
|
+
collect_synced_status(params) && params[SYNC_FAILED_LIST][FAILED_SEGMENTS].include?(segment_id)
|
152
|
+
end
|
153
|
+
|
154
|
+
private
|
155
|
+
|
156
|
+
def add_failed_detail(params, failed_detailed_project, sync_failed_list, ignore_segment = false)
|
157
|
+
params.gdc_logger&.warn failed_detailed_project[:message]
|
158
|
+
sync_failed_list[:failed_detailed_projects] << failed_detailed_project
|
159
|
+
|
160
|
+
if ignore_segment
|
161
|
+
add_failed_detail_segment(failed_detailed_project[:segment_id], sync_failed_list)
|
162
|
+
else
|
163
|
+
add_failed_detail_client(failed_detailed_project[:client_id], failed_detailed_project[:project_id], sync_failed_list)
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
def sync_failed_list(params)
|
168
|
+
if params.include?(SYNC_FAILED_LIST)
|
169
|
+
params[SYNC_FAILED_LIST]
|
170
|
+
else
|
171
|
+
nil
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
def add_failed_detail_client(client_id, project_id, sync_failed_list)
|
176
|
+
sync_failed_list[FAILED_CLIENTS] << client_id if client_id
|
177
|
+
|
178
|
+
sync_failed_list[FAILED_PROJECTS] << project_id if project_id
|
179
|
+
end
|
180
|
+
|
181
|
+
def add_failed_detail_segment(segment_id, sync_failed_list)
|
182
|
+
if segment_id
|
183
|
+
sync_failed_list[FAILED_SEGMENTS] << segment_id
|
184
|
+
|
185
|
+
client_project_mappings = sync_failed_list[:client_project_mappings]
|
186
|
+
client_project_mappings.each do |client_id, client_project_mapping|
|
187
|
+
add_failed_detail_client(client_id, client_project_mapping[:project_id], sync_failed_list) if client_project_mapping[:segment_id] == segment_id
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
34
191
|
end
|
35
192
|
end
|
36
193
|
end
|
@@ -47,8 +47,9 @@ module GoodData
|
|
47
47
|
end
|
48
48
|
|
49
49
|
begin
|
50
|
+
params.gdc_logger.info "Starting to find DataProduct ID: #{data_product_id}"
|
50
51
|
data_product = domain.data_products(data_product_id)
|
51
|
-
rescue RestClient::BadRequest
|
52
|
+
rescue RestClient::BadRequest, RestClient::NotFound
|
52
53
|
params.gdc_logger.info "Can not find DataProduct #{params.data_product}, creating it instead"
|
53
54
|
data_product = domain.create_data_product(id: params.data_product)
|
54
55
|
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# (C) 2019-2022 GoodData Corporation
|
3
|
+
require_relative 'base_action'
|
4
|
+
|
5
|
+
module GoodData
|
6
|
+
module LCM2
|
7
|
+
class CollectProjectsWarningStatus < BaseAction
|
8
|
+
DESCRIPTION = 'Collect synced projects status'
|
9
|
+
|
10
|
+
PARAMS = define_params(self) do
|
11
|
+
description 'Abort on error'
|
12
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
13
|
+
|
14
|
+
description 'Logger'
|
15
|
+
param :gdc_logger, instance_of(Type::GdLogger), required: false
|
16
|
+
|
17
|
+
description 'Collect synced status'
|
18
|
+
param :collect_synced_status, instance_of(Type::BooleanType), required: false
|
19
|
+
|
20
|
+
description 'Sync failed list'
|
21
|
+
param :sync_failed_list, instance_of(Type::HashType), required: false
|
22
|
+
end
|
23
|
+
|
24
|
+
RESULT_HEADER = %i[segment client project_pid failed_action]
|
25
|
+
|
26
|
+
class << self
|
27
|
+
def call(params)
|
28
|
+
results = []
|
29
|
+
return results unless collect_synced_status(params)
|
30
|
+
|
31
|
+
sync_failed_list = params[SYNC_FAILED_LIST]
|
32
|
+
if sync_failed_list
|
33
|
+
failed_detailed_projects = sync_failed_list[:failed_detailed_projects]
|
34
|
+
failed_detailed_projects.each do |failed_detailed_project|
|
35
|
+
results << {
|
36
|
+
segment: failed_detailed_project[:segment],
|
37
|
+
client: failed_detailed_project[:client_id],
|
38
|
+
project_pid: failed_detailed_project[:project_id],
|
39
|
+
failed_action: failed_detailed_project[:action]
|
40
|
+
}
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
results
|
45
|
+
end
|
46
|
+
|
47
|
+
def print_result(params)
|
48
|
+
collect_synced_status(params)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -32,9 +32,22 @@ module GoodData
|
|
32
32
|
|
33
33
|
description 'Domain'
|
34
34
|
param :domain, instance_of(Type::StringType), required: false
|
35
|
+
|
36
|
+
description 'Abort on error'
|
37
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
38
|
+
|
39
|
+
description 'Logger'
|
40
|
+
param :gdc_logger, instance_of(Type::GdLogger), required: false
|
41
|
+
|
42
|
+
description 'Collect synced status'
|
43
|
+
param :collect_synced_status, instance_of(Type::BooleanType), required: false
|
44
|
+
|
45
|
+
description 'Sync failed list'
|
46
|
+
param :sync_failed_list, instance_of(Type::HashType), required: false
|
35
47
|
end
|
36
48
|
|
37
49
|
RESULT_HEADER = [
|
50
|
+
:segment,
|
38
51
|
:from_name,
|
39
52
|
:from_pid,
|
40
53
|
:to_name,
|
@@ -89,6 +102,7 @@ module GoodData
|
|
89
102
|
client_project = segment_client.project
|
90
103
|
to_pid = client_project.pid
|
91
104
|
results << {
|
105
|
+
segment: segment.segment_id,
|
92
106
|
from_name: master_name,
|
93
107
|
from_pid: master_pid,
|
94
108
|
to_name: client_project.title,
|
@@ -0,0 +1,87 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# (C) 2019-2022 GoodData Corporation
|
3
|
+
require_relative 'base_action'
|
4
|
+
|
5
|
+
module GoodData
|
6
|
+
module LCM2
|
7
|
+
class InitializeContinueOnErrorOption < BaseAction
|
8
|
+
DESCRIPTION = 'Initialize continue on error option'
|
9
|
+
|
10
|
+
PARAMS = define_params(self) do
|
11
|
+
description 'Organization Name'
|
12
|
+
param :organization, instance_of(Type::StringType), required: false
|
13
|
+
|
14
|
+
description 'Domain'
|
15
|
+
param :domain, instance_of(Type::StringType), required: false
|
16
|
+
|
17
|
+
description 'DataProduct'
|
18
|
+
param :data_product, instance_of(Type::GDDataProductType), required: false
|
19
|
+
|
20
|
+
description 'Restricts synchronization to specified segments'
|
21
|
+
param :segments_filter, array_of(instance_of(Type::StringType)), required: false
|
22
|
+
|
23
|
+
description 'Abort on error'
|
24
|
+
param :abort_on_error, instance_of(Type::StringType), required: false
|
25
|
+
|
26
|
+
description 'Client Used for Connecting to GD'
|
27
|
+
param :gdc_gd_client, instance_of(Type::GdClientType), required: true
|
28
|
+
end
|
29
|
+
|
30
|
+
class << self
|
31
|
+
def call(params)
|
32
|
+
project_mappings = ThreadSafe::Hash.new
|
33
|
+
client_mappings = ThreadSafe::Hash.new
|
34
|
+
continue_on_error = continue_on_error(params)
|
35
|
+
|
36
|
+
if continue_on_error
|
37
|
+
client = params.gdc_gd_client
|
38
|
+
domain_name = params.organization || params.domain
|
39
|
+
domain = client.domain(domain_name)
|
40
|
+
data_product = params.data_product
|
41
|
+
data_product_segments = domain.segments(:all, data_product)
|
42
|
+
|
43
|
+
if params.segments_filter
|
44
|
+
data_product_segments.select! do |segment|
|
45
|
+
params.segments_filter.include?(segment.segment_id)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
data_product_segments.pmap do |segment|
|
50
|
+
segment.clients.map do |segment_client|
|
51
|
+
project = segment_client.project
|
52
|
+
next unless project
|
53
|
+
|
54
|
+
project_mappings[project.pid.to_sym] = {
|
55
|
+
client_id: segment_client.client_id,
|
56
|
+
segment_id: segment.segment_id
|
57
|
+
}
|
58
|
+
client_mappings[segment_client.client_id.to_sym] = {
|
59
|
+
project_id: project.pid,
|
60
|
+
segment_id: segment.segment_id
|
61
|
+
}
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
{
|
67
|
+
params: {
|
68
|
+
collect_synced_status: continue_on_error,
|
69
|
+
sync_failed_list: {
|
70
|
+
project_client_mappings: project_mappings,
|
71
|
+
client_project_mappings: client_mappings,
|
72
|
+
failed_detailed_projects: [],
|
73
|
+
failed_projects: [],
|
74
|
+
failed_clients: [],
|
75
|
+
failed_segments: []
|
76
|
+
}
|
77
|
+
}
|
78
|
+
}
|
79
|
+
end
|
80
|
+
|
81
|
+
def print_result(_params)
|
82
|
+
false
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|