gooddata 2.2.0-java → 2.3.0-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (67) hide show
  1. checksums.yaml +5 -5
  2. data/.gdc-ii-config.yaml +42 -1
  3. data/.github/workflows/build.yml +14 -13
  4. data/.github/workflows/pre-merge.yml +13 -13
  5. data/.pronto.yml +1 -0
  6. data/.rubocop.yml +2 -14
  7. data/CHANGELOG.md +9 -0
  8. data/Dockerfile +13 -7
  9. data/Dockerfile.jruby +5 -5
  10. data/Dockerfile.ruby +5 -7
  11. data/Gemfile +4 -2
  12. data/README.md +5 -4
  13. data/Rakefile +1 -1
  14. data/SDK_VERSION +1 -1
  15. data/VERSION +1 -1
  16. data/bin/run_brick.rb +7 -0
  17. data/ci/mysql/pom.xml +6 -1
  18. data/ci/redshift/pom.xml +3 -4
  19. data/docker-compose.lcm.yml +42 -1
  20. data/docker-compose.yml +42 -0
  21. data/gooddata.gemspec +21 -22
  22. data/lcm.rake +9 -0
  23. data/lib/gooddata/bricks/base_pipeline.rb +26 -0
  24. data/lib/gooddata/bricks/brick.rb +0 -1
  25. data/lib/gooddata/bricks/middleware/execution_result_middleware.rb +3 -3
  26. data/lib/gooddata/bricks/pipeline.rb +2 -14
  27. data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +18 -8
  28. data/lib/gooddata/cloud_resources/redshift/drivers/.gitkeepme +0 -0
  29. data/lib/gooddata/cloud_resources/redshift/redshift_client.rb +0 -2
  30. data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +1 -1
  31. data/lib/gooddata/lcm/actions/base_action.rb +157 -0
  32. data/lib/gooddata/lcm/actions/collect_data_product.rb +2 -1
  33. data/lib/gooddata/lcm/actions/collect_projects_warning_status.rb +53 -0
  34. data/lib/gooddata/lcm/actions/collect_segment_clients.rb +14 -0
  35. data/lib/gooddata/lcm/actions/initialize_continue_on_error_option.rb +87 -0
  36. data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +28 -2
  37. data/lib/gooddata/lcm/actions/provision_clients.rb +34 -5
  38. data/lib/gooddata/lcm/actions/synchronize_cas.rb +24 -4
  39. data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -4
  40. data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +28 -3
  41. data/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb +48 -11
  42. data/lib/gooddata/lcm/actions/synchronize_kd_dashboard_permission.rb +103 -0
  43. data/lib/gooddata/lcm/actions/synchronize_ldm.rb +60 -15
  44. data/lib/gooddata/lcm/actions/synchronize_ldm_layout.rb +98 -0
  45. data/lib/gooddata/lcm/actions/synchronize_pp_dashboard_permission.rb +108 -0
  46. data/lib/gooddata/lcm/actions/synchronize_schedules.rb +31 -1
  47. data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +14 -9
  48. data/lib/gooddata/lcm/actions/synchronize_user_groups.rb +30 -4
  49. data/lib/gooddata/lcm/actions/synchronize_users.rb +11 -10
  50. data/lib/gooddata/lcm/actions/update_metric_formats.rb +21 -4
  51. data/lib/gooddata/lcm/exceptions/lcm_execution_warning.rb +15 -0
  52. data/lib/gooddata/lcm/helpers/check_helper.rb +19 -0
  53. data/lib/gooddata/lcm/lcm2.rb +45 -4
  54. data/lib/gooddata/lcm/user_bricks_helper.rb +9 -0
  55. data/lib/gooddata/mixins/inspector.rb +1 -1
  56. data/lib/gooddata/models/ldm_layout.rb +38 -0
  57. data/lib/gooddata/models/project.rb +197 -22
  58. data/lib/gooddata/models/project_creator.rb +83 -6
  59. data/lib/gooddata/models/segment.rb +2 -1
  60. data/lib/gooddata/models/user_filters/user_filter_builder.rb +104 -15
  61. data/lib/gooddata/rest/connection.rb +5 -3
  62. data/lib/gooddata/rest/phmap.rb +1 -0
  63. data/lib/gooddata.rb +1 -0
  64. data/lib/gooddata_brick_base.rb +35 -0
  65. data/sonar-project.properties +6 -0
  66. metadata +60 -55
  67. data/lib/gooddata/cloud_resources/redshift/drivers/log4j.properties +0 -15
data/gooddata.gemspec CHANGED
@@ -1,13 +1,11 @@
1
1
  # encoding: utf-8
2
2
  $LOAD_PATH.push File.expand_path('../lib/', __FILE__)
3
3
  require 'gooddata/version'
4
-
5
4
  Gem::Specification.new do |s|
6
5
  s.name = 'gooddata'
7
6
  s.version = GoodData::VERSION
8
7
  s.licenses = ['BSD-3-Clause']
9
8
  s.platform = 'java' if RUBY_PLATFORM =~ /java/
10
-
11
9
  s.required_rubygems_version = Gem::Requirement.new('>= 0') if s.respond_to? :required_rubygems_version=
12
10
  s.authors = [
13
11
  'Pavel Kolesnikov',
@@ -18,27 +16,27 @@ Gem::Specification.new do |s|
18
16
  'Petr Gadorek',
19
17
  'Jakub Mahnert'
20
18
  ]
21
-
22
19
  s.summary = 'A convenient Ruby wrapper around the GoodData RESTful API'
23
20
  s.description = 'Use the GoodData::Client class to integrate GoodData into your own application or use the CLI to work with GoodData directly from the command line.'
24
21
  s.email = 'lcm@gooddata.com'
25
22
  s.extra_rdoc_files = %w(LICENSE README.md)
26
-
27
23
  s.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR).map { |f| f unless %w(NOTICES.txt LICENSE_FOR_RUBY_SDK_COMPONENT.txt).include?(f) }
28
24
  s.files.reject! { |fn| fn.start_with? 'spec/' }
29
25
  s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
30
-
31
26
  s.homepage = 'http://github.com/gooddata/gooddata-ruby'
32
27
  s.require_paths = ['lib']
33
-
34
28
  # s.add_development_dependency 'bundler', "~> 1.14"
35
29
  s.add_development_dependency 'license_finder', '~> 2.0'
36
- s.add_development_dependency 'rake', '~> 11.1'
30
+ s.add_development_dependency 'rake', '~> 13.0'
37
31
  s.add_development_dependency 'redcarpet', '~> 3.1' if RUBY_PLATFORM != 'java'
38
- s.add_development_dependency 'rspec', '~> 3.5.0'
39
- s.add_development_dependency 'rspec-expectations', '~> 3.5'
40
- s.add_development_dependency 'rspec_junit_formatter', '~> 0.3.0'
41
- s.add_development_dependency 'rubocop', '~> 0.59.1'
32
+ s.add_development_dependency 'rspec', '~> 3.12.0'
33
+ s.add_development_dependency 'rspec-expectations', '~> 3.12'
34
+ s.add_development_dependency 'rspec_junit_formatter', '~> 0.6.0'
35
+ if RUBY_VERSION >= '2.6'
36
+ s.add_development_dependency 'rubocop', '>= 1.28'
37
+ else
38
+ s.add_development_dependency 'rubocop', '~> 0.81'
39
+ end
42
40
  s.add_development_dependency 'simplecov', '~> 0.12'
43
41
  s.add_development_dependency 'webmock', '~> 2.3.1'
44
42
  s.add_development_dependency 'yard', '~> 0.9.11'
@@ -46,16 +44,16 @@ Gem::Specification.new do |s|
46
44
  s.add_development_dependency 'pry'
47
45
  s.add_development_dependency 'pry-byebug', '~> 3.6' if RUBY_PLATFORM != 'java'
48
46
 
49
- s.add_development_dependency 'pronto', '~> 0.10' if RUBY_PLATFORM != 'java'
50
- s.add_development_dependency 'pronto-rubocop', '~> 0.9' if RUBY_PLATFORM != 'java'
51
- s.add_development_dependency 'pronto-reek', '~> 0.9' if RUBY_PLATFORM != 'java'
47
+ s.add_development_dependency 'pronto', '>= 0.10' if RUBY_PLATFORM != 'java'
48
+ s.add_development_dependency 'pronto-rubocop', '>= 0.9' if RUBY_PLATFORM != 'java'
49
+ s.add_development_dependency 'pronto-reek', '>= 0.9' if RUBY_PLATFORM != 'java'
52
50
  s.add_development_dependency 'vcr', '5.0.0'
53
51
  s.add_development_dependency 'hashdiff', '~> 0.4'
54
52
 
55
53
  s.add_development_dependency 'sqlite3' if RUBY_PLATFORM != 'java'
56
54
 
57
- if RUBY_VERSION >= '2.5'
58
- s.add_dependency 'activesupport', '>= 6.0.3.1', '< 6.2'
55
+ if RUBY_VERSION >= '2.5'
56
+ s.add_dependency 'activesupport', '< 7.0.0'
59
57
  else
60
58
  s.add_dependency 'activesupport', '>= 5.2.4.3', '< 6.0'
61
59
  end
@@ -66,19 +64,20 @@ Gem::Specification.new do |s|
66
64
  else
67
65
  s.add_dependency 'docile', '> 1.1', '< 1.4.0'
68
66
  end
69
- s.add_dependency 'azure-storage-blob', '~> 1.1.0'
70
- s.add_dependency 'nokogiri', '~> 1.10.0'
67
+ s.add_dependency 'azure-storage-blob', '~> 2.0'
68
+ s.add_dependency 'nokogiri', '~> 1', '>= 1.10.8'
71
69
  s.add_dependency 'gli', '~> 2.15'
72
- s.add_dependency 'gooddata_datawarehouse', '~> 0.0.10' if RUBY_PLATFORM == 'java'
70
+ s.add_dependency 'gooddata_datawarehouse', '~> 0.0.11' if RUBY_PLATFORM == 'java'
73
71
  s.add_dependency 'highline', '= 2.0.0.pre.develop.14'
74
- s.add_dependency 'json_pure', '~> 1.8'
72
+ s.add_dependency 'json_pure', '~> 2.6'
75
73
  s.add_dependency 'multi_json', '~> 1.12'
76
74
  s.add_dependency 'parseconfig', '~> 1.0'
77
75
  s.add_dependency 'pmap', '~> 1.1'
76
+ s.add_dependency 'sequel', '< 5.72.0'
78
77
  s.add_dependency 'remote_syslog_logger', '~> 1.0.3'
79
- s.add_dependency 'restforce', '>= 2.4', '< 4.0'
78
+ s.add_dependency 'restforce', '>= 2.4'
80
79
  s.add_dependency 'rest-client', '~> 2.0'
81
- s.add_dependency 'rubyzip', '~> 1.2', '>= 1.2.1'
80
+ s.add_dependency 'rubyzip'
82
81
  s.add_dependency 'terminal-table', '~> 1.7'
83
82
  s.add_dependency 'thread_safe'
84
83
  s.add_dependency 'backports'
data/lcm.rake CHANGED
@@ -138,6 +138,15 @@ namespace :maven do
138
138
 
139
139
  system('mvn -f ci/redshift/pom.xml clean install -P binary-packaging')
140
140
  system('cp -rf ci/redshift/target/*.jar lib/gooddata/cloud_resources/redshift/drivers/')
141
+
142
+ system('mvn -f ci/postgresql/pom.xml clean install -P binary-packaging')
143
+ system('cp -rf ci/postgresql/target/*.jar lib/gooddata/cloud_resources/postgresql/drivers/')
144
+
145
+ system('mvn -f ci/mssql/pom.xml clean install -P binary-packaging')
146
+ system('cp -rf ci/mssql/target/*.jar lib/gooddata/cloud_resources/mssql/drivers/')
147
+
148
+ system('mvn -f ci/mysql/pom.xml clean install -P binary-packaging')
149
+ system('cp -rf ci/mysql/target/*.jar lib/gooddata/cloud_resources/mysql/drivers/')
141
150
  end
142
151
  end
143
152
 
@@ -0,0 +1,26 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2010-2022 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ module GoodData
9
+ module Bricks
10
+ class BasePipeline
11
+ # Pipeline preparation code
12
+ def self.prepare(pipeline)
13
+ pipeline.reverse.reduce(nil) do |memo, app|
14
+ if memo.nil?
15
+ app.respond_to?(:new) ? app.new : app
16
+ elsif app.respond_to?(:new)
17
+ app.new(app: memo)
18
+ else
19
+ app.app = memo
20
+ app
21
+ end
22
+ end
23
+ end
24
+ end
25
+ end
26
+ end
@@ -5,7 +5,6 @@
5
5
  # LICENSE file in the root directory of this source tree.
6
6
 
7
7
  require_relative 'utils'
8
- require_relative 'middleware/middleware'
9
8
 
10
9
  module GoodData
11
10
  module Bricks
@@ -39,7 +39,7 @@ module GoodData
39
39
  message: message
40
40
  }
41
41
  }
42
- update_result(result)
42
+ update_result(result, status)
43
43
  end
44
44
 
45
45
  private
@@ -52,9 +52,9 @@ module GoodData
52
52
  @@result_log_path = result_log_path
53
53
  end
54
54
 
55
- def self.update_result(result)
55
+ def self.update_result(result, status = ExecutionStatus::ERROR)
56
56
  if @@result_log_path.nil?
57
- GoodData.gd_logger.warn("action=update_execution_result status=error Not found execution result logger file.") unless GoodData.gd_logger.nil?
57
+ GoodData.gd_logger&.warn("action=update_execution_result status=#{status} Not found execution result logger file.")
58
58
  return
59
59
  end
60
60
 
@@ -10,23 +10,11 @@ require_relative 'release_brick'
10
10
  require_relative 'provisioning_brick'
11
11
  require_relative 'rollout_brick'
12
12
  require_relative 'hello_world_brick'
13
+ require_relative 'base_pipeline'
13
14
 
14
15
  module GoodData
15
16
  module Bricks
16
- class Pipeline
17
- # Pipeline preparation code
18
- def self.prepare(pipeline)
19
- pipeline.reverse.reduce(nil) do |memo, app|
20
- if memo.nil?
21
- app.respond_to?(:new) ? app.new : app
22
- elsif app.respond_to?(:new)
23
- app.new(app: memo)
24
- else
25
- app.app = memo
26
- app
27
- end
28
- end
29
- end
17
+ class Pipeline < GoodData::Bricks::BasePipeline
30
18
 
31
19
  def self.users_brick_pipeline
32
20
  prepare([
@@ -21,10 +21,12 @@ module GoodData
21
21
  JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
22
22
  MYSQL_DEFAULT_PORT = 3306
23
23
  JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://'
24
- VERIFY_FULL = 'VERIFY_IDENTITY'
25
- PREFER = 'PREFERRED'
26
- REQUIRE = 'REQUIRED'
24
+ VERIFY_FULL = '&useSSL=true&verifyServerCertificate=true'
25
+ PREFER = '&useSSL=true&requireSSL=false&verifyServerCertificate=false'
26
+ REQUIRE = '&useSSL=true&requireSSL=true&verifyServerCertificate=false'
27
27
  MYSQL_FETCH_SIZE = 1000
28
+ MONGO_BI_FETCH_SIZE = 1
29
+ MONGO_BI_TYPE = 'MongoDBConnector'
28
30
 
29
31
  class << self
30
32
  def accept?(type)
@@ -39,6 +41,7 @@ module GoodData
39
41
  @database = options['mysql_client']['connection']['database']
40
42
  @authentication = options['mysql_client']['connection']['authentication']
41
43
  @ssl_mode = options['mysql_client']['connection']['sslMode']
44
+ @database_type = options['mysql_client']['connection']['databaseType']
42
45
  raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
43
46
 
44
47
  @url = build_url(options['mysql_client']['connection']['url'])
@@ -46,7 +49,7 @@ module GoodData
46
49
  raise('Missing connection info for Mysql client')
47
50
  end
48
51
 
49
- Java.com.mysql.cj.jdbc.Driver
52
+ Java.com.mysql.jdbc.Driver
50
53
  end
51
54
 
52
55
  def realize_query(query, _params)
@@ -56,7 +59,7 @@ module GoodData
56
59
  filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
57
60
  measure = Benchmark.measure do
58
61
  statement = @connection.create_statement
59
- statement.set_fetch_size(MYSQL_FETCH_SIZE)
62
+ statement.set_fetch_size(fetch_size)
60
63
  has_result = statement.execute(query)
61
64
  if has_result
62
65
  result = statement.get_result_set
@@ -76,12 +79,11 @@ module GoodData
76
79
  end
77
80
 
78
81
  def connect
79
- GoodData.logger.info "Setting up connection to Mysql #{@url}"
82
+ GoodData.logger.info "Setting up connection to Mysql #{@database_type} #{@url} "
80
83
 
81
84
  prop = java.util.Properties.new
82
85
  prop.setProperty('user', @authentication['basic']['userName'])
83
86
  prop.setProperty('password', @authentication['basic']['password'])
84
-
85
87
  @connection = java.sql.DriverManager.getConnection(@url, prop)
86
88
  @connection.set_auto_commit(false)
87
89
  end
@@ -93,7 +95,11 @@ module GoodData
93
95
  host = matches[0][0]
94
96
  port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT
95
97
 
96
- "#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{get_ssl_mode(@ssl_mode)}&useCursorFetch=true&enabledTLSProtocols=TLSv1.2"
98
+ "#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?#{get_ssl_mode(@ssl_mode)}#{add_extended}&useCursorFetch=true&enabledTLSProtocols=TLSv1.2"
99
+ end
100
+
101
+ def fetch_size
102
+ @database_type == MONGO_BI_TYPE ? MONGO_BI_FETCH_SIZE : MYSQL_FETCH_SIZE
97
103
  end
98
104
 
99
105
  def get_ssl_mode(ssl_mode)
@@ -106,6 +112,10 @@ module GoodData
106
112
 
107
113
  mode
108
114
  end
115
+
116
+ def add_extended
117
+ @database_type == MONGO_BI_TYPE ? '&authenticationPlugins=org.mongodb.mongosql.auth.plugin.MongoSqlAuthenticationPlugin&useLocalTransactionState=true' : ''
118
+ end
109
119
  end
110
120
  end
111
121
  end
@@ -39,8 +39,6 @@ module GoodData
39
39
  @debug = options['debug'] == true || options['debug'] == 'true'
40
40
 
41
41
  Java.com.amazon.redshift.jdbc42.Driver
42
- base = Pathname(__FILE__).dirname
43
- org.apache.log4j.PropertyConfigurator.configure("#{base}/drivers/log4j.properties")
44
42
  end
45
43
 
46
44
  def realize_query(query, _params)
@@ -18,7 +18,7 @@ end
18
18
  module GoodData
19
19
  module CloudResources
20
20
  class SnowflakeClient < CloudResourceClient
21
- SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=GoodData_Platform'
21
+ SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=gooddata_platform'
22
22
  SNOWFLAKE_SEPARATOR_PARAM = '?'
23
23
 
24
24
  class << self
@@ -18,6 +18,11 @@ module GoodData
18
18
  class << self
19
19
  include Dsl::Dsl
20
20
 
21
+ SYNC_FAILED_LIST = 'sync_failed_list'.to_sym
22
+ FAILED_PROJECTS = 'failed_projects'.to_sym
23
+ FAILED_CLIENTS = 'failed_clients'.to_sym
24
+ FAILED_SEGMENTS = 'failed_segments'.to_sym
25
+
21
26
  def check_params(specification, params)
22
27
  Helpers.check_params(specification, params)
23
28
  end
@@ -31,6 +36,158 @@ module GoodData
31
36
  params.setup_filters(specification) # enables params validation
32
37
  result
33
38
  end
39
+
40
+ def print_result(_params)
41
+ true
42
+ end
43
+
44
+ def continue_on_error(params)
45
+ Helpers.continue_on_error(params)
46
+ end
47
+
48
+ def collect_synced_status(params)
49
+ Helpers.collect_synced_status(params)
50
+ end
51
+
52
+ def add_failed_project(project_id, message, failed_action, params)
53
+ if collect_synced_status(params) && !sync_failed_project(project_id, params)
54
+ sync_failed_list = sync_failed_list(params)
55
+ project_client_mappings = sync_failed_list[:project_client_mappings]
56
+ project_client_mapping = project_client_mappings ? project_client_mappings[project_id.to_sym] : nil
57
+ client_id = project_client_mapping ? project_client_mapping[:client_id] : nil
58
+ segment_id = project_client_mapping ? project_client_mapping[:segment_id] : nil
59
+
60
+ failed_detailed_project = {
61
+ project_id: project_id,
62
+ client_id: client_id,
63
+ segment: segment_id,
64
+ message: message,
65
+ action: failed_action
66
+ }
67
+ add_failed_detail(params, failed_detailed_project, sync_failed_list)
68
+ end
69
+ end
70
+
71
+ def add_failed_client(client_id, message, error_action, params)
72
+ if collect_synced_status(params) && !sync_failed_client(client_id, params)
73
+ sync_failed_list = sync_failed_list(params)
74
+ client_project_mappings = sync_failed_list[:client_project_mappings]
75
+ client_project_mapping = client_project_mappings ? client_project_mappings[client_id.to_sym] : nil
76
+ project_id = client_project_mapping ? client_project_mapping[:project_id] : nil
77
+ segment_id = client_project_mapping ? client_project_mapping[:segment_id] : nil
78
+
79
+ failed_detailed_client = {
80
+ project_id: project_id,
81
+ client_id: client_id,
82
+ segment: segment_id,
83
+ message: message,
84
+ action: error_action
85
+ }
86
+ add_failed_detail(params, failed_detailed_client, sync_failed_list)
87
+ end
88
+ end
89
+
90
+ def add_failed_segment(segment_id, message, error_action, params)
91
+ if collect_synced_status(params) && !sync_failed_segment(segment_id, params)
92
+ sync_failed_list = sync_failed_list(params)
93
+ failed_detailed_segment = {
94
+ project_id: nil,
95
+ client_id: nil,
96
+ segment: segment_id,
97
+ message: message,
98
+ action: error_action
99
+ }
100
+ add_failed_detail(params, failed_detailed_segment, sync_failed_list, true)
101
+ end
102
+ end
103
+
104
+ # Add new clients to project client mapping
105
+ #
106
+ # @param [String] project_id project identify will be added to mapping
107
+ # @param [String] client_id client identify will be added to mapping
108
+ # @param [String] segment_id segment identify will be added to mapping
109
+ # @param [Hash] params the hash contains list of parameters and values
110
+ def add_new_clients_to_project_client_mapping(project_id, client_id, segment_id, params)
111
+ if collect_synced_status(params)
112
+ sync_failed_list = sync_failed_list(params)
113
+ client_project_mappings = sync_failed_list[:client_project_mappings]
114
+ project_client_mappings = sync_failed_list[:project_client_mappings]
115
+ client_project_mappings[client_id.to_sym] = {
116
+ project_id: project_id,
117
+ segment_id: segment_id
118
+ } if client_project_mappings
119
+
120
+ project_client_mappings[project_id.to_sym] = {
121
+ client_id: client_id,
122
+ segment_id: segment_id
123
+ } if project_client_mappings
124
+ end
125
+ end
126
+
127
+ def process_failed_project(project_id, failed_message, failed_projects, continue_on_error)
128
+ fail(failed_message) unless continue_on_error
129
+
130
+ failed_projects << {
131
+ project_id: project_id,
132
+ message: failed_message
133
+ }
134
+ end
135
+
136
+ def process_failed_projects(failed_projects, failed_action, params)
137
+ failed_projects.each do |failed_project|
138
+ add_failed_project(failed_project[:project_id], failed_project[:message], failed_action, params)
139
+ end
140
+ end
141
+
142
+ def sync_failed_project(project_id, params)
143
+ collect_synced_status(params) && params[SYNC_FAILED_LIST][FAILED_PROJECTS].include?(project_id)
144
+ end
145
+
146
+ def sync_failed_client(client_id, params)
147
+ collect_synced_status(params) && params[SYNC_FAILED_LIST][FAILED_CLIENTS].include?(client_id)
148
+ end
149
+
150
+ def sync_failed_segment(segment_id, params)
151
+ collect_synced_status(params) && params[SYNC_FAILED_LIST][FAILED_SEGMENTS].include?(segment_id)
152
+ end
153
+
154
+ private
155
+
156
+ def add_failed_detail(params, failed_detailed_project, sync_failed_list, ignore_segment = false)
157
+ params.gdc_logger&.warn failed_detailed_project[:message]
158
+ sync_failed_list[:failed_detailed_projects] << failed_detailed_project
159
+
160
+ if ignore_segment
161
+ add_failed_detail_segment(failed_detailed_project[:segment_id], sync_failed_list)
162
+ else
163
+ add_failed_detail_client(failed_detailed_project[:client_id], failed_detailed_project[:project_id], sync_failed_list)
164
+ end
165
+ end
166
+
167
+ def sync_failed_list(params)
168
+ if params.include?(SYNC_FAILED_LIST)
169
+ params[SYNC_FAILED_LIST]
170
+ else
171
+ nil
172
+ end
173
+ end
174
+
175
+ def add_failed_detail_client(client_id, project_id, sync_failed_list)
176
+ sync_failed_list[FAILED_CLIENTS] << client_id if client_id
177
+
178
+ sync_failed_list[FAILED_PROJECTS] << project_id if project_id
179
+ end
180
+
181
+ def add_failed_detail_segment(segment_id, sync_failed_list)
182
+ if segment_id
183
+ sync_failed_list[FAILED_SEGMENTS] << segment_id
184
+
185
+ client_project_mappings = sync_failed_list[:client_project_mappings]
186
+ client_project_mappings.each do |client_id, client_project_mapping|
187
+ add_failed_detail_client(client_id, client_project_mapping[:project_id], sync_failed_list) if client_project_mapping[:segment_id] == segment_id
188
+ end
189
+ end
190
+ end
34
191
  end
35
192
  end
36
193
  end
@@ -47,8 +47,9 @@ module GoodData
47
47
  end
48
48
 
49
49
  begin
50
+ params.gdc_logger.info "Starting to find DataProduct ID: #{data_product_id}"
50
51
  data_product = domain.data_products(data_product_id)
51
- rescue RestClient::BadRequest
52
+ rescue RestClient::BadRequest, RestClient::NotFound
52
53
  params.gdc_logger.info "Can not find DataProduct #{params.data_product}, creating it instead"
53
54
  data_product = domain.create_data_product(id: params.data_product)
54
55
  end
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+ # (C) 2019-2022 GoodData Corporation
3
+ require_relative 'base_action'
4
+
5
+ module GoodData
6
+ module LCM2
7
+ class CollectProjectsWarningStatus < BaseAction
8
+ DESCRIPTION = 'Collect synced projects status'
9
+
10
+ PARAMS = define_params(self) do
11
+ description 'Abort on error'
12
+ param :abort_on_error, instance_of(Type::StringType), required: false
13
+
14
+ description 'Logger'
15
+ param :gdc_logger, instance_of(Type::GdLogger), required: false
16
+
17
+ description 'Collect synced status'
18
+ param :collect_synced_status, instance_of(Type::BooleanType), required: false
19
+
20
+ description 'Sync failed list'
21
+ param :sync_failed_list, instance_of(Type::HashType), required: false
22
+ end
23
+
24
+ RESULT_HEADER = %i[segment client project_pid failed_action]
25
+
26
+ class << self
27
+ def call(params)
28
+ results = []
29
+ return results unless collect_synced_status(params)
30
+
31
+ sync_failed_list = params[SYNC_FAILED_LIST]
32
+ if sync_failed_list
33
+ failed_detailed_projects = sync_failed_list[:failed_detailed_projects]
34
+ failed_detailed_projects.each do |failed_detailed_project|
35
+ results << {
36
+ segment: failed_detailed_project[:segment],
37
+ client: failed_detailed_project[:client_id],
38
+ project_pid: failed_detailed_project[:project_id],
39
+ failed_action: failed_detailed_project[:action]
40
+ }
41
+ end
42
+ end
43
+
44
+ results
45
+ end
46
+
47
+ def print_result(params)
48
+ collect_synced_status(params)
49
+ end
50
+ end
51
+ end
52
+ end
53
+ end
@@ -32,9 +32,22 @@ module GoodData
32
32
 
33
33
  description 'Domain'
34
34
  param :domain, instance_of(Type::StringType), required: false
35
+
36
+ description 'Abort on error'
37
+ param :abort_on_error, instance_of(Type::StringType), required: false
38
+
39
+ description 'Logger'
40
+ param :gdc_logger, instance_of(Type::GdLogger), required: false
41
+
42
+ description 'Collect synced status'
43
+ param :collect_synced_status, instance_of(Type::BooleanType), required: false
44
+
45
+ description 'Sync failed list'
46
+ param :sync_failed_list, instance_of(Type::HashType), required: false
35
47
  end
36
48
 
37
49
  RESULT_HEADER = [
50
+ :segment,
38
51
  :from_name,
39
52
  :from_pid,
40
53
  :to_name,
@@ -89,6 +102,7 @@ module GoodData
89
102
  client_project = segment_client.project
90
103
  to_pid = client_project.pid
91
104
  results << {
105
+ segment: segment.segment_id,
92
106
  from_name: master_name,
93
107
  from_pid: master_pid,
94
108
  to_name: client_project.title,
@@ -0,0 +1,87 @@
1
+ # frozen_string_literal: true
2
+ # (C) 2019-2022 GoodData Corporation
3
+ require_relative 'base_action'
4
+
5
+ module GoodData
6
+ module LCM2
7
+ class InitializeContinueOnErrorOption < BaseAction
8
+ DESCRIPTION = 'Initialize continue on error option'
9
+
10
+ PARAMS = define_params(self) do
11
+ description 'Organization Name'
12
+ param :organization, instance_of(Type::StringType), required: false
13
+
14
+ description 'Domain'
15
+ param :domain, instance_of(Type::StringType), required: false
16
+
17
+ description 'DataProduct'
18
+ param :data_product, instance_of(Type::GDDataProductType), required: false
19
+
20
+ description 'Restricts synchronization to specified segments'
21
+ param :segments_filter, array_of(instance_of(Type::StringType)), required: false
22
+
23
+ description 'Abort on error'
24
+ param :abort_on_error, instance_of(Type::StringType), required: false
25
+
26
+ description 'Client Used for Connecting to GD'
27
+ param :gdc_gd_client, instance_of(Type::GdClientType), required: true
28
+ end
29
+
30
+ class << self
31
+ def call(params)
32
+ project_mappings = ThreadSafe::Hash.new
33
+ client_mappings = ThreadSafe::Hash.new
34
+ continue_on_error = continue_on_error(params)
35
+
36
+ if continue_on_error
37
+ client = params.gdc_gd_client
38
+ domain_name = params.organization || params.domain
39
+ domain = client.domain(domain_name)
40
+ data_product = params.data_product
41
+ data_product_segments = domain.segments(:all, data_product)
42
+
43
+ if params.segments_filter
44
+ data_product_segments.select! do |segment|
45
+ params.segments_filter.include?(segment.segment_id)
46
+ end
47
+ end
48
+
49
+ data_product_segments.pmap do |segment|
50
+ segment.clients.map do |segment_client|
51
+ project = segment_client.project
52
+ next unless project
53
+
54
+ project_mappings[project.pid.to_sym] = {
55
+ client_id: segment_client.client_id,
56
+ segment_id: segment.segment_id
57
+ }
58
+ client_mappings[segment_client.client_id.to_sym] = {
59
+ project_id: project.pid,
60
+ segment_id: segment.segment_id
61
+ }
62
+ end
63
+ end
64
+ end
65
+
66
+ {
67
+ params: {
68
+ collect_synced_status: continue_on_error,
69
+ sync_failed_list: {
70
+ project_client_mappings: project_mappings,
71
+ client_project_mappings: client_mappings,
72
+ failed_detailed_projects: [],
73
+ failed_projects: [],
74
+ failed_clients: [],
75
+ failed_segments: []
76
+ }
77
+ }
78
+ }
79
+ end
80
+
81
+ def print_result(_params)
82
+ false
83
+ end
84
+ end
85
+ end
86
+ end
87
+ end