gooddata 1.3.3-java → 1.3.4-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +129 -67
- data/CHANGELOG.md +31 -0
- data/CONTRIBUTING.md +16 -5
- data/Dockerfile +4 -4
- data/Dockerfile.jruby +0 -2
- data/README.md +1 -1
- data/SDK_VERSION +1 -0
- data/VERSION +1 -1
- data/bin/run_brick.rb +14 -3
- data/docker-compose.yml +1 -0
- data/gooddata.gemspec +6 -4
- data/lcm.rake +13 -25
- data/lib/gooddata.rb +4 -0
- data/lib/gooddata/bricks/middleware/aws_middleware.rb +1 -1
- data/lib/gooddata/bricks/middleware/bench_middleware.rb +3 -3
- data/lib/gooddata/bricks/middleware/dwh_middleware.rb +1 -1
- data/lib/gooddata/bricks/middleware/fs_download_middleware.rb +1 -1
- data/lib/gooddata/bricks/middleware/fs_upload_middleware.rb +1 -1
- data/lib/gooddata/bricks/middleware/gooddata_middleware.rb +4 -8
- data/lib/gooddata/bricks/middleware/logger_middleware.rb +11 -2
- data/lib/gooddata/bricks/middleware/mask_logger_decorator.rb +38 -0
- data/lib/gooddata/cli/cli.rb +0 -1
- data/lib/gooddata/cli/commands/domain_cmd.rb +0 -1
- data/lib/gooddata/cli/commands/project_cmd.rb +0 -1
- data/lib/gooddata/cli/hooks.rb +0 -1
- data/lib/gooddata/connection.rb +1 -1
- data/lib/gooddata/extensions/object.rb +0 -4
- data/lib/gooddata/helpers/global_helpers.rb +3 -3
- data/lib/gooddata/lcm/actions/hello_world.rb +1 -1
- data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +6 -4
- data/lib/gooddata/lcm/actions/synchronize_users.rb +38 -75
- data/lib/gooddata/lcm/brick_logger.rb +6 -6
- data/lib/gooddata/lcm/dsl/dsl.rb +0 -7
- data/lib/gooddata/lcm/lcm.rb +6 -6
- data/lib/gooddata/lcm/lcm2.rb +7 -23
- data/lib/gooddata/mixins/inspector.rb +0 -1
- data/lib/gooddata/models/process.rb +9 -5
- data/lib/gooddata/models/project.rb +13 -16
- data/lib/gooddata/models/project_creator.rb +2 -2
- data/lib/gooddata/models/schedule.rb +5 -2
- data/lib/gooddata/models/user_filters/user_filter_builder.rb +2 -0
- data/lib/gooddata/rest/client.rb +3 -3
- data/lib/gooddata/rest/connection.rb +3 -3
- data/lib/gooddata/version.rb +12 -2
- metadata +14 -34
data/gooddata.gemspec
CHANGED
@@ -31,11 +31,11 @@ Gem::Specification.new do |s|
|
|
31
31
|
s.homepage = 'http://github.com/gooddata/gooddata-ruby'
|
32
32
|
s.require_paths = ['lib']
|
33
33
|
|
34
|
-
s.add_development_dependency 'bundler', "~> 1.14"
|
34
|
+
# s.add_development_dependency 'bundler', "~> 1.14"
|
35
35
|
s.add_development_dependency 'license_finder', '~> 2.0'
|
36
36
|
s.add_development_dependency 'rake', '~> 11.1'
|
37
37
|
s.add_development_dependency 'redcarpet', '~> 3.1' if RUBY_PLATFORM != 'java'
|
38
|
-
s.add_development_dependency 'rspec', '~> 3.5'
|
38
|
+
s.add_development_dependency 'rspec', '~> 3.5.0'
|
39
39
|
s.add_development_dependency 'rspec-expectations', '~> 3.5'
|
40
40
|
s.add_development_dependency 'rspec_junit_formatter', '~> 0.3.0'
|
41
41
|
s.add_development_dependency 'rubocop', '~> 0.59.1'
|
@@ -43,7 +43,6 @@ Gem::Specification.new do |s|
|
|
43
43
|
s.add_development_dependency 'webmock', '~> 2.3.1'
|
44
44
|
s.add_development_dependency 'yard', '~> 0.9.11'
|
45
45
|
s.add_development_dependency 'yard-rspec', '~> 0.1'
|
46
|
-
s.add_development_dependency 'ZenTest', '~> 4.11'
|
47
46
|
s.add_development_dependency 'pry'
|
48
47
|
s.add_development_dependency 'pry-byebug', '~> 3.6' if RUBY_PLATFORM != 'java'
|
49
48
|
|
@@ -53,7 +52,10 @@ Gem::Specification.new do |s|
|
|
53
52
|
s.add_development_dependency 'pronto-flay', '~> 0.9.0' if RUBY_PLATFORM != 'java'
|
54
53
|
s.add_development_dependency 'vcr'
|
55
54
|
|
56
|
-
s.
|
55
|
+
s.add_development_dependency 'sqlite3' if RUBY_PLATFORM != 'java'
|
56
|
+
|
57
|
+
s.add_dependency 'activesupport', '> 4.2.9', '< 5.2'
|
58
|
+
|
57
59
|
s.add_dependency 'aws-sdk-s3', '~> 1.16'
|
58
60
|
s.add_dependency 'docile', '~> 1.1'
|
59
61
|
s.add_dependency 'erubis', '~> 2.7'
|
data/lcm.rake
CHANGED
@@ -3,6 +3,8 @@ require 'fileutils'
|
|
3
3
|
require 'pathname'
|
4
4
|
require 'rspec/core/rake_task'
|
5
5
|
|
6
|
+
test_cases = %i[integration slow userprov load smoke]
|
7
|
+
|
6
8
|
# Schema for new Bricks.
|
7
9
|
brick_info_schema = {
|
8
10
|
"type" => "object",
|
@@ -74,34 +76,20 @@ end
|
|
74
76
|
|
75
77
|
RSpec::Core::RakeTask.new(:test)
|
76
78
|
namespace :test do
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
desc 'Run load tests'
|
83
|
-
RSpec::Core::RakeTask.new(:load) do |t|
|
84
|
-
t.pattern = 'spec/lcm/load/**/*.rb'
|
85
|
-
end
|
86
|
-
|
87
|
-
desc 'Run slow tests'
|
88
|
-
RSpec::Core::RakeTask.new(:slow) do |t|
|
89
|
-
t.pattern = 'spec/lcm/slow/**/*.rb'
|
90
|
-
end
|
91
|
-
|
92
|
-
namespace :integration do
|
93
|
-
desc 'Run integration tests in Docker'
|
94
|
-
task :docker do
|
95
|
-
system('docker-compose -f docker-compose.lcm.yml up --force-recreate --abort-on-container-exit --exit-code-from appstore appstore') ||
|
96
|
-
fail('Test execution failed!')
|
79
|
+
test_cases.each do |test_case|
|
80
|
+
desc "Run #{test_case} tests"
|
81
|
+
RSpec::Core::RakeTask.new(test_case) do |task|
|
82
|
+
task.pattern = "spec/lcm/#{test_case}/**/*.rb"
|
97
83
|
end
|
98
84
|
end
|
99
|
-
namespace :load do
|
100
|
-
desc 'Run load tests in Docker'
|
101
|
-
task :docker do
|
102
|
-
system('docker-compose -f docker-compose.lcm.yml run --rm appstore bundle exec rake -f lcm.rake test:load')
|
103
85
|
|
104
|
-
|
86
|
+
namespace :docker do
|
87
|
+
test_cases.each do |t|
|
88
|
+
desc "Run #{t} tests in Docker"
|
89
|
+
task t do
|
90
|
+
system("docker-compose -f docker-compose.lcm.yml run --rm appstore bundle exec rake -f lcm.rake test:#{t}") ||
|
91
|
+
fail('Test execution failed!')
|
92
|
+
end
|
105
93
|
end
|
106
94
|
end
|
107
95
|
end
|
data/lib/gooddata.rb
CHANGED
@@ -22,6 +22,7 @@ require_relative 'gooddata/lcm/lcm'
|
|
22
22
|
require_relative 'gooddata/lcm/lcm2'
|
23
23
|
require_relative 'gooddata/models/models'
|
24
24
|
require_relative 'gooddata/bricks/pipeline'
|
25
|
+
require 'gooddata/extensions/object'
|
25
26
|
|
26
27
|
# Files
|
27
28
|
require_relative 'gooddata/app/app'
|
@@ -33,3 +34,6 @@ require_relative 'gooddata/version'
|
|
33
34
|
# Extensions
|
34
35
|
require_relative 'gooddata/extensions/extensions'
|
35
36
|
require 'backports/2.1.0/array/to_h'
|
37
|
+
|
38
|
+
# Helpers
|
39
|
+
require 'gooddata/helpers/global_helpers'
|
@@ -13,7 +13,7 @@ module GoodData
|
|
13
13
|
def call(params)
|
14
14
|
params = params.to_hash
|
15
15
|
if params.key?('aws_client')
|
16
|
-
|
16
|
+
GoodData.logger.info('Setting up AWS-S3 connection')
|
17
17
|
raise 'Unable to connect to AWS. Parameter "aws_client" seems to be empty' unless params['aws_client']
|
18
18
|
raise 'Unable to connect to AWS. Parameter "access_key_id" is missing' if params['aws_client']['access_key_id'].blank?
|
19
19
|
raise 'Unable to connect to AWS. Parameter "secret_access_key" is missing' if params['aws_client']['secret_access_key'].blank?
|
@@ -13,11 +13,11 @@ module GoodData
|
|
13
13
|
class BenchMiddleware < Bricks::Middleware
|
14
14
|
def call(params)
|
15
15
|
params = params.to_hash
|
16
|
-
|
16
|
+
GoodData.logger.info('Starting timer')
|
17
17
|
result = nil
|
18
18
|
report = Benchmark.measure { result = @app.call(params) }
|
19
|
-
|
20
|
-
|
19
|
+
GoodData.logger.info('Stopping timer')
|
20
|
+
GoodData.logger.info(report.pretty_inspect)
|
21
21
|
result
|
22
22
|
end
|
23
23
|
end
|
@@ -13,7 +13,7 @@ module GoodData
|
|
13
13
|
# Connects to the warehouse (ADS) and enriches parameters with GoodData::Datawarehouse
|
14
14
|
class WarehouseMiddleware < Bricks::Middleware
|
15
15
|
def call(params)
|
16
|
-
if params.key?('ads_client')
|
16
|
+
if params.key?('ads_client')
|
17
17
|
GoodData.logger.info "Setting up ADS connection to #{params['ads_client']['ads_id'] || params['ads_client']['jdbc_url']}"
|
18
18
|
unless params['ads_client']['username'] || params['GDC_USERNAME']
|
19
19
|
raise "ADS middleware needs username either as part of " \
|
@@ -33,7 +33,7 @@ module GoodData
|
|
33
33
|
dav.verify_server = false
|
34
34
|
dav.credentials(params['GDC_USERNAME'], params['GDC_PASSWORD'])
|
35
35
|
dav.find(path, recursive: true, suppress_errors: true) do |item|
|
36
|
-
|
36
|
+
GoodData.logger.info('Checking: ' + item.url.to_s)
|
37
37
|
name = (item.uri - webdav_uri).to_s
|
38
38
|
File.open(name, 'w') do |f|
|
39
39
|
f << item.content
|
@@ -25,7 +25,7 @@ module GoodData
|
|
25
25
|
GoodData.client.get '/gdc/account/token', dont_reauth: true
|
26
26
|
url = GoodData.project_webdav_path
|
27
27
|
GoodData.upload_to_project_webdav(path)
|
28
|
-
|
28
|
+
GoodData.logger.info("Uploaded local file \"#{path}\" to url \"#{url + path}\"")
|
29
29
|
end
|
30
30
|
end
|
31
31
|
end
|
@@ -33,10 +33,6 @@ module GoodData
|
|
33
33
|
params = GoodData::Helpers.stringify_keys(params)
|
34
34
|
# params = GoodData::Helpers.symbolize_keys(params)
|
35
35
|
|
36
|
-
# Set logger
|
37
|
-
logger = params['GDC_LOGGER']
|
38
|
-
GoodData.logger = logger
|
39
|
-
|
40
36
|
# Set parallelism
|
41
37
|
max_concurrency = params['max_concurrency'] || params['MAX_CONCURRENCY']
|
42
38
|
if max_concurrency && max_concurrency.to_i > 0
|
@@ -91,14 +87,14 @@ module GoodData
|
|
91
87
|
begin
|
92
88
|
client.disconnect
|
93
89
|
rescue
|
94
|
-
|
90
|
+
GoodData.logger.warn('Tried to disconnect client. Was unsuccessful. Proceeding anyway.')
|
95
91
|
end
|
96
92
|
|
97
93
|
# Try to disconnect development_client
|
98
94
|
begin
|
99
95
|
development_client.disconnect if development_client != client
|
100
96
|
rescue
|
101
|
-
|
97
|
+
GoodData.logger.warn('Tried to disconnect development_client. Was unsuccessful. Proceeding anyway.')
|
102
98
|
end
|
103
99
|
|
104
100
|
returning_value
|
@@ -107,11 +103,11 @@ module GoodData
|
|
107
103
|
class << self
|
108
104
|
def connect(server, verify_ssl, username, password, sst_token) # rubocop:disable Metrics/ParameterLists
|
109
105
|
if username.nil? || password.nil?
|
110
|
-
|
106
|
+
GoodData.logger.info("Connecting with SST to server #{server}")
|
111
107
|
raise 'SST (SuperSecureToken) not present in params' if sst_token.nil?
|
112
108
|
conn = GoodData.connect(sst_token: sst_token, server: server, verify_ssl: verify_ssl)
|
113
109
|
else
|
114
|
-
|
110
|
+
GoodData.logger.info("Connecting as #{username} to server #{server}")
|
115
111
|
conn = GoodData.connect(username, password, server: server, verify_ssl: verify_ssl)
|
116
112
|
end
|
117
113
|
conn.stats_on
|
@@ -19,20 +19,29 @@ using StringExtensions
|
|
19
19
|
using NilExtensions
|
20
20
|
|
21
21
|
require_relative 'base_middleware'
|
22
|
+
require_relative 'mask_logger_decorator'
|
22
23
|
|
23
24
|
module GoodData
|
24
25
|
module Bricks
|
25
26
|
class LoggerMiddleware < Bricks::Middleware
|
26
27
|
def call(params)
|
27
28
|
params = params.to_hash
|
28
|
-
logger = nil
|
29
29
|
if params['GDC_LOGGING_OFF']
|
30
30
|
logger = NilLogger.new
|
31
|
+
elsif params['GDC_LOG_DIRECTORY'] && params['GDC_EXECUTION_ID']
|
32
|
+
log_directory = params['GDC_LOG_DIRECTORY']
|
33
|
+
execution_id = params['GDC_EXECUTION_ID']
|
34
|
+
FileUtils.mkpath log_directory
|
35
|
+
logger = Logger.new("#{log_directory}/#{execution_id}.log")
|
36
|
+
logger.level = params['GDC_LOG_LEVEL'] || 'info'
|
37
|
+
values_to_mask = params['values_to_mask'] || []
|
38
|
+
logger = MaskLoggerDecorator.new(logger, values_to_mask) if values_to_mask.any?
|
31
39
|
else
|
32
40
|
logger = params[:GDC_LOGGER_FILE].nil? ? Logger.new(STDOUT) : Logger.new(params[:GDC_LOGGER_FILE])
|
33
41
|
logger.level = params['GDC_LOG_LEVEL'] || 'info'
|
34
|
-
logger.info('Pipeline starts')
|
35
42
|
end
|
43
|
+
GoodData.logger = logger
|
44
|
+
logger.info('Pipeline starts')
|
36
45
|
params['GDC_LOGGER'] = logger
|
37
46
|
GoodData.logging_http_on if params['HTTP_LOGGING'] && params['HTTP_LOGGING'].to_b
|
38
47
|
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# Copyright (c) 2010-2018 GoodData Corporation. All rights reserved.
|
2
|
+
# This source code is licensed under the BSD-style license found in the
|
3
|
+
# LICENSE file in the root directory of this source tree.
|
4
|
+
|
5
|
+
module GoodData
|
6
|
+
module Bricks
|
7
|
+
# Logger decorator with ability to mask sensitive values
|
8
|
+
class MaskLoggerDecorator
|
9
|
+
# entry-point
|
10
|
+
# @param [Logger] logger logger to decorated
|
11
|
+
# @param [Array] values_to_mask sensitive values to be masked out from logs
|
12
|
+
def initialize(logger, values_to_mask = [])
|
13
|
+
@logger = logger
|
14
|
+
@values_to_mask = values_to_mask
|
15
|
+
end
|
16
|
+
|
17
|
+
# log methods to be decorated
|
18
|
+
%i[debug error fatal info unknown warn].each do |level|
|
19
|
+
define_method level do |message|
|
20
|
+
@logger.send(level, mask(message))
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
# Masks given message.
|
27
|
+
# @param message [String] message to mask
|
28
|
+
# @return masked_message [String] masked message
|
29
|
+
def mask(message)
|
30
|
+
unless message.nil?
|
31
|
+
@values_to_mask.reduce(message) do |masked_message, value_to_mask|
|
32
|
+
masked_message.gsub(value_to_mask, "******")
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
data/lib/gooddata/cli/cli.rb
CHANGED
data/lib/gooddata/cli/hooks.rb
CHANGED
data/lib/gooddata/connection.rb
CHANGED
@@ -37,7 +37,7 @@ module GoodData
|
|
37
37
|
|
38
38
|
class << self
|
39
39
|
def error(msg)
|
40
|
-
|
40
|
+
GoodData.logger.error(msg)
|
41
41
|
exit 1
|
42
42
|
end
|
43
43
|
|
@@ -140,7 +140,7 @@ module GoodData
|
|
140
140
|
|
141
141
|
if full_outer
|
142
142
|
(lookup.keys - marked_lookup.keys).each do |key|
|
143
|
-
|
143
|
+
GoodData.logger.info(lookup[key])
|
144
144
|
results << lookup[key].first.to_hash
|
145
145
|
end
|
146
146
|
end
|
@@ -232,7 +232,7 @@ module GoodData
|
|
232
232
|
|
233
233
|
def decrypt(database64, key)
|
234
234
|
if key.nil? || key.empty?
|
235
|
-
|
235
|
+
GoodData.logger.warn('WARNING: No encryption key provided.')
|
236
236
|
return 'no_key_provided'
|
237
237
|
end
|
238
238
|
|
@@ -120,8 +120,9 @@ module GoodData
|
|
120
120
|
dry_run: params[:dry_run].to_b,
|
121
121
|
users_brick_input: params.users_brick_users
|
122
122
|
}
|
123
|
+
all_clients = domain.clients(:all, data_product).to_a
|
123
124
|
|
124
|
-
|
125
|
+
GoodData.logger.info("Synchronizing in mode \"#{mode}\"")
|
125
126
|
case mode
|
126
127
|
when 'sync_project', 'sync_one_project_based_on_pid', 'sync_one_project_based_on_custom_id'
|
127
128
|
if mode == 'sync_one_project_based_on_pid'
|
@@ -138,14 +139,15 @@ module GoodData
|
|
138
139
|
fail "The #{multiple_projects_column} cannot be empty" if id.blank?
|
139
140
|
|
140
141
|
if mode == 'sync_multiple_projects_based_on_custom_id'
|
141
|
-
|
142
|
+
c = all_clients.detect { |specific_client| specific_client.id == id }
|
143
|
+
current_project = c.project
|
142
144
|
elsif mode == 'sync_multiple_projects_based_on_pid'
|
143
145
|
current_project = client.projects(id)
|
144
146
|
end
|
145
147
|
sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
|
146
148
|
end
|
147
149
|
when 'sync_domain_client_workspaces'
|
148
|
-
domain_clients =
|
150
|
+
domain_clients = all_clients
|
149
151
|
if params.segments
|
150
152
|
segment_uris = params.segments.map(&:uri)
|
151
153
|
domain_clients = domain_clients.select { |c| segment_uris.include?(c.segment_uri) }
|
@@ -159,7 +161,7 @@ module GoodData
|
|
159
161
|
users = users_by_project[client_id]
|
160
162
|
fail "Client id cannot be empty" if client_id.blank?
|
161
163
|
|
162
|
-
c =
|
164
|
+
c = all_clients.detect { |specific_client| specific_client.id == client_id }
|
163
165
|
if params.segments && !segment_uris.include?(c.segment_uri)
|
164
166
|
params.gdc_logger.warn "Client #{client_id} is outside segments_filter #{params.segments}"
|
165
167
|
next
|
@@ -181,6 +181,14 @@ module GoodData
|
|
181
181
|
# aiming at solving the problem that the customer cannot give us the
|
182
182
|
# value of a project id in the data since he does not know it upfront
|
183
183
|
# and we cannot influence its value.
|
184
|
+
common_params = {
|
185
|
+
domain: domain,
|
186
|
+
whitelists: whitelists,
|
187
|
+
ignore_failures: ignore_failures,
|
188
|
+
remove_users_from_project: remove_users_from_project,
|
189
|
+
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
190
|
+
create_non_existing_user_groups: create_non_existing_user_groups
|
191
|
+
}
|
184
192
|
results = case mode
|
185
193
|
when 'add_to_organization'
|
186
194
|
domain.create_users(new_users.uniq { |u| u[:login] || u[:email] })
|
@@ -190,24 +198,12 @@ module GoodData
|
|
190
198
|
params.gdc_logger.warn "Deleting #{users.count} users from domain #{domain_name}"
|
191
199
|
users.map(&:delete)
|
192
200
|
when 'sync_project'
|
193
|
-
project.import_users(new_users,
|
194
|
-
domain: domain,
|
195
|
-
whitelists: whitelists,
|
196
|
-
ignore_failures: ignore_failures,
|
197
|
-
remove_users_from_project: remove_users_from_project,
|
198
|
-
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
199
|
-
create_non_existing_user_groups: create_non_existing_user_groups)
|
201
|
+
project.import_users(new_users, common_params)
|
200
202
|
when 'sync_multiple_projects_based_on_pid'
|
201
203
|
new_users.group_by { |u| u[:pid] }.flat_map do |project_id, users|
|
202
204
|
begin
|
203
205
|
project = client.projects(project_id)
|
204
|
-
project.import_users(users,
|
205
|
-
domain: domain,
|
206
|
-
whitelists: whitelists,
|
207
|
-
ignore_failures: ignore_failures,
|
208
|
-
remove_users_from_project: remove_users_from_project,
|
209
|
-
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
210
|
-
create_non_existing_user_groups: create_non_existing_user_groups)
|
206
|
+
project.import_users(users, common_params)
|
211
207
|
rescue RestClient::ResourceNotFound
|
212
208
|
fail "Project \"#{project_id}\" was not found. Please check your project ids in the source file"
|
213
209
|
rescue RestClient::Gone
|
@@ -218,13 +214,7 @@ module GoodData
|
|
218
214
|
end
|
219
215
|
when 'sync_one_project_based_on_pid'
|
220
216
|
filtered_users = new_users.select { |u| u[:pid] == project.pid }
|
221
|
-
project.import_users(filtered_users,
|
222
|
-
domain: domain,
|
223
|
-
whitelists: whitelists,
|
224
|
-
ignore_failures: ignore_failures,
|
225
|
-
remove_users_from_project: remove_users_from_project,
|
226
|
-
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
227
|
-
create_non_existing_user_groups: create_non_existing_user_groups)
|
217
|
+
project.import_users(filtered_users, common_params)
|
228
218
|
when 'sync_one_project_based_on_custom_id'
|
229
219
|
filter_value = UserBricksHelper.resolve_client_id(domain, project, data_product)
|
230
220
|
|
@@ -243,32 +233,21 @@ module GoodData
|
|
243
233
|
)
|
244
234
|
end
|
245
235
|
|
246
|
-
|
247
|
-
project.import_users(filtered_users,
|
248
|
-
domain: domain,
|
249
|
-
whitelists: whitelists,
|
250
|
-
ignore_failures: ignore_failures,
|
251
|
-
remove_users_from_project: remove_users_from_project,
|
252
|
-
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
253
|
-
create_non_existing_user_groups: create_non_existing_user_groups)
|
236
|
+
GoodData.logger.info("Project #{project.pid} will receive #{filtered_users.count} from #{new_users.count} users")
|
237
|
+
project.import_users(filtered_users, common_params)
|
254
238
|
when 'sync_multiple_projects_based_on_custom_id'
|
239
|
+
all_clients = domain.clients(:all, data_product).to_a
|
255
240
|
new_users.group_by { |u| u[:pid] }.flat_map do |client_id, users|
|
256
241
|
fail "Client id cannot be empty" if client_id.blank?
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
end
|
242
|
+
|
243
|
+
c = all_clients.detect { |specific_client| specific_client.id == client_id }
|
244
|
+
fail "The client \"#{client_id}\" does not exist in data product \"#{data_product.data_product_id}\"" if c.nil?
|
245
|
+
|
246
|
+
project = c.project
|
263
247
|
fail "Client #{client_id} does not have project." unless project
|
264
|
-
|
265
|
-
project.
|
266
|
-
|
267
|
-
whitelists: whitelists,
|
268
|
-
ignore_failures: ignore_failures,
|
269
|
-
remove_users_from_project: remove_users_from_project,
|
270
|
-
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
271
|
-
create_non_existing_user_groups: create_non_existing_user_groups)
|
248
|
+
|
249
|
+
GoodData.logger.info("Project #{project.pid} of client #{client_id} will receive #{users.count} users")
|
250
|
+
project.import_users(users, common_params)
|
272
251
|
end
|
273
252
|
when 'sync_domain_client_workspaces'
|
274
253
|
domain_clients = domain.clients(:all, data_product)
|
@@ -280,22 +259,18 @@ module GoodData
|
|
280
259
|
res = []
|
281
260
|
res += new_users.group_by { |u| u[:pid] }.flat_map do |client_id, users|
|
282
261
|
fail "Client id cannot be empty" if client_id.blank?
|
283
|
-
|
262
|
+
|
263
|
+
c = domain_clients.detect { |specific_client| specific_client.id == client_id }
|
284
264
|
if params.segments && !segment_uris.include?(c.segment_uri)
|
285
|
-
|
265
|
+
GoodData.logger.info("Client #{client_id} is outside segments_filter #{params.segments}")
|
286
266
|
next
|
287
267
|
end
|
288
268
|
project = c.project
|
289
269
|
fail "Client #{client_id} does not have project." unless project
|
270
|
+
|
290
271
|
working_client_ids << client_id.to_s
|
291
|
-
|
292
|
-
project.import_users(users,
|
293
|
-
domain: domain,
|
294
|
-
whitelists: whitelists,
|
295
|
-
ignore_failures: ignore_failures,
|
296
|
-
remove_users_from_project: remove_users_from_project,
|
297
|
-
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
298
|
-
create_non_existing_user_groups: create_non_existing_user_groups)
|
272
|
+
GoodData.logger.info("Project #{project.pid} of client #{client_id} will receive #{users.count} users")
|
273
|
+
project.import_users(users, common_params)
|
299
274
|
end
|
300
275
|
|
301
276
|
params.gdc_logger.debug("Working client ids are: #{working_client_ids.join(', ')}")
|
@@ -306,51 +281,39 @@ module GoodData
|
|
306
281
|
begin
|
307
282
|
project = c.project
|
308
283
|
rescue => e
|
309
|
-
|
284
|
+
GoodData.logger.error("Error when accessing project of client #{c.client_id}. Error: #{e}")
|
310
285
|
next
|
311
286
|
end
|
312
287
|
unless project
|
313
|
-
|
288
|
+
GoodData.logger.info("Client #{c.client_id} has no project.")
|
314
289
|
next
|
315
290
|
end
|
316
291
|
if project.deleted?
|
317
|
-
|
292
|
+
GoodData.logger.info("Project #{project.pid} of client #{c.client_id} is deleted.")
|
318
293
|
next
|
319
294
|
end
|
320
|
-
|
321
|
-
res += project.import_users([],
|
322
|
-
domain: domain,
|
323
|
-
whitelists: whitelists,
|
324
|
-
ignore_failures: ignore_failures,
|
325
|
-
remove_users_from_project: remove_users_from_project,
|
326
|
-
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
327
|
-
create_non_existing_user_groups: create_non_existing_user_groups)
|
295
|
+
GoodData.logger.info("Synchronizing all users in project #{project.pid} of client #{c.client_id}")
|
296
|
+
res += project.import_users([], common_params)
|
328
297
|
end
|
329
298
|
end
|
330
299
|
|
331
300
|
res
|
332
301
|
when 'sync_domain_and_project'
|
333
302
|
domain.create_users(new_users, ignore_failures: ignore_failures)
|
334
|
-
project.import_users(new_users,
|
335
|
-
domain: domain,
|
336
|
-
whitelists: whitelists,
|
337
|
-
ignore_failures: ignore_failures,
|
338
|
-
remove_users_from_project: remove_users_from_project,
|
339
|
-
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
340
|
-
create_non_existing_user_groups: create_non_existing_user_groups)
|
303
|
+
project.import_users(new_users, common_params)
|
341
304
|
end
|
342
305
|
|
343
306
|
results.compact!
|
344
307
|
counts = results.group_by { |r| r[:type] }.map { |g, r| [g, r.count] }
|
345
308
|
counts.each do |category, count|
|
346
|
-
|
309
|
+
GoodData.logger.info("There were #{count} events of type #{category}")
|
347
310
|
end
|
348
311
|
errors = results.select { |r| r[:type] == :error || r[:type] == :failed }
|
349
312
|
return if errors.empty?
|
350
313
|
|
351
|
-
|
352
|
-
|
353
|
-
|
314
|
+
GoodData.logger.info('Printing 10 first errors')
|
315
|
+
GoodData.logger.info('========================')
|
316
|
+
GoodData.logger.info(errors.take(10).pretty_inspect)
|
354
317
|
fail 'There was an error syncing users'
|
355
318
|
end
|
356
319
|
|