gooddata 1.3.3 → 1.3.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.travis.yml +129 -67
- data/CHANGELOG.md +31 -0
- data/CONTRIBUTING.md +16 -5
- data/Dockerfile +4 -4
- data/Dockerfile.jruby +0 -2
- data/README.md +1 -1
- data/SDK_VERSION +1 -0
- data/VERSION +1 -1
- data/bin/run_brick.rb +14 -3
- data/docker-compose.yml +1 -0
- data/gooddata.gemspec +6 -4
- data/lcm.rake +13 -25
- data/lib/gooddata.rb +4 -0
- data/lib/gooddata/bricks/middleware/aws_middleware.rb +1 -1
- data/lib/gooddata/bricks/middleware/bench_middleware.rb +3 -3
- data/lib/gooddata/bricks/middleware/dwh_middleware.rb +1 -1
- data/lib/gooddata/bricks/middleware/fs_download_middleware.rb +1 -1
- data/lib/gooddata/bricks/middleware/fs_upload_middleware.rb +1 -1
- data/lib/gooddata/bricks/middleware/gooddata_middleware.rb +4 -8
- data/lib/gooddata/bricks/middleware/logger_middleware.rb +11 -2
- data/lib/gooddata/bricks/middleware/mask_logger_decorator.rb +38 -0
- data/lib/gooddata/cli/cli.rb +0 -1
- data/lib/gooddata/cli/commands/domain_cmd.rb +0 -1
- data/lib/gooddata/cli/commands/project_cmd.rb +0 -1
- data/lib/gooddata/cli/hooks.rb +0 -1
- data/lib/gooddata/connection.rb +1 -1
- data/lib/gooddata/extensions/object.rb +0 -4
- data/lib/gooddata/helpers/global_helpers.rb +3 -3
- data/lib/gooddata/lcm/actions/hello_world.rb +1 -1
- data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +6 -4
- data/lib/gooddata/lcm/actions/synchronize_users.rb +38 -75
- data/lib/gooddata/lcm/brick_logger.rb +6 -6
- data/lib/gooddata/lcm/dsl/dsl.rb +0 -7
- data/lib/gooddata/lcm/lcm.rb +6 -6
- data/lib/gooddata/lcm/lcm2.rb +7 -23
- data/lib/gooddata/mixins/inspector.rb +0 -1
- data/lib/gooddata/models/process.rb +9 -5
- data/lib/gooddata/models/project.rb +13 -16
- data/lib/gooddata/models/project_creator.rb +2 -2
- data/lib/gooddata/models/schedule.rb +5 -2
- data/lib/gooddata/models/user_filters/user_filter_builder.rb +2 -0
- data/lib/gooddata/rest/client.rb +3 -3
- data/lib/gooddata/rest/connection.rb +3 -3
- data/lib/gooddata/version.rb +12 -2
- metadata +29 -35
data/gooddata.gemspec
CHANGED
@@ -31,11 +31,11 @@ Gem::Specification.new do |s|
|
|
31
31
|
s.homepage = 'http://github.com/gooddata/gooddata-ruby'
|
32
32
|
s.require_paths = ['lib']
|
33
33
|
|
34
|
-
s.add_development_dependency 'bundler', "~> 1.14"
|
34
|
+
# s.add_development_dependency 'bundler', "~> 1.14"
|
35
35
|
s.add_development_dependency 'license_finder', '~> 2.0'
|
36
36
|
s.add_development_dependency 'rake', '~> 11.1'
|
37
37
|
s.add_development_dependency 'redcarpet', '~> 3.1' if RUBY_PLATFORM != 'java'
|
38
|
-
s.add_development_dependency 'rspec', '~> 3.5'
|
38
|
+
s.add_development_dependency 'rspec', '~> 3.5.0'
|
39
39
|
s.add_development_dependency 'rspec-expectations', '~> 3.5'
|
40
40
|
s.add_development_dependency 'rspec_junit_formatter', '~> 0.3.0'
|
41
41
|
s.add_development_dependency 'rubocop', '~> 0.59.1'
|
@@ -43,7 +43,6 @@ Gem::Specification.new do |s|
|
|
43
43
|
s.add_development_dependency 'webmock', '~> 2.3.1'
|
44
44
|
s.add_development_dependency 'yard', '~> 0.9.11'
|
45
45
|
s.add_development_dependency 'yard-rspec', '~> 0.1'
|
46
|
-
s.add_development_dependency 'ZenTest', '~> 4.11'
|
47
46
|
s.add_development_dependency 'pry'
|
48
47
|
s.add_development_dependency 'pry-byebug', '~> 3.6' if RUBY_PLATFORM != 'java'
|
49
48
|
|
@@ -53,7 +52,10 @@ Gem::Specification.new do |s|
|
|
53
52
|
s.add_development_dependency 'pronto-flay', '~> 0.9.0' if RUBY_PLATFORM != 'java'
|
54
53
|
s.add_development_dependency 'vcr'
|
55
54
|
|
56
|
-
s.
|
55
|
+
s.add_development_dependency 'sqlite3' if RUBY_PLATFORM != 'java'
|
56
|
+
|
57
|
+
s.add_dependency 'activesupport', '> 4.2.9', '< 5.2'
|
58
|
+
|
57
59
|
s.add_dependency 'aws-sdk-s3', '~> 1.16'
|
58
60
|
s.add_dependency 'docile', '~> 1.1'
|
59
61
|
s.add_dependency 'erubis', '~> 2.7'
|
data/lcm.rake
CHANGED
@@ -3,6 +3,8 @@ require 'fileutils'
|
|
3
3
|
require 'pathname'
|
4
4
|
require 'rspec/core/rake_task'
|
5
5
|
|
6
|
+
test_cases = %i[integration slow userprov load smoke]
|
7
|
+
|
6
8
|
# Schema for new Bricks.
|
7
9
|
brick_info_schema = {
|
8
10
|
"type" => "object",
|
@@ -74,34 +76,20 @@ end
|
|
74
76
|
|
75
77
|
RSpec::Core::RakeTask.new(:test)
|
76
78
|
namespace :test do
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
desc 'Run load tests'
|
83
|
-
RSpec::Core::RakeTask.new(:load) do |t|
|
84
|
-
t.pattern = 'spec/lcm/load/**/*.rb'
|
85
|
-
end
|
86
|
-
|
87
|
-
desc 'Run slow tests'
|
88
|
-
RSpec::Core::RakeTask.new(:slow) do |t|
|
89
|
-
t.pattern = 'spec/lcm/slow/**/*.rb'
|
90
|
-
end
|
91
|
-
|
92
|
-
namespace :integration do
|
93
|
-
desc 'Run integration tests in Docker'
|
94
|
-
task :docker do
|
95
|
-
system('docker-compose -f docker-compose.lcm.yml up --force-recreate --abort-on-container-exit --exit-code-from appstore appstore') ||
|
96
|
-
fail('Test execution failed!')
|
79
|
+
test_cases.each do |test_case|
|
80
|
+
desc "Run #{test_case} tests"
|
81
|
+
RSpec::Core::RakeTask.new(test_case) do |task|
|
82
|
+
task.pattern = "spec/lcm/#{test_case}/**/*.rb"
|
97
83
|
end
|
98
84
|
end
|
99
|
-
namespace :load do
|
100
|
-
desc 'Run load tests in Docker'
|
101
|
-
task :docker do
|
102
|
-
system('docker-compose -f docker-compose.lcm.yml run --rm appstore bundle exec rake -f lcm.rake test:load')
|
103
85
|
|
104
|
-
|
86
|
+
namespace :docker do
|
87
|
+
test_cases.each do |t|
|
88
|
+
desc "Run #{t} tests in Docker"
|
89
|
+
task t do
|
90
|
+
system("docker-compose -f docker-compose.lcm.yml run --rm appstore bundle exec rake -f lcm.rake test:#{t}") ||
|
91
|
+
fail('Test execution failed!')
|
92
|
+
end
|
105
93
|
end
|
106
94
|
end
|
107
95
|
end
|
data/lib/gooddata.rb
CHANGED
@@ -22,6 +22,7 @@ require_relative 'gooddata/lcm/lcm'
|
|
22
22
|
require_relative 'gooddata/lcm/lcm2'
|
23
23
|
require_relative 'gooddata/models/models'
|
24
24
|
require_relative 'gooddata/bricks/pipeline'
|
25
|
+
require 'gooddata/extensions/object'
|
25
26
|
|
26
27
|
# Files
|
27
28
|
require_relative 'gooddata/app/app'
|
@@ -33,3 +34,6 @@ require_relative 'gooddata/version'
|
|
33
34
|
# Extensions
|
34
35
|
require_relative 'gooddata/extensions/extensions'
|
35
36
|
require 'backports/2.1.0/array/to_h'
|
37
|
+
|
38
|
+
# Helpers
|
39
|
+
require 'gooddata/helpers/global_helpers'
|
@@ -13,7 +13,7 @@ module GoodData
|
|
13
13
|
def call(params)
|
14
14
|
params = params.to_hash
|
15
15
|
if params.key?('aws_client')
|
16
|
-
|
16
|
+
GoodData.logger.info('Setting up AWS-S3 connection')
|
17
17
|
raise 'Unable to connect to AWS. Parameter "aws_client" seems to be empty' unless params['aws_client']
|
18
18
|
raise 'Unable to connect to AWS. Parameter "access_key_id" is missing' if params['aws_client']['access_key_id'].blank?
|
19
19
|
raise 'Unable to connect to AWS. Parameter "secret_access_key" is missing' if params['aws_client']['secret_access_key'].blank?
|
@@ -13,11 +13,11 @@ module GoodData
|
|
13
13
|
class BenchMiddleware < Bricks::Middleware
|
14
14
|
def call(params)
|
15
15
|
params = params.to_hash
|
16
|
-
|
16
|
+
GoodData.logger.info('Starting timer')
|
17
17
|
result = nil
|
18
18
|
report = Benchmark.measure { result = @app.call(params) }
|
19
|
-
|
20
|
-
|
19
|
+
GoodData.logger.info('Stopping timer')
|
20
|
+
GoodData.logger.info(report.pretty_inspect)
|
21
21
|
result
|
22
22
|
end
|
23
23
|
end
|
@@ -13,7 +13,7 @@ module GoodData
|
|
13
13
|
# Connects to the warehouse (ADS) and enriches parameters with GoodData::Datawarehouse
|
14
14
|
class WarehouseMiddleware < Bricks::Middleware
|
15
15
|
def call(params)
|
16
|
-
if params.key?('ads_client')
|
16
|
+
if params.key?('ads_client')
|
17
17
|
GoodData.logger.info "Setting up ADS connection to #{params['ads_client']['ads_id'] || params['ads_client']['jdbc_url']}"
|
18
18
|
unless params['ads_client']['username'] || params['GDC_USERNAME']
|
19
19
|
raise "ADS middleware needs username either as part of " \
|
@@ -33,7 +33,7 @@ module GoodData
|
|
33
33
|
dav.verify_server = false
|
34
34
|
dav.credentials(params['GDC_USERNAME'], params['GDC_PASSWORD'])
|
35
35
|
dav.find(path, recursive: true, suppress_errors: true) do |item|
|
36
|
-
|
36
|
+
GoodData.logger.info('Checking: ' + item.url.to_s)
|
37
37
|
name = (item.uri - webdav_uri).to_s
|
38
38
|
File.open(name, 'w') do |f|
|
39
39
|
f << item.content
|
@@ -25,7 +25,7 @@ module GoodData
|
|
25
25
|
GoodData.client.get '/gdc/account/token', dont_reauth: true
|
26
26
|
url = GoodData.project_webdav_path
|
27
27
|
GoodData.upload_to_project_webdav(path)
|
28
|
-
|
28
|
+
GoodData.logger.info("Uploaded local file \"#{path}\" to url \"#{url + path}\"")
|
29
29
|
end
|
30
30
|
end
|
31
31
|
end
|
@@ -33,10 +33,6 @@ module GoodData
|
|
33
33
|
params = GoodData::Helpers.stringify_keys(params)
|
34
34
|
# params = GoodData::Helpers.symbolize_keys(params)
|
35
35
|
|
36
|
-
# Set logger
|
37
|
-
logger = params['GDC_LOGGER']
|
38
|
-
GoodData.logger = logger
|
39
|
-
|
40
36
|
# Set parallelism
|
41
37
|
max_concurrency = params['max_concurrency'] || params['MAX_CONCURRENCY']
|
42
38
|
if max_concurrency && max_concurrency.to_i > 0
|
@@ -91,14 +87,14 @@ module GoodData
|
|
91
87
|
begin
|
92
88
|
client.disconnect
|
93
89
|
rescue
|
94
|
-
|
90
|
+
GoodData.logger.warn('Tried to disconnect client. Was unsuccessful. Proceeding anyway.')
|
95
91
|
end
|
96
92
|
|
97
93
|
# Try to disconnect development_client
|
98
94
|
begin
|
99
95
|
development_client.disconnect if development_client != client
|
100
96
|
rescue
|
101
|
-
|
97
|
+
GoodData.logger.warn('Tried to disconnect development_client. Was unsuccessful. Proceeding anyway.')
|
102
98
|
end
|
103
99
|
|
104
100
|
returning_value
|
@@ -107,11 +103,11 @@ module GoodData
|
|
107
103
|
class << self
|
108
104
|
def connect(server, verify_ssl, username, password, sst_token) # rubocop:disable Metrics/ParameterLists
|
109
105
|
if username.nil? || password.nil?
|
110
|
-
|
106
|
+
GoodData.logger.info("Connecting with SST to server #{server}")
|
111
107
|
raise 'SST (SuperSecureToken) not present in params' if sst_token.nil?
|
112
108
|
conn = GoodData.connect(sst_token: sst_token, server: server, verify_ssl: verify_ssl)
|
113
109
|
else
|
114
|
-
|
110
|
+
GoodData.logger.info("Connecting as #{username} to server #{server}")
|
115
111
|
conn = GoodData.connect(username, password, server: server, verify_ssl: verify_ssl)
|
116
112
|
end
|
117
113
|
conn.stats_on
|
@@ -19,20 +19,29 @@ using StringExtensions
|
|
19
19
|
using NilExtensions
|
20
20
|
|
21
21
|
require_relative 'base_middleware'
|
22
|
+
require_relative 'mask_logger_decorator'
|
22
23
|
|
23
24
|
module GoodData
|
24
25
|
module Bricks
|
25
26
|
class LoggerMiddleware < Bricks::Middleware
|
26
27
|
def call(params)
|
27
28
|
params = params.to_hash
|
28
|
-
logger = nil
|
29
29
|
if params['GDC_LOGGING_OFF']
|
30
30
|
logger = NilLogger.new
|
31
|
+
elsif params['GDC_LOG_DIRECTORY'] && params['GDC_EXECUTION_ID']
|
32
|
+
log_directory = params['GDC_LOG_DIRECTORY']
|
33
|
+
execution_id = params['GDC_EXECUTION_ID']
|
34
|
+
FileUtils.mkpath log_directory
|
35
|
+
logger = Logger.new("#{log_directory}/#{execution_id}.log")
|
36
|
+
logger.level = params['GDC_LOG_LEVEL'] || 'info'
|
37
|
+
values_to_mask = params['values_to_mask'] || []
|
38
|
+
logger = MaskLoggerDecorator.new(logger, values_to_mask) if values_to_mask.any?
|
31
39
|
else
|
32
40
|
logger = params[:GDC_LOGGER_FILE].nil? ? Logger.new(STDOUT) : Logger.new(params[:GDC_LOGGER_FILE])
|
33
41
|
logger.level = params['GDC_LOG_LEVEL'] || 'info'
|
34
|
-
logger.info('Pipeline starts')
|
35
42
|
end
|
43
|
+
GoodData.logger = logger
|
44
|
+
logger.info('Pipeline starts')
|
36
45
|
params['GDC_LOGGER'] = logger
|
37
46
|
GoodData.logging_http_on if params['HTTP_LOGGING'] && params['HTTP_LOGGING'].to_b
|
38
47
|
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# Copyright (c) 2010-2018 GoodData Corporation. All rights reserved.
|
2
|
+
# This source code is licensed under the BSD-style license found in the
|
3
|
+
# LICENSE file in the root directory of this source tree.
|
4
|
+
|
5
|
+
module GoodData
|
6
|
+
module Bricks
|
7
|
+
# Logger decorator with ability to mask sensitive values
|
8
|
+
class MaskLoggerDecorator
|
9
|
+
# entry-point
|
10
|
+
# @param [Logger] logger logger to decorated
|
11
|
+
# @param [Array] values_to_mask sensitive values to be masked out from logs
|
12
|
+
def initialize(logger, values_to_mask = [])
|
13
|
+
@logger = logger
|
14
|
+
@values_to_mask = values_to_mask
|
15
|
+
end
|
16
|
+
|
17
|
+
# log methods to be decorated
|
18
|
+
%i[debug error fatal info unknown warn].each do |level|
|
19
|
+
define_method level do |message|
|
20
|
+
@logger.send(level, mask(message))
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
# Masks given message.
|
27
|
+
# @param message [String] message to mask
|
28
|
+
# @return masked_message [String] masked message
|
29
|
+
def mask(message)
|
30
|
+
unless message.nil?
|
31
|
+
@values_to_mask.reduce(message) do |masked_message, value_to_mask|
|
32
|
+
masked_message.gsub(value_to_mask, "******")
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
data/lib/gooddata/cli/cli.rb
CHANGED
data/lib/gooddata/cli/hooks.rb
CHANGED
data/lib/gooddata/connection.rb
CHANGED
@@ -37,7 +37,7 @@ module GoodData
|
|
37
37
|
|
38
38
|
class << self
|
39
39
|
def error(msg)
|
40
|
-
|
40
|
+
GoodData.logger.error(msg)
|
41
41
|
exit 1
|
42
42
|
end
|
43
43
|
|
@@ -140,7 +140,7 @@ module GoodData
|
|
140
140
|
|
141
141
|
if full_outer
|
142
142
|
(lookup.keys - marked_lookup.keys).each do |key|
|
143
|
-
|
143
|
+
GoodData.logger.info(lookup[key])
|
144
144
|
results << lookup[key].first.to_hash
|
145
145
|
end
|
146
146
|
end
|
@@ -232,7 +232,7 @@ module GoodData
|
|
232
232
|
|
233
233
|
def decrypt(database64, key)
|
234
234
|
if key.nil? || key.empty?
|
235
|
-
|
235
|
+
GoodData.logger.warn('WARNING: No encryption key provided.')
|
236
236
|
return 'no_key_provided'
|
237
237
|
end
|
238
238
|
|
@@ -120,8 +120,9 @@ module GoodData
|
|
120
120
|
dry_run: params[:dry_run].to_b,
|
121
121
|
users_brick_input: params.users_brick_users
|
122
122
|
}
|
123
|
+
all_clients = domain.clients(:all, data_product).to_a
|
123
124
|
|
124
|
-
|
125
|
+
GoodData.logger.info("Synchronizing in mode \"#{mode}\"")
|
125
126
|
case mode
|
126
127
|
when 'sync_project', 'sync_one_project_based_on_pid', 'sync_one_project_based_on_custom_id'
|
127
128
|
if mode == 'sync_one_project_based_on_pid'
|
@@ -138,14 +139,15 @@ module GoodData
|
|
138
139
|
fail "The #{multiple_projects_column} cannot be empty" if id.blank?
|
139
140
|
|
140
141
|
if mode == 'sync_multiple_projects_based_on_custom_id'
|
141
|
-
|
142
|
+
c = all_clients.detect { |specific_client| specific_client.id == id }
|
143
|
+
current_project = c.project
|
142
144
|
elsif mode == 'sync_multiple_projects_based_on_pid'
|
143
145
|
current_project = client.projects(id)
|
144
146
|
end
|
145
147
|
sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
|
146
148
|
end
|
147
149
|
when 'sync_domain_client_workspaces'
|
148
|
-
domain_clients =
|
150
|
+
domain_clients = all_clients
|
149
151
|
if params.segments
|
150
152
|
segment_uris = params.segments.map(&:uri)
|
151
153
|
domain_clients = domain_clients.select { |c| segment_uris.include?(c.segment_uri) }
|
@@ -159,7 +161,7 @@ module GoodData
|
|
159
161
|
users = users_by_project[client_id]
|
160
162
|
fail "Client id cannot be empty" if client_id.blank?
|
161
163
|
|
162
|
-
c =
|
164
|
+
c = all_clients.detect { |specific_client| specific_client.id == client_id }
|
163
165
|
if params.segments && !segment_uris.include?(c.segment_uri)
|
164
166
|
params.gdc_logger.warn "Client #{client_id} is outside segments_filter #{params.segments}"
|
165
167
|
next
|
@@ -181,6 +181,14 @@ module GoodData
|
|
181
181
|
# aiming at solving the problem that the customer cannot give us the
|
182
182
|
# value of a project id in the data since he does not know it upfront
|
183
183
|
# and we cannot influence its value.
|
184
|
+
common_params = {
|
185
|
+
domain: domain,
|
186
|
+
whitelists: whitelists,
|
187
|
+
ignore_failures: ignore_failures,
|
188
|
+
remove_users_from_project: remove_users_from_project,
|
189
|
+
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
190
|
+
create_non_existing_user_groups: create_non_existing_user_groups
|
191
|
+
}
|
184
192
|
results = case mode
|
185
193
|
when 'add_to_organization'
|
186
194
|
domain.create_users(new_users.uniq { |u| u[:login] || u[:email] })
|
@@ -190,24 +198,12 @@ module GoodData
|
|
190
198
|
params.gdc_logger.warn "Deleting #{users.count} users from domain #{domain_name}"
|
191
199
|
users.map(&:delete)
|
192
200
|
when 'sync_project'
|
193
|
-
project.import_users(new_users,
|
194
|
-
domain: domain,
|
195
|
-
whitelists: whitelists,
|
196
|
-
ignore_failures: ignore_failures,
|
197
|
-
remove_users_from_project: remove_users_from_project,
|
198
|
-
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
199
|
-
create_non_existing_user_groups: create_non_existing_user_groups)
|
201
|
+
project.import_users(new_users, common_params)
|
200
202
|
when 'sync_multiple_projects_based_on_pid'
|
201
203
|
new_users.group_by { |u| u[:pid] }.flat_map do |project_id, users|
|
202
204
|
begin
|
203
205
|
project = client.projects(project_id)
|
204
|
-
project.import_users(users,
|
205
|
-
domain: domain,
|
206
|
-
whitelists: whitelists,
|
207
|
-
ignore_failures: ignore_failures,
|
208
|
-
remove_users_from_project: remove_users_from_project,
|
209
|
-
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
210
|
-
create_non_existing_user_groups: create_non_existing_user_groups)
|
206
|
+
project.import_users(users, common_params)
|
211
207
|
rescue RestClient::ResourceNotFound
|
212
208
|
fail "Project \"#{project_id}\" was not found. Please check your project ids in the source file"
|
213
209
|
rescue RestClient::Gone
|
@@ -218,13 +214,7 @@ module GoodData
|
|
218
214
|
end
|
219
215
|
when 'sync_one_project_based_on_pid'
|
220
216
|
filtered_users = new_users.select { |u| u[:pid] == project.pid }
|
221
|
-
project.import_users(filtered_users,
|
222
|
-
domain: domain,
|
223
|
-
whitelists: whitelists,
|
224
|
-
ignore_failures: ignore_failures,
|
225
|
-
remove_users_from_project: remove_users_from_project,
|
226
|
-
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
227
|
-
create_non_existing_user_groups: create_non_existing_user_groups)
|
217
|
+
project.import_users(filtered_users, common_params)
|
228
218
|
when 'sync_one_project_based_on_custom_id'
|
229
219
|
filter_value = UserBricksHelper.resolve_client_id(domain, project, data_product)
|
230
220
|
|
@@ -243,32 +233,21 @@ module GoodData
|
|
243
233
|
)
|
244
234
|
end
|
245
235
|
|
246
|
-
|
247
|
-
project.import_users(filtered_users,
|
248
|
-
domain: domain,
|
249
|
-
whitelists: whitelists,
|
250
|
-
ignore_failures: ignore_failures,
|
251
|
-
remove_users_from_project: remove_users_from_project,
|
252
|
-
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
253
|
-
create_non_existing_user_groups: create_non_existing_user_groups)
|
236
|
+
GoodData.logger.info("Project #{project.pid} will receive #{filtered_users.count} from #{new_users.count} users")
|
237
|
+
project.import_users(filtered_users, common_params)
|
254
238
|
when 'sync_multiple_projects_based_on_custom_id'
|
239
|
+
all_clients = domain.clients(:all, data_product).to_a
|
255
240
|
new_users.group_by { |u| u[:pid] }.flat_map do |client_id, users|
|
256
241
|
fail "Client id cannot be empty" if client_id.blank?
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
end
|
242
|
+
|
243
|
+
c = all_clients.detect { |specific_client| specific_client.id == client_id }
|
244
|
+
fail "The client \"#{client_id}\" does not exist in data product \"#{data_product.data_product_id}\"" if c.nil?
|
245
|
+
|
246
|
+
project = c.project
|
263
247
|
fail "Client #{client_id} does not have project." unless project
|
264
|
-
|
265
|
-
project.
|
266
|
-
|
267
|
-
whitelists: whitelists,
|
268
|
-
ignore_failures: ignore_failures,
|
269
|
-
remove_users_from_project: remove_users_from_project,
|
270
|
-
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
271
|
-
create_non_existing_user_groups: create_non_existing_user_groups)
|
248
|
+
|
249
|
+
GoodData.logger.info("Project #{project.pid} of client #{client_id} will receive #{users.count} users")
|
250
|
+
project.import_users(users, common_params)
|
272
251
|
end
|
273
252
|
when 'sync_domain_client_workspaces'
|
274
253
|
domain_clients = domain.clients(:all, data_product)
|
@@ -280,22 +259,18 @@ module GoodData
|
|
280
259
|
res = []
|
281
260
|
res += new_users.group_by { |u| u[:pid] }.flat_map do |client_id, users|
|
282
261
|
fail "Client id cannot be empty" if client_id.blank?
|
283
|
-
|
262
|
+
|
263
|
+
c = domain_clients.detect { |specific_client| specific_client.id == client_id }
|
284
264
|
if params.segments && !segment_uris.include?(c.segment_uri)
|
285
|
-
|
265
|
+
GoodData.logger.info("Client #{client_id} is outside segments_filter #{params.segments}")
|
286
266
|
next
|
287
267
|
end
|
288
268
|
project = c.project
|
289
269
|
fail "Client #{client_id} does not have project." unless project
|
270
|
+
|
290
271
|
working_client_ids << client_id.to_s
|
291
|
-
|
292
|
-
project.import_users(users,
|
293
|
-
domain: domain,
|
294
|
-
whitelists: whitelists,
|
295
|
-
ignore_failures: ignore_failures,
|
296
|
-
remove_users_from_project: remove_users_from_project,
|
297
|
-
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
298
|
-
create_non_existing_user_groups: create_non_existing_user_groups)
|
272
|
+
GoodData.logger.info("Project #{project.pid} of client #{client_id} will receive #{users.count} users")
|
273
|
+
project.import_users(users, common_params)
|
299
274
|
end
|
300
275
|
|
301
276
|
params.gdc_logger.debug("Working client ids are: #{working_client_ids.join(', ')}")
|
@@ -306,51 +281,39 @@ module GoodData
|
|
306
281
|
begin
|
307
282
|
project = c.project
|
308
283
|
rescue => e
|
309
|
-
|
284
|
+
GoodData.logger.error("Error when accessing project of client #{c.client_id}. Error: #{e}")
|
310
285
|
next
|
311
286
|
end
|
312
287
|
unless project
|
313
|
-
|
288
|
+
GoodData.logger.info("Client #{c.client_id} has no project.")
|
314
289
|
next
|
315
290
|
end
|
316
291
|
if project.deleted?
|
317
|
-
|
292
|
+
GoodData.logger.info("Project #{project.pid} of client #{c.client_id} is deleted.")
|
318
293
|
next
|
319
294
|
end
|
320
|
-
|
321
|
-
res += project.import_users([],
|
322
|
-
domain: domain,
|
323
|
-
whitelists: whitelists,
|
324
|
-
ignore_failures: ignore_failures,
|
325
|
-
remove_users_from_project: remove_users_from_project,
|
326
|
-
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
327
|
-
create_non_existing_user_groups: create_non_existing_user_groups)
|
295
|
+
GoodData.logger.info("Synchronizing all users in project #{project.pid} of client #{c.client_id}")
|
296
|
+
res += project.import_users([], common_params)
|
328
297
|
end
|
329
298
|
end
|
330
299
|
|
331
300
|
res
|
332
301
|
when 'sync_domain_and_project'
|
333
302
|
domain.create_users(new_users, ignore_failures: ignore_failures)
|
334
|
-
project.import_users(new_users,
|
335
|
-
domain: domain,
|
336
|
-
whitelists: whitelists,
|
337
|
-
ignore_failures: ignore_failures,
|
338
|
-
remove_users_from_project: remove_users_from_project,
|
339
|
-
do_not_touch_users_that_are_not_mentioned: do_not_touch_users_that_are_not_mentioned,
|
340
|
-
create_non_existing_user_groups: create_non_existing_user_groups)
|
303
|
+
project.import_users(new_users, common_params)
|
341
304
|
end
|
342
305
|
|
343
306
|
results.compact!
|
344
307
|
counts = results.group_by { |r| r[:type] }.map { |g, r| [g, r.count] }
|
345
308
|
counts.each do |category, count|
|
346
|
-
|
309
|
+
GoodData.logger.info("There were #{count} events of type #{category}")
|
347
310
|
end
|
348
311
|
errors = results.select { |r| r[:type] == :error || r[:type] == :failed }
|
349
312
|
return if errors.empty?
|
350
313
|
|
351
|
-
|
352
|
-
|
353
|
-
|
314
|
+
GoodData.logger.info('Printing 10 first errors')
|
315
|
+
GoodData.logger.info('========================')
|
316
|
+
GoodData.logger.info(errors.take(10).pretty_inspect)
|
354
317
|
fail 'There was an error syncing users'
|
355
318
|
end
|
356
319
|
|