gooddata 1.3.0 → 1.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.dockerignore +2 -0
- data/CHANGELOG.md +48 -0
- data/CONTRIBUTING.md +19 -12
- data/Dockerfile +37 -0
- data/Gemfile +0 -0
- data/README.md +1 -1
- data/Rakefile +17 -1
- data/bin/run_brick.rb +31 -0
- data/gooddata.gemspec +1 -0
- data/lib/gooddata/bricks/hello_world_brick.rb +21 -0
- data/lib/gooddata/bricks/middleware/gooddata_middleware.rb +12 -0
- data/lib/gooddata/bricks/middleware/logger_middleware.rb +12 -0
- data/lib/gooddata/bricks/pipeline.rb +12 -0
- data/lib/gooddata/exceptions/filter_maqlization.rb +0 -6
- data/lib/gooddata/extensions/class.rb +4 -0
- data/lib/gooddata/extensions/enumerable.rb +0 -3
- data/lib/gooddata/extensions/extensions.rb +0 -3
- data/lib/gooddata/extensions/false.rb +8 -16
- data/lib/gooddata/extensions/hash.rb +10 -41
- data/lib/gooddata/extensions/integer.rb +9 -3
- data/lib/gooddata/extensions/nil.rb +5 -13
- data/lib/gooddata/extensions/object.rb +0 -11
- data/lib/gooddata/extensions/string.rb +11 -5
- data/lib/gooddata/extensions/true.rb +8 -16
- data/lib/gooddata/helpers/global_helpers.rb +12 -0
- data/lib/gooddata/helpers/global_helpers_params.rb +5 -3
- data/lib/gooddata/lcm/actions/apply_custom_maql.rb +6 -0
- data/lib/gooddata/lcm/actions/base_action.rb +8 -2
- data/lib/gooddata/lcm/actions/collect_multiple_projects_column.rb +46 -0
- data/lib/gooddata/lcm/actions/collect_users_brick_users.rb +9 -2
- data/lib/gooddata/lcm/actions/execute_schedules.rb +0 -2
- data/lib/gooddata/lcm/actions/hello_world.rb +1 -1
- data/lib/gooddata/lcm/actions/synchronize_cas.rb +6 -0
- data/lib/gooddata/lcm/actions/synchronize_ldm.rb +6 -0
- data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +70 -107
- data/lib/gooddata/lcm/actions/synchronize_users.rb +1 -13
- data/lib/gooddata/lcm/brick_logger.rb +26 -0
- data/lib/gooddata/lcm/lcm2.rb +46 -7
- data/lib/gooddata/lcm/types/base_type.rb +4 -0
- data/lib/gooddata/lcm/types/class/class.rb +2 -0
- data/lib/gooddata/lcm/types/complex/complex.rb +2 -0
- data/lib/gooddata/lcm/types/special/array.rb +2 -0
- data/lib/gooddata/models/blueprint/project_blueprint.rb +5 -5
- data/lib/gooddata/models/blueprint/to_manifest.rb +0 -2
- data/lib/gooddata/models/domain.rb +1 -0
- data/lib/gooddata/models/from_wire.rb +0 -2
- data/lib/gooddata/models/profile.rb +1 -1
- data/lib/gooddata/models/project.rb +5 -0
- data/lib/gooddata/models/user_filters/user_filter_builder.rb +49 -32
- data/lib/gooddata/models/user_group.rb +3 -0
- data/lib/gooddata/rest/client.rb +4 -4
- data/lib/gooddata/rest/object.rb +2 -0
- data/lib/gooddata/version.rb +1 -1
- metadata +23 -5
- data/lib/gooddata/extensions/big_decimal.rb +0 -17
- data/lib/gooddata/extensions/numeric.rb +0 -15
- data/lib/gooddata/extensions/symbol.rb +0 -15
@@ -201,7 +201,6 @@ module GoodData
|
|
201
201
|
new_users.group_by { |u| u[:pid] }.flat_map do |project_id, users|
|
202
202
|
begin
|
203
203
|
project = client.projects(project_id)
|
204
|
-
fail "You (user executing the script - #{client.user.login}) is not admin in project \"#{project_id}\"." unless project.am_i_admin?
|
205
204
|
project.import_users(users,
|
206
205
|
domain: domain,
|
207
206
|
whitelists: whitelists,
|
@@ -371,21 +370,10 @@ module GoodData
|
|
371
370
|
country_column = params.country_column || 'country'
|
372
371
|
phone_column = params.phone_column || 'phone'
|
373
372
|
ip_whitelist_column = params.ip_whitelist_column || 'ip_whitelist'
|
374
|
-
mode = params.sync_mode
|
375
373
|
|
376
374
|
sso_provider = params.sso_provider
|
377
375
|
authentication_modes = params.authentication_modes || []
|
378
376
|
|
379
|
-
multiple_projects_column = params.multiple_projects_column
|
380
|
-
unless multiple_projects_column
|
381
|
-
client_modes = %w(sync_domain_client_workspaces sync_one_project_based_on_custom_id sync_multiple_projects_based_on_custom_id)
|
382
|
-
multiple_projects_column = if client_modes.include?(mode)
|
383
|
-
'client_id'
|
384
|
-
else
|
385
|
-
'project_id'
|
386
|
-
end
|
387
|
-
end
|
388
|
-
|
389
377
|
dwh = params.ads_client
|
390
378
|
if dwh
|
391
379
|
data = dwh.execute_select(params.input_source.query)
|
@@ -423,7 +411,7 @@ module GoodData
|
|
423
411
|
:sso_provider => sso_provider || row[sso_provider_column] || row[sso_provider_column.to_sym],
|
424
412
|
:authentication_modes => modes,
|
425
413
|
:user_group => user_group,
|
426
|
-
:pid => multiple_projects_column.nil? ? nil : (row[multiple_projects_column] || row[multiple_projects_column.to_sym]),
|
414
|
+
:pid => params.multiple_projects_column.nil? ? nil : (row[params.multiple_projects_column] || row[params.multiple_projects_column.to_sym]),
|
427
415
|
:language => row[language_column] || row[language_column.to_sym],
|
428
416
|
:company => row[company_column] || row[company_column.to_sym],
|
429
417
|
:position => row[position_column] || row[position_column.to_sym],
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
#
|
3
|
+
# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
|
4
|
+
# This source code is licensed under the BSD-style license found in the
|
5
|
+
# LICENSE file in the root directory of this source tree.
|
6
|
+
|
7
|
+
module GoodData
|
8
|
+
# Simple file logger.
|
9
|
+
class BrickFileLogger
|
10
|
+
# entry-point
|
11
|
+
# @param [String] log_directory directory to create log files
|
12
|
+
# @param [String] mode - brick mode (short name if brick)
|
13
|
+
def initialize(log_directory, mode)
|
14
|
+
@log_directory = log_directory
|
15
|
+
@mode = mode
|
16
|
+
end
|
17
|
+
|
18
|
+
# Creates file in log directory with given content. Logging is disabled when log_directory is nil.
|
19
|
+
#
|
20
|
+
# @param [String] status brick phase/status (start, finished, error,...)
|
21
|
+
# @param [String] content log file content
|
22
|
+
def log_action(status, content)
|
23
|
+
File.write("#{@log_directory}/#{@mode}_#{status}.json", content)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
data/lib/gooddata/lcm/lcm2.rb
CHANGED
@@ -6,10 +6,26 @@
|
|
6
6
|
|
7
7
|
require 'terminal-table'
|
8
8
|
|
9
|
+
require 'gooddata/extensions/class'
|
10
|
+
require 'gooddata/extensions/true'
|
11
|
+
require 'gooddata/extensions/false'
|
12
|
+
require 'gooddata/extensions/integer'
|
13
|
+
require 'gooddata/extensions/string'
|
14
|
+
require 'gooddata/extensions/nil'
|
15
|
+
|
16
|
+
require 'active_support/core_ext/hash/compact'
|
17
|
+
|
9
18
|
require_relative 'actions/actions'
|
19
|
+
require_relative 'brick_logger'
|
10
20
|
require_relative 'dsl/dsl'
|
11
21
|
require_relative 'helpers/helpers'
|
12
22
|
|
23
|
+
using TrueExtensions
|
24
|
+
using FalseExtensions
|
25
|
+
using IntegerExtensions
|
26
|
+
using StringExtensions
|
27
|
+
using NilExtensions
|
28
|
+
|
13
29
|
module GoodData
|
14
30
|
module LCM2
|
15
31
|
class SmartHash < Hash
|
@@ -139,12 +155,14 @@ module GoodData
|
|
139
155
|
|
140
156
|
users: [
|
141
157
|
CollectDataProduct,
|
158
|
+
CollectMultipleProjectsColumn,
|
142
159
|
CollectSegments,
|
143
160
|
SynchronizeUsers
|
144
161
|
],
|
145
162
|
|
146
163
|
user_filters: [
|
147
164
|
CollectDataProduct,
|
165
|
+
CollectMultipleProjectsColumn,
|
148
166
|
CollectUsersBrickUsers,
|
149
167
|
CollectSegments,
|
150
168
|
SynchronizeUserFilters
|
@@ -152,6 +170,10 @@ module GoodData
|
|
152
170
|
|
153
171
|
schedules_execution: [
|
154
172
|
ExecuteSchedules
|
173
|
+
],
|
174
|
+
|
175
|
+
hello_world: [
|
176
|
+
HelloWorld
|
155
177
|
]
|
156
178
|
}
|
157
179
|
|
@@ -269,6 +291,16 @@ module GoodData
|
|
269
291
|
|
270
292
|
# Get actions for mode specified
|
271
293
|
actions = get_mode_actions(mode)
|
294
|
+
|
295
|
+
if params.key?('log_directory')
|
296
|
+
brick_logger = BrickFileLogger.new(params['log_directory'], mode)
|
297
|
+
logging_enabled = true
|
298
|
+
else
|
299
|
+
logging_enabled = false
|
300
|
+
end
|
301
|
+
if logging_enabled
|
302
|
+
brick_logger.log_action('start', JSON.pretty_generate(params))
|
303
|
+
end
|
272
304
|
if params.actions
|
273
305
|
actions = params.actions.map do |action|
|
274
306
|
"GoodData::LCM2::#{action}".split('::').inject(Object) do |o, c|
|
@@ -283,8 +315,6 @@ module GoodData
|
|
283
315
|
|
284
316
|
# TODO: Check all action params first
|
285
317
|
|
286
|
-
new_params = params
|
287
|
-
|
288
318
|
fail_early = if params.key?(:fail_early)
|
289
319
|
params.fail_early.to_b
|
290
320
|
else
|
@@ -321,6 +351,7 @@ module GoodData
|
|
321
351
|
backtrace: e.backtrace
|
322
352
|
}
|
323
353
|
break if fail_early
|
354
|
+
results << nil
|
324
355
|
end
|
325
356
|
|
326
357
|
# in case fail_early = false, we need to execute another action
|
@@ -342,19 +373,27 @@ module GoodData
|
|
342
373
|
results << res
|
343
374
|
end
|
344
375
|
|
345
|
-
# Fail whole execution if there is any failed action
|
346
|
-
fail(JSON.pretty_generate(errors)) if strict_mode && errors.any?
|
347
|
-
|
348
376
|
brick_results = {}
|
349
377
|
actions.each_with_index do |action, index|
|
350
378
|
brick_results[action.short_name] = results[index]
|
351
379
|
end
|
352
380
|
|
353
|
-
{
|
381
|
+
result = {
|
354
382
|
actions: actions.map(&:short_name),
|
355
383
|
results: brick_results,
|
356
|
-
params: params
|
384
|
+
params: params,
|
385
|
+
success: errors.empty?
|
357
386
|
}
|
387
|
+
|
388
|
+
# Fail whole execution if there is any failed action
|
389
|
+
fail(JSON.pretty_generate(errors)) if strict_mode && errors.any?
|
390
|
+
|
391
|
+
result
|
392
|
+
|
393
|
+
ensure
|
394
|
+
if logging_enabled
|
395
|
+
brick_logger.log_action('finished', JSON.pretty_generate(result))
|
396
|
+
end
|
358
397
|
end
|
359
398
|
|
360
399
|
def run_action(action, params)
|
@@ -1,9 +1,9 @@
|
|
1
|
-
# encoding: UTF-8
|
2
|
-
#
|
3
1
|
# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
|
4
2
|
# This source code is licensed under the BSD-style license found in the
|
5
3
|
# LICENSE file in the root directory of this source tree.
|
6
4
|
|
5
|
+
require 'active_support/core_ext/hash/compact'
|
6
|
+
|
7
7
|
module GoodData
|
8
8
|
module Model
|
9
9
|
class ProjectBlueprint
|
@@ -125,7 +125,7 @@ module GoodData
|
|
125
125
|
def self.dataset?(project, name, options = {})
|
126
126
|
find_dataset(project, name, options)
|
127
127
|
true
|
128
|
-
rescue
|
128
|
+
rescue StandardError
|
129
129
|
false
|
130
130
|
end
|
131
131
|
|
@@ -163,7 +163,7 @@ module GoodData
|
|
163
163
|
def self.date_dimension?(project, name)
|
164
164
|
find_date_dimension(project, name)
|
165
165
|
true
|
166
|
-
rescue
|
166
|
+
rescue StandardError
|
167
167
|
false
|
168
168
|
end
|
169
169
|
|
@@ -766,7 +766,7 @@ module GoodData
|
|
766
766
|
errors.concat datasets.reduce([]) { |acc, elem| acc.concat(elem.validate) }
|
767
767
|
errors.concat datasets.reduce([]) { |acc, elem| acc.concat(elem.validate_gd_data_type_errors) }
|
768
768
|
errors
|
769
|
-
rescue
|
769
|
+
rescue StandardError
|
770
770
|
raise GoodData::ValidationError
|
771
771
|
end
|
772
772
|
|
@@ -12,6 +12,10 @@ require 'pmap'
|
|
12
12
|
require 'zip'
|
13
13
|
require 'net/smtp'
|
14
14
|
|
15
|
+
require 'active_support/core_ext/hash/except'
|
16
|
+
require 'active_support/core_ext/hash/compact'
|
17
|
+
require 'active_support/core_ext/hash/slice'
|
18
|
+
|
15
19
|
require_relative '../exceptions/no_project_error'
|
16
20
|
|
17
21
|
require_relative '../helpers/auth_helpers'
|
@@ -1813,6 +1817,7 @@ module GoodData
|
|
1813
1817
|
alias_method :create_users, :set_users_roles
|
1814
1818
|
|
1815
1819
|
def add_data_permissions(filters, options = {})
|
1820
|
+
GoodData.logger.info("Synchronizing #{filters.count} filters in project #{pid}")
|
1816
1821
|
GoodData::UserFilterBuilder.execute_mufs(filters, { client: client, project: self }.merge(options))
|
1817
1822
|
end
|
1818
1823
|
|
@@ -5,7 +5,18 @@
|
|
5
5
|
# LICENSE file in the root directory of this source tree.
|
6
6
|
|
7
7
|
require_relative '../project_log_formatter'
|
8
|
+
|
8
9
|
require 'active_support/core_ext/hash/indifferent_access'
|
10
|
+
require 'active_support/core_ext/hash/compact'
|
11
|
+
|
12
|
+
require 'gooddata/extensions/true'
|
13
|
+
require 'gooddata/extensions/false'
|
14
|
+
require 'gooddata/extensions/integer'
|
15
|
+
|
16
|
+
using FalseExtensions
|
17
|
+
using TrueExtensions
|
18
|
+
using IntegerExtensions
|
19
|
+
using NilExtensions
|
9
20
|
|
10
21
|
module GoodData
|
11
22
|
module UserFilterBuilder
|
@@ -199,47 +210,53 @@ module GoodData
|
|
199
210
|
# Creates a MAQL expression(s) based on the filter defintion.
|
200
211
|
# Takes the filter definition looks up any necessary values and provides API executable MAQL
|
201
212
|
def self.create_expression(filter, labels_cache, lookups_cache, attr_cache, options = {})
|
202
|
-
errors = []
|
203
213
|
values = filter[:values]
|
204
214
|
label = labels_cache[filter[:label]]
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
end
|
213
|
-
else
|
214
|
-
label.find_value_uri(v)
|
215
|
-
end
|
216
|
-
rescue
|
217
|
-
errors << {
|
218
|
-
type: :error,
|
219
|
-
label: label.title,
|
220
|
-
value: v
|
221
|
-
}
|
222
|
-
nil
|
215
|
+
errors = []
|
216
|
+
|
217
|
+
element_uris_by_values = Hash[values.map do |v|
|
218
|
+
if lookups_cache.key?(label.uri)
|
219
|
+
[v, lookups_cache[label.uri][v]]
|
220
|
+
else
|
221
|
+
[v, label.find_value_uri(v)]
|
223
222
|
end
|
223
|
+
end]
|
224
|
+
|
225
|
+
missing_value_errors = element_uris_by_values.select { |_, v| v.nil? }.map do |k, _|
|
226
|
+
{
|
227
|
+
type: :error,
|
228
|
+
label: label.title,
|
229
|
+
value: k,
|
230
|
+
reason: 'Can not find the value of the attribute referenced in the MUF'
|
231
|
+
}
|
224
232
|
end
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
233
|
+
errors += missing_value_errors unless options[:ignore_missing_values]
|
234
|
+
|
235
|
+
element_uris = element_uris_by_values.values.compact
|
236
|
+
# happens when data is not yet loaded in the project
|
237
|
+
no_values = element_uris.empty?
|
238
|
+
|
239
|
+
expression = if no_values && options[:restrict_if_missing_all_values] && options[:type] == :muf
|
240
|
+
# create a filter that is always false to ensure the user can not see any data
|
241
|
+
# as the proper MUF can not be constructed yet
|
242
|
+
case options[:type]
|
243
|
+
when :muf
|
244
|
+
'1 <> 1'
|
245
|
+
when :variable
|
246
|
+
nil
|
247
|
+
end
|
248
|
+
elsif no_values
|
249
|
+
# create a filter that is always true to ensure the user can see all data
|
230
250
|
'TRUE'
|
231
251
|
elsif filter[:over] && filter[:to]
|
232
252
|
over = attr_cache[filter[:over]]
|
233
253
|
to = attr_cache[filter[:to]]
|
234
|
-
"([#{label.attribute_uri}] IN (#{element_uris.
|
254
|
+
"([#{label.attribute_uri}] IN (#{element_uris.sort.map { |e| '[' + e + ']' }.join(', ')})) OVER [#{over && over.uri}] TO [#{to && to.uri}]"
|
235
255
|
else
|
236
|
-
"[#{label.attribute_uri}] IN (#{element_uris.
|
256
|
+
"[#{label.attribute_uri}] IN (#{element_uris.sort.map { |e| '[' + e + ']' }.join(', ')})"
|
237
257
|
end
|
238
|
-
|
239
|
-
|
240
|
-
else
|
241
|
-
[expression, errors]
|
242
|
-
end
|
258
|
+
|
259
|
+
[expression, errors]
|
243
260
|
end
|
244
261
|
|
245
262
|
# Encapuslates the creation of filter
|
@@ -273,7 +290,7 @@ module GoodData
|
|
273
290
|
expression, errors = create_expression(f, labels_cache, lookups_cache, attrs_cache, options)
|
274
291
|
safe_login = login.downcase
|
275
292
|
profiles_uri = if options[:type] == :muf
|
276
|
-
project_user = project_users.find { |u| u.login ==
|
293
|
+
project_user = project_users.find { |u| u.login == safe_login }
|
277
294
|
project_user.nil? ? ('/gdc/account/profile/' + safe_login) : project_user.profile_url
|
278
295
|
elsif options[:type] == :variable
|
279
296
|
(users_cache[login] && users_cache[login].uri)
|
@@ -12,6 +12,9 @@ require_relative '../mixins/links'
|
|
12
12
|
require_relative '../mixins/rest_resource'
|
13
13
|
require_relative '../mixins/uri_getter'
|
14
14
|
|
15
|
+
require 'active_support/core_ext/hash/except'
|
16
|
+
require 'active_support/core_ext/hash/compact'
|
17
|
+
|
15
18
|
module GoodData
|
16
19
|
# Representation of User Group
|
17
20
|
#
|
data/lib/gooddata/rest/client.rb
CHANGED
@@ -314,27 +314,27 @@ module GoodData
|
|
314
314
|
# Generalizaton of poller. Since we have quite a variation of how async proceses are handled
|
315
315
|
# this is a helper that should help you with resources where the information about "Are we done"
|
316
316
|
# is inside the response. It expects the URI as an input where it can poll and a block that should
|
317
|
-
# return either true
|
317
|
+
# return either true (meaning sleep and repeat) or false (meaning we are done). It returns the
|
318
318
|
# value of last poll. In majority of cases these are the data that you need
|
319
319
|
#
|
320
320
|
# @param link [String] Link for polling
|
321
321
|
# @param options [Hash] Options
|
322
322
|
# @return [Hash] Result of polling
|
323
323
|
def poll_on_response(link, options = {}, &bl)
|
324
|
-
sleep_interval = options[:sleep_interval] || DEFAULT_SLEEP_INTERVAL
|
325
324
|
time_limit = options[:time_limit] || DEFAULT_POLL_TIME_LIMIT
|
326
325
|
process = options[:process] == false ? false : true
|
327
326
|
|
328
327
|
# get the first status and start the timer
|
329
328
|
response = get(link, process: process)
|
330
329
|
poll_start = Time.now
|
331
|
-
|
330
|
+
retry_time = GoodData::Rest::Connection::RETRY_TIME_INITIAL_VALUE
|
332
331
|
while bl.call(response)
|
333
332
|
limit_breached = time_limit && (Time.now - poll_start > time_limit)
|
334
333
|
if limit_breached
|
335
334
|
fail ExecutionLimitExceeded, "The time limit #{time_limit} secs for polling on #{link} is over"
|
336
335
|
end
|
337
|
-
sleep
|
336
|
+
sleep retry_time
|
337
|
+
retry_time *= GoodData::Rest::Connection::RETRY_TIME_COEFFICIENT
|
338
338
|
GoodData::Rest::Client.retryable(:tries => Helpers::GD_MAX_RETRY, :refresh_token => proc { connection.refresh_token }) do
|
339
339
|
response = get(link, process: process)
|
340
340
|
end
|