gooddata 1.3.6 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitignore +1 -0
- data/.travis.yml +19 -4
- data/CHANGELOG.md +37 -1
- data/CONTRIBUTING.md +4 -0
- data/Gemfile +9 -0
- data/Jenkinsfile-chart +14 -0
- data/RELEASING.md +5 -8
- data/Rakefile +1 -1
- data/SDK_VERSION +1 -1
- data/bin/run_brick.rb +1 -1
- data/bin/test_projects_cleanup.rb +20 -8
- data/ci.rake +2 -2
- data/k8s/charts/lcm-bricks/Chart.yaml +4 -0
- data/k8s/charts/lcm-bricks/templates/prometheus/alertingRules.yaml +96 -0
- data/lcm.rake +1 -1
- data/lib/gooddata/bricks/middleware/context_logger_decorator.rb +31 -0
- data/lib/gooddata/bricks/middleware/context_manager.rb +68 -0
- data/lib/gooddata/bricks/middleware/gooddata_middleware.rb +15 -9
- data/lib/gooddata/bricks/middleware/logger_middleware.rb +20 -0
- data/lib/gooddata/bricks/middleware/mask_logger_decorator.rb +35 -5
- data/lib/gooddata/client.rb +2 -3
- data/lib/gooddata/core/gd_logger.rb +92 -0
- data/lib/gooddata/core/logging.rb +24 -7
- data/lib/gooddata/core/nil_logger.rb +1 -2
- data/lib/gooddata/core/splunk_logger.rb +23 -0
- data/lib/gooddata/lcm/actions/synchronize_users.rb +2 -1
- data/lib/gooddata/lcm/helpers/check_helper.rb +3 -15
- data/lib/gooddata/lcm/helpers/safe_failure_helper.rb +19 -0
- data/lib/gooddata/lcm/lcm2.rb +21 -10
- data/lib/gooddata/mixins/property_accessor.rb +30 -0
- data/lib/gooddata/models/execution.rb +5 -0
- data/lib/gooddata/models/project.rb +6 -4
- data/lib/gooddata/rest/client.rb +17 -6
- data/lib/gooddata/rest/connection.rb +20 -6
- data/lib/gooddata/rest/rest_aggregator.rb +46 -0
- metadata +13 -4
data/lib/gooddata/lcm/lcm2.rb
CHANGED
@@ -268,6 +268,8 @@ module GoodData
|
|
268
268
|
def perform(mode, params = {})
|
269
269
|
params = convert_params(params)
|
270
270
|
|
271
|
+
GoodData.gd_logger.brick = mode
|
272
|
+
|
271
273
|
# Get actions for mode specified
|
272
274
|
actions = get_mode_actions(mode)
|
273
275
|
|
@@ -353,21 +355,30 @@ module GoodData
|
|
353
355
|
success: errors.empty?
|
354
356
|
}
|
355
357
|
|
356
|
-
|
357
|
-
|
358
|
-
|
358
|
+
if errors.any?
|
359
|
+
error_message = JSON.pretty_generate(errors)
|
360
|
+
GoodData.logger.error(error_message)
|
361
|
+
|
362
|
+
# Fail whole execution if there is any failed action
|
363
|
+
fail(error_message) if strict_mode
|
364
|
+
end
|
359
365
|
|
360
366
|
result
|
361
367
|
end
|
362
368
|
|
363
369
|
def run_action(action, params)
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
370
|
+
begin
|
371
|
+
GoodData.gd_logger.start_action action, GoodData.gd_logger
|
372
|
+
GoodData.logger.info("Running #{action.name} action ...")
|
373
|
+
params.clear_filters
|
374
|
+
# Check if all required parameters were passed
|
375
|
+
BaseAction.check_params(action.const_get('PARAMS'), params)
|
376
|
+
params.setup_filters(action.const_get('PARAMS'))
|
377
|
+
out = action.send(:call, params)
|
378
|
+
ensure
|
379
|
+
params.clear_filters
|
380
|
+
GoodData.gd_logger.end_action GoodData.gd_logger
|
381
|
+
end
|
371
382
|
out
|
372
383
|
end
|
373
384
|
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# Copyright (c) 2010-2018 GoodData Corporation. All rights reserved.
|
2
|
+
# This source code is licensed under the BSD-style license found in the
|
3
|
+
# LICENSE file in the root directory of this source tree.
|
4
|
+
|
5
|
+
module GoodData
|
6
|
+
module Mixin
|
7
|
+
module PropertyAccessor
|
8
|
+
def property_reader(where, *props)
|
9
|
+
props.each do |prop|
|
10
|
+
define_method(prop, proc {
|
11
|
+
instance_variable_get(where)[prop]
|
12
|
+
})
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def property_writer(where, *props)
|
17
|
+
props.each do |prop|
|
18
|
+
define_method("#{prop}=", proc { |val|
|
19
|
+
instance_variable_get(where)[prop] = val
|
20
|
+
})
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def property_accessor(*args)
|
25
|
+
property_reader(*args)
|
26
|
+
property_writer(*args)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -92,7 +92,12 @@ module GoodData
|
|
92
92
|
#
|
93
93
|
# @return [GoodData::Execution] Execution result
|
94
94
|
def wait_for_result(options = {})
|
95
|
+
start_time = Time.now
|
96
|
+
timeout = options[:timeout]
|
95
97
|
res = client.poll_on_response(uri, options) do |body|
|
98
|
+
timeout_exceeded = timeout && (start_time + timeout) < Time.now
|
99
|
+
fail 'Waiting for schedule execution timed out.' if timeout_exceeded
|
100
|
+
|
96
101
|
body['execution'] && (body['execution']['status'] == 'RUNNING' || body['execution']['status'] == 'SCHEDULED')
|
97
102
|
end
|
98
103
|
@json = res
|
@@ -1606,7 +1606,7 @@ module GoodData
|
|
1606
1606
|
whitelisted_new_users, whitelisted_users = whitelist_users(new_users.map(&:to_hash), users_list, options[:whitelists])
|
1607
1607
|
|
1608
1608
|
# First check that if groups are provided we have them set up
|
1609
|
-
check_groups(new_users.map(&:to_hash).flat_map { |u| u[:user_group] || [] }.uniq, options)
|
1609
|
+
options[:user_groups_cache] = check_groups(new_users.map(&:to_hash).flat_map { |u| u[:user_group] || [] }.uniq, options[:user_groups_cache], options)
|
1610
1610
|
|
1611
1611
|
# conform the role on list of new users so we can diff them with the users coming from the project
|
1612
1612
|
diffable_new_with_default_role = whitelisted_new_users.map do |u|
|
@@ -1706,7 +1706,7 @@ module GoodData
|
|
1706
1706
|
user_groups(g).set_members(remote_users)
|
1707
1707
|
end
|
1708
1708
|
mentioned_groups = mappings.map(&:last).uniq
|
1709
|
-
groups_to_cleanup =
|
1709
|
+
groups_to_cleanup = options[:user_groups_cache].reject { |g| mentioned_groups.include?(g.name) }
|
1710
1710
|
# clean all groups not mentioned with exception of whitelisted users
|
1711
1711
|
groups_to_cleanup.each do |g|
|
1712
1712
|
g.set_members(whitelist_users(g.members.map(&:to_hash), [], options[:whitelists], :include).first.map { |x| x[:uri] })
|
@@ -1744,8 +1744,9 @@ module GoodData
|
|
1744
1744
|
end
|
1745
1745
|
end
|
1746
1746
|
|
1747
|
-
def check_groups(specified_groups, options = {})
|
1748
|
-
|
1747
|
+
def check_groups(specified_groups, user_groups_cache = nil, options = {})
|
1748
|
+
user_groups_cache = user_groups if user_groups_cache.nil? || user_groups_cache.empty?
|
1749
|
+
groups = user_groups_cache.map(&:name)
|
1749
1750
|
missing_groups = specified_groups - groups
|
1750
1751
|
if options[:create_non_existing_user_groups]
|
1751
1752
|
missing_groups.each do |g|
|
@@ -1758,6 +1759,7 @@ module GoodData
|
|
1758
1759
|
"#{groups.join(',')} and you asked for #{missing_groups.join(',')}"
|
1759
1760
|
end
|
1760
1761
|
end
|
1762
|
+
user_groups_cache
|
1761
1763
|
end
|
1762
1764
|
|
1763
1765
|
# Update user
|
data/lib/gooddata/rest/client.rb
CHANGED
@@ -67,6 +67,17 @@ module GoodData
|
|
67
67
|
# @param password [String] Password to be used for authentication
|
68
68
|
# @return [GoodData::Rest::Client] Client
|
69
69
|
def connect(username, password = 'aaaa', opts = {})
|
70
|
+
execution_id = ""
|
71
|
+
if username.is_a?(Hash) && username.key?(:execution_id)
|
72
|
+
execution_id = username[:execution_id]
|
73
|
+
username.delete(:execution_id)
|
74
|
+
end
|
75
|
+
|
76
|
+
if opts.key?(:execution_id)
|
77
|
+
execution_id = opts[:execution_id]
|
78
|
+
opts.delete(:execution_id)
|
79
|
+
end
|
80
|
+
|
70
81
|
if username.nil? && password.nil?
|
71
82
|
username = ENV['GD_GEM_USER']
|
72
83
|
password = ENV['GD_GEM_PASSWORD']
|
@@ -88,7 +99,7 @@ module GoodData
|
|
88
99
|
new_opts[:password] = password
|
89
100
|
end
|
90
101
|
|
91
|
-
new_opts = { verify_ssl: true }.merge(new_opts)
|
102
|
+
new_opts = { verify_ssl: true, execution_id: execution_id }.merge(new_opts)
|
92
103
|
if username.is_a?(Hash) && username[:cookies]
|
93
104
|
new_opts[:sst_token] = username[:cookies]['GDCAuthSST']
|
94
105
|
new_opts[:cookies] = username[:cookies]
|
@@ -195,7 +206,7 @@ module GoodData
|
|
195
206
|
def disconnect
|
196
207
|
if stats_on?
|
197
208
|
GoodData.logger.info("API call statistics to server #{@connection.server}")
|
198
|
-
GoodData.logger.info(@connection.stats_table)
|
209
|
+
GoodData.logger.info(@connection.stats_table.to_s)
|
199
210
|
end
|
200
211
|
@connection.disconnect
|
201
212
|
end
|
@@ -256,14 +267,14 @@ module GoodData
|
|
256
267
|
#
|
257
268
|
# @param uri [String] Target URI
|
258
269
|
def delete(uri, opts = {})
|
259
|
-
@connection.delete uri, opts
|
270
|
+
@connection.delete uri, opts.merge(stats_on: stats_on?)
|
260
271
|
end
|
261
272
|
|
262
273
|
# HTTP GET
|
263
274
|
#
|
264
275
|
# @param uri [String] Target URI
|
265
276
|
def get(uri, opts = {}, & block)
|
266
|
-
@connection.get uri, opts, & block
|
277
|
+
@connection.get uri, opts.merge(stats_on: stats_on?), & block
|
267
278
|
end
|
268
279
|
|
269
280
|
def project_webdav_path(opts = { project: GoodData.project })
|
@@ -349,14 +360,14 @@ module GoodData
|
|
349
360
|
#
|
350
361
|
# @param uri [String] Target URI
|
351
362
|
def put(uri, data, opts = {})
|
352
|
-
@connection.put uri, data, opts
|
363
|
+
@connection.put uri, data, opts.merge(stats_on: stats_on?)
|
353
364
|
end
|
354
365
|
|
355
366
|
# HTTP POST
|
356
367
|
#
|
357
368
|
# @param uri [String] Target URI
|
358
369
|
def post(uri, data, opts = {})
|
359
|
-
@connection.post uri, data, opts
|
370
|
+
@connection.post uri, data, opts.merge(stats_on: stats_on?)
|
360
371
|
end
|
361
372
|
|
362
373
|
# Uploads file to staging
|
@@ -131,6 +131,7 @@ module GoodData
|
|
131
131
|
def initialize(opts)
|
132
132
|
super()
|
133
133
|
@stats = ThreadSafe::Hash.new
|
134
|
+
@execution_id = opts[:execution_id]
|
134
135
|
|
135
136
|
headers = opts[:headers] || {}
|
136
137
|
@webdav_headers = DEFAULT_WEBDAV_HEADERS.merge(headers)
|
@@ -328,10 +329,12 @@ module GoodData
|
|
328
329
|
def request(method, uri, data, options = {}, &user_block)
|
329
330
|
request_id = options[:request_id] || generate_request_id
|
330
331
|
log_info(options.merge(request_id: request_id))
|
332
|
+
stats_on = options[:stats_on]
|
333
|
+
|
331
334
|
payload = data.is_a?(Hash) ? data.to_json : data
|
332
335
|
|
333
336
|
GoodData.rest_logger.info "#{method.to_s.upcase}: #{@server.url}#{uri}, #{scrub_params(data, KEYS_TO_SCRUB)}"
|
334
|
-
profile
|
337
|
+
profile method.to_s.upcase, uri, request_id, stats_on do
|
335
338
|
b = proc do
|
336
339
|
params = fresh_request_params(request_id).merge(options)
|
337
340
|
begin
|
@@ -452,7 +455,7 @@ module GoodData
|
|
452
455
|
end
|
453
456
|
|
454
457
|
def generate_request_id
|
455
|
-
"#{session_id}:#{call_id}"
|
458
|
+
"#{@execution_id}:#{session_id}:#{call_id}"
|
456
459
|
end
|
457
460
|
|
458
461
|
private
|
@@ -591,13 +594,13 @@ ERR
|
|
591
594
|
raise $ERROR_INFO
|
592
595
|
end
|
593
596
|
|
594
|
-
def profile(
|
597
|
+
def profile(method, path, request_id, stats_on, &block)
|
595
598
|
t1 = Time.now
|
596
599
|
res = block.call
|
597
600
|
t2 = Time.now
|
598
601
|
delta = t2 - t1
|
599
602
|
|
600
|
-
|
603
|
+
add_stat_record method, path, delta, t1, request_id if stats_on
|
601
604
|
res
|
602
605
|
end
|
603
606
|
|
@@ -617,9 +620,10 @@ ERR
|
|
617
620
|
new_params
|
618
621
|
end
|
619
622
|
|
620
|
-
def
|
623
|
+
def add_stat_record(method, path, delta, time_stamp, request_id)
|
621
624
|
synchronize do
|
622
|
-
orig_title =
|
625
|
+
orig_title = "#{method.to_s.upcase} #{path}"
|
626
|
+
title = "#{method.to_s.upcase} #{path}"
|
623
627
|
|
624
628
|
placeholders = true
|
625
629
|
|
@@ -648,6 +652,16 @@ ERR
|
|
648
652
|
stat[:entries] << orig_title if placeholders
|
649
653
|
|
650
654
|
stats[title] = stat
|
655
|
+
|
656
|
+
endpoint = self.class.map_placeholders path.clone
|
657
|
+
duration = delta
|
658
|
+
time_stamp = time_stamp.utc.strftime "%Y-%m-%dT%H:%M:%S.%L"
|
659
|
+
domain = server_url.gsub %r{http://|https://}, ""
|
660
|
+
execution_id = request_id
|
661
|
+
|
662
|
+
GoodData.gd_logger.update_store(domain, method, duration, endpoint)
|
663
|
+
GoodData.gd_logger.add Logger::INFO, { endpoint: endpoint, duration: duration, domain: domain,
|
664
|
+
execution_id: execution_id, time_stamp: time_stamp }, "rest_call"
|
651
665
|
end
|
652
666
|
end
|
653
667
|
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
|
2
|
+
# This source code is licensed under the BSD-style license found in the
|
3
|
+
# LICENSE file in the root directory of this source tree.
|
4
|
+
|
5
|
+
module GoodData
|
6
|
+
module Rest
|
7
|
+
# class is responsible for storage and aggregation of REST calls information
|
8
|
+
module Aggregator
|
9
|
+
attr_reader :store
|
10
|
+
|
11
|
+
def initialize_store
|
12
|
+
@store = {}
|
13
|
+
end
|
14
|
+
|
15
|
+
def clear_store
|
16
|
+
@store.clear
|
17
|
+
end
|
18
|
+
|
19
|
+
def update_store(domain, method, duration, endpoint)
|
20
|
+
domain = domain.to_sym
|
21
|
+
method = method.to_sym
|
22
|
+
endpoint = endpoint.to_sym
|
23
|
+
@store[domain] = {} unless @store.key?(domain)
|
24
|
+
@store[domain][method] = {} unless @store[domain].key?(method)
|
25
|
+
if @store[domain][method].key?(endpoint)
|
26
|
+
record = @store[domain][method][endpoint]
|
27
|
+
record[:min] = [duration, record[:min]].min
|
28
|
+
record[:max] = [duration, record[:max]].max
|
29
|
+
record[:avg] = (record[:avg] * record[:count] + duration).to_f / (record[:count] + 1)
|
30
|
+
record[:count] += 1
|
31
|
+
@store[domain][method][endpoint] = record
|
32
|
+
else
|
33
|
+
@store[domain][method][endpoint] = {
|
34
|
+
:min => duration,
|
35
|
+
:max => duration,
|
36
|
+
:avg => duration,
|
37
|
+
:count => 1,
|
38
|
+
:method => method.to_s,
|
39
|
+
:endpoint => endpoint.to_s,
|
40
|
+
:domain => domain.to_s
|
41
|
+
}
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gooddata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Pavel Kolesnikov
|
@@ -14,7 +14,7 @@ authors:
|
|
14
14
|
autorequire:
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
|
-
date: 2019-
|
17
|
+
date: 2019-02-14 00:00:00.000000000 Z
|
18
18
|
dependencies:
|
19
19
|
- !ruby/object:Gem::Dependency
|
20
20
|
name: license_finder
|
@@ -586,6 +586,7 @@ files:
|
|
586
586
|
- Gemfile
|
587
587
|
- Guardfile
|
588
588
|
- Jenkinsfile
|
589
|
+
- Jenkinsfile-chart
|
589
590
|
- LICENSE
|
590
591
|
- LICENSE.rb
|
591
592
|
- README.md
|
@@ -616,6 +617,8 @@ files:
|
|
616
617
|
- docker-compose.yml
|
617
618
|
- gooddata
|
618
619
|
- gooddata.gemspec
|
620
|
+
- k8s/charts/lcm-bricks/Chart.yaml
|
621
|
+
- k8s/charts/lcm-bricks/templates/prometheus/alertingRules.yaml
|
619
622
|
- lcm.rake
|
620
623
|
- lib/gooddata.rb
|
621
624
|
- lib/gooddata/app/app.rb
|
@@ -628,6 +631,8 @@ files:
|
|
628
631
|
- lib/gooddata/bricks/middleware/base_middleware.rb
|
629
632
|
- lib/gooddata/bricks/middleware/bench_middleware.rb
|
630
633
|
- lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb
|
634
|
+
- lib/gooddata/bricks/middleware/context_logger_decorator.rb
|
635
|
+
- lib/gooddata/bricks/middleware/context_manager.rb
|
631
636
|
- lib/gooddata/bricks/middleware/decode_params_middleware.rb
|
632
637
|
- lib/gooddata/bricks/middleware/dwh_middleware.rb
|
633
638
|
- lib/gooddata/bricks/middleware/fs_download_middleware.rb
|
@@ -671,10 +676,12 @@ files:
|
|
671
676
|
- lib/gooddata/commands/user.rb
|
672
677
|
- lib/gooddata/connection.rb
|
673
678
|
- lib/gooddata/core/core.rb
|
679
|
+
- lib/gooddata/core/gd_logger.rb
|
674
680
|
- lib/gooddata/core/logging.rb
|
675
681
|
- lib/gooddata/core/nil_logger.rb
|
676
682
|
- lib/gooddata/core/project.rb
|
677
683
|
- lib/gooddata/core/rest.rb
|
684
|
+
- lib/gooddata/core/splunk_logger.rb
|
678
685
|
- lib/gooddata/core/user.rb
|
679
686
|
- lib/gooddata/data/data.rb
|
680
687
|
- lib/gooddata/data/guesser.rb
|
@@ -769,6 +776,7 @@ files:
|
|
769
776
|
- lib/gooddata/lcm/helpers/check_helper.rb
|
770
777
|
- lib/gooddata/lcm/helpers/helpers.rb
|
771
778
|
- lib/gooddata/lcm/helpers/release_table_helper.rb
|
779
|
+
- lib/gooddata/lcm/helpers/safe_failure_helper.rb
|
772
780
|
- lib/gooddata/lcm/helpers/tags_helper.rb
|
773
781
|
- lib/gooddata/lcm/lcm.rb
|
774
782
|
- lib/gooddata/lcm/lcm2.rb
|
@@ -838,6 +846,7 @@ files:
|
|
838
846
|
- lib/gooddata/mixins/not_label.rb
|
839
847
|
- lib/gooddata/mixins/not_metric.rb
|
840
848
|
- lib/gooddata/mixins/obj_id.rb
|
849
|
+
- lib/gooddata/mixins/property_accessor.rb
|
841
850
|
- lib/gooddata/mixins/rest_getters.rb
|
842
851
|
- lib/gooddata/mixins/rest_resource.rb
|
843
852
|
- lib/gooddata/mixins/root_key_getter.rb
|
@@ -933,6 +942,7 @@ files:
|
|
933
942
|
- lib/gooddata/rest/phmap.rb
|
934
943
|
- lib/gooddata/rest/resource.rb
|
935
944
|
- lib/gooddata/rest/rest.rb
|
945
|
+
- lib/gooddata/rest/rest_aggregator.rb
|
936
946
|
- lib/gooddata/version.rb
|
937
947
|
- lib/templates/bricks/brick.rb.erb
|
938
948
|
- lib/templates/bricks/main.rb.erb
|
@@ -964,8 +974,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
964
974
|
- !ruby/object:Gem::Version
|
965
975
|
version: '0'
|
966
976
|
requirements: []
|
967
|
-
|
968
|
-
rubygems_version: 2.6.11
|
977
|
+
rubygems_version: 3.0.2
|
969
978
|
signing_key:
|
970
979
|
specification_version: 4
|
971
980
|
summary: A convenient Ruby wrapper around the GoodData RESTful API
|