gooddata 2.1.2 → 2.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/CHANGELOG.md +5 -0
- data/Dockerfile +13 -7
- data/SDK_VERSION +1 -1
- data/bin/hello_world.sh +1 -1
- data/bin/help.sh +1 -1
- data/bin/provision.sh +1 -1
- data/bin/release.sh +1 -1
- data/bin/rollout.sh +1 -1
- data/bin/run_smoke_tests.sh +1 -1
- data/bin/test_projects_cleanup.rb +1 -1
- data/bin/user_filters.sh +1 -1
- data/bin/users.sh +1 -1
- data/lcm.rake +15 -1
- data/lib/gooddata/bricks/middleware/context_manager.rb +3 -3
- data/lib/gooddata/helpers/csv_helper.rb +27 -0
- data/lib/gooddata/lcm/actions/collect_segment_clients.rb +23 -7
- data/lib/gooddata/lcm/actions/create_segment_masters.rb +12 -8
- data/lib/gooddata/lcm/actions/ensure_release_table.rb +26 -18
- data/lib/gooddata/lcm/actions/synchronize_clients.rb +11 -7
- data/lib/gooddata/lcm/actions/update_release_table.rb +12 -8
- data/lib/gooddata/lcm/helpers/release_table_helper.rb +20 -1
- data/lib/gooddata/models/model.rb +15 -14
- metadata +4 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA1:
|
|
3
|
+
metadata.gz: 83f55d544af84a7ee0d0ad7e87b934d3f882caf7
|
|
4
|
+
data.tar.gz: 3c7f27e350cfb97b366b28e09c4ce0e085802b2b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 252f9be193469d53692aae3346bb69f78fa1d6d3647ac01f86f2397c5af1ae0759f07d9224e6cd520b7978248d232cb03878a3b426274e82b65d18ec9f62b6a4
|
|
7
|
+
data.tar.gz: f8bfe2a30a24428dfe0a744d6842e45990016d4a1b750ce217634f7c2c3efcce77d9e318cca741fa3ab715531d5c9349efde0fcbc6d269fb067aac239ef52b86
|
data/CHANGELOG.md
CHANGED
data/Dockerfile
CHANGED
|
@@ -11,19 +11,25 @@ LABEL git_commit=$GIT_COMMIT
|
|
|
11
11
|
LABEL bricks_version=$BRICKS_VERSION
|
|
12
12
|
|
|
13
13
|
# which is required by RVM
|
|
14
|
-
RUN yum install -y curl which \
|
|
14
|
+
RUN yum install -y curl which patch make git \
|
|
15
15
|
&& yum clean all \
|
|
16
16
|
&& rm -rf /var/cache/yum
|
|
17
17
|
|
|
18
|
-
RUN gpg2 --keyserver hkp://pool.sks-keyservers.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
|
|
19
|
-
RUN curl -sSL https://get.rvm.io | bash -s stable
|
|
20
|
-
|
|
21
18
|
# Switch to directory with sources
|
|
22
19
|
WORKDIR /src
|
|
23
|
-
|
|
20
|
+
|
|
21
|
+
RUN groupadd -g 1003 lcmuser && \
|
|
22
|
+
useradd -r -u 1003 -g lcmuser lcmuser && \
|
|
23
|
+
mkhomedir_helper lcmuser && \
|
|
24
|
+
chown lcmuser: /home && \
|
|
25
|
+
chown lcmuser: /src
|
|
26
|
+
USER lcmuser
|
|
27
|
+
|
|
28
|
+
RUN gpg2 --keyserver hkp://pool.sks-keyservers.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
|
|
29
|
+
RUN curl -sSL https://get.rvm.io | bash -s stable
|
|
24
30
|
|
|
25
31
|
# login shell is required by rvm
|
|
26
|
-
RUN /bin/bash -l -c ". /
|
|
32
|
+
RUN /bin/bash -l -c ". /home/lcmuser/.rvm/scripts/rvm && rvm install jruby-9.2.5.0 && gem update --system \
|
|
27
33
|
&& gem install bundler rake"
|
|
28
34
|
|
|
29
35
|
ENV GOODDATA_RUBY_COMMIT=$GIT_COMMIT
|
|
@@ -35,6 +41,6 @@ ADD ./VERSION .
|
|
|
35
41
|
ADD ./Gemfile .
|
|
36
42
|
ADD ./gooddata.gemspec .
|
|
37
43
|
|
|
38
|
-
RUN /bin/bash -l -c ". /
|
|
44
|
+
RUN /bin/bash -l -c ". /home/lcmuser/.rvm/scripts/rvm && bundle install"
|
|
39
45
|
|
|
40
46
|
CMD [ "./bin/help.sh" ]
|
data/SDK_VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
2.1.
|
|
1
|
+
2.1.3
|
data/bin/hello_world.sh
CHANGED
data/bin/help.sh
CHANGED
data/bin/provision.sh
CHANGED
data/bin/release.sh
CHANGED
data/bin/rollout.sh
CHANGED
data/bin/run_smoke_tests.sh
CHANGED
|
@@ -85,7 +85,7 @@ def clean_up!(client, force, days)
|
|
|
85
85
|
delete_project_by_title(/LCM SPEC PROJECT/, projects, days, force)
|
|
86
86
|
delete_project_by_title(/LCM spec Client With Conflicting LDM Changes/, projects, days, force)
|
|
87
87
|
delete_project_by_title(/LCM spec master project/, projects, days, force)
|
|
88
|
-
delete_project_by_title(/users brick load test/,
|
|
88
|
+
delete_project_by_title(/users brick load test/, projects, days, force)
|
|
89
89
|
delete_ads_by_title(/Development ADS/, client, days, force)
|
|
90
90
|
delete_ads_by_title(/Production ADS/, client, days, force)
|
|
91
91
|
delete_ads_by_title(/TEST ADS/, client, days, force)
|
data/bin/user_filters.sh
CHANGED
data/bin/users.sh
CHANGED
data/lcm.rake
CHANGED
|
@@ -3,7 +3,7 @@ require 'fileutils'
|
|
|
3
3
|
require 'pathname'
|
|
4
4
|
require 'rspec/core/rake_task'
|
|
5
5
|
|
|
6
|
-
test_cases = %i[integration slow userprov load
|
|
6
|
+
test_cases = %i[integration slow userprov load]
|
|
7
7
|
|
|
8
8
|
# Schema for new Bricks.
|
|
9
9
|
brick_info_schema = {
|
|
@@ -83,6 +83,20 @@ namespace :test do
|
|
|
83
83
|
end
|
|
84
84
|
end
|
|
85
85
|
|
|
86
|
+
desc 'Run smoke tests'
|
|
87
|
+
RSpec::Core::RakeTask.new(:smoke) do |task|
|
|
88
|
+
ENV['GD_LCM_SPEC_SEGMENTS_MULTIPLIER'] = '1'
|
|
89
|
+
ENV['GD_LCM_SPEC_WORKSPACES_MULTIPLIER'] = '1'
|
|
90
|
+
|
|
91
|
+
ENV['GD_LCM_SPEC_USER_COUNT'] = '1'
|
|
92
|
+
ENV['GD_LCM_SPEC_PROJECT_COUNT'] = '1'
|
|
93
|
+
|
|
94
|
+
ENV['VCR_ON'] = 'false'
|
|
95
|
+
ENV['GD_LCM_SMOKE_TEST'] = 'true'
|
|
96
|
+
|
|
97
|
+
task.pattern = 'spec/lcm/load/**/*_spec.rb'
|
|
98
|
+
end
|
|
99
|
+
|
|
86
100
|
namespace :docker do
|
|
87
101
|
test_cases.each do |t|
|
|
88
102
|
desc "Run #{t} tests in Docker"
|
|
@@ -37,7 +37,7 @@ module GoodData
|
|
|
37
37
|
end
|
|
38
38
|
|
|
39
39
|
def time_from_action_start(now = Time.now)
|
|
40
|
-
fail_if_development 'No action is being profiled' unless action
|
|
40
|
+
GoodData::LCM2::Helpers.fail_if_development 'No action is being profiled' unless action
|
|
41
41
|
(now - @action_start) * 1000
|
|
42
42
|
end
|
|
43
43
|
|
|
@@ -47,7 +47,7 @@ module GoodData
|
|
|
47
47
|
# @param [Logger] logger, logger that should log current context info
|
|
48
48
|
# @param [Time] now, allows to specify exact time, when outer call was performed
|
|
49
49
|
def start_action(next_action, logger = nil, now = Time.now)
|
|
50
|
-
fail_if_development 'An action is already being profiled' if action
|
|
50
|
+
GoodData::LCM2::Helpers.fail_if_development 'An action is already being profiled' if action
|
|
51
51
|
|
|
52
52
|
self.action = next_action
|
|
53
53
|
@action_start = now
|
|
@@ -59,7 +59,7 @@ module GoodData
|
|
|
59
59
|
#
|
|
60
60
|
# @param [Logger] logger, logger that should log current context info
|
|
61
61
|
def end_action(logger = nil)
|
|
62
|
-
fail_if_development 'No matching action to start found' unless action
|
|
62
|
+
GoodData::LCM2::Helpers.fail_if_development 'No matching action to start found' unless action
|
|
63
63
|
|
|
64
64
|
logger.info '' if logger
|
|
65
65
|
self.status = :not_in_action
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
# LICENSE file in the root directory of this source tree.
|
|
6
6
|
|
|
7
7
|
require 'csv'
|
|
8
|
+
require 'fileutils'
|
|
8
9
|
|
|
9
10
|
module GoodData
|
|
10
11
|
module Helpers
|
|
@@ -38,6 +39,18 @@ module GoodData
|
|
|
38
39
|
res
|
|
39
40
|
end
|
|
40
41
|
|
|
42
|
+
# Read data from csv as an array of hashes with symbol keys and parsed integers
|
|
43
|
+
# @option filename String
|
|
44
|
+
def read_as_hash(filename)
|
|
45
|
+
res = []
|
|
46
|
+
return res unless File.exist? filename
|
|
47
|
+
|
|
48
|
+
CSV.parse(File.read(filename), headers: true, header_converters: :symbol, converters: :integer).map do |row|
|
|
49
|
+
res << row.to_hash
|
|
50
|
+
end
|
|
51
|
+
res
|
|
52
|
+
end
|
|
53
|
+
|
|
41
54
|
# Write data to CSV
|
|
42
55
|
# @option opts [String] :path File to write data to
|
|
43
56
|
# @option opts [Array] :data Mandatory array of data to write
|
|
@@ -55,6 +68,20 @@ module GoodData
|
|
|
55
68
|
end
|
|
56
69
|
end
|
|
57
70
|
end
|
|
71
|
+
|
|
72
|
+
# Ammend a hash to CSV in a smart manner
|
|
73
|
+
# @option filename String
|
|
74
|
+
# @option data Hash
|
|
75
|
+
def ammend_line(filename, data)
|
|
76
|
+
current_data = read_as_hash(filename)
|
|
77
|
+
data_to_write = (current_data << data).map(&:sort).map { |r| Hash[r] }
|
|
78
|
+
FileUtils.mkpath(filename.split('/')[0...-1].join('/'))
|
|
79
|
+
CSV.open(filename, 'w', write_headers: true, headers: data_to_write.first.keys) do |csv|
|
|
80
|
+
data_to_write.each do |d|
|
|
81
|
+
csv << d.values
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
58
85
|
end
|
|
59
86
|
end
|
|
60
87
|
end
|
|
@@ -16,7 +16,7 @@ module GoodData
|
|
|
16
16
|
param :gdc_gd_client, instance_of(Type::GdClientType), required: true
|
|
17
17
|
|
|
18
18
|
description 'ADS Client'
|
|
19
|
-
param :ads_client, instance_of(Type::AdsClientType), required:
|
|
19
|
+
param :ads_client, instance_of(Type::AdsClientType), required: false
|
|
20
20
|
|
|
21
21
|
description 'Segments to manage'
|
|
22
22
|
param :segments, array_of(instance_of(Type::SegmentType)), required: true
|
|
@@ -26,6 +26,12 @@ module GoodData
|
|
|
26
26
|
|
|
27
27
|
description 'DataProduct'
|
|
28
28
|
param :data_product, instance_of(Type::GDDataProductType), required: false
|
|
29
|
+
|
|
30
|
+
description 'Organization Name'
|
|
31
|
+
param :organization, instance_of(Type::StringType), required: false
|
|
32
|
+
|
|
33
|
+
description 'Domain'
|
|
34
|
+
param :domain, instance_of(Type::StringType), required: false
|
|
29
35
|
end
|
|
30
36
|
|
|
31
37
|
RESULT_HEADER = [
|
|
@@ -47,12 +53,22 @@ module GoodData
|
|
|
47
53
|
|
|
48
54
|
raise "Client(s) missing workspace: #{missing_project_clients.join(', ')}. Please make sure all clients have workspace." unless missing_project_clients.empty?
|
|
49
55
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
+
domain_name = params.organization || params.domain
|
|
57
|
+
fail "Either organisation or domain has to be specified in params" unless domain_name
|
|
58
|
+
|
|
59
|
+
client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
|
|
60
|
+
|
|
61
|
+
if params.ads_client
|
|
62
|
+
latest_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
|
|
63
|
+
params.release_table_name,
|
|
64
|
+
params.ads_client,
|
|
65
|
+
segment.segment_id
|
|
66
|
+
)
|
|
67
|
+
else
|
|
68
|
+
latest_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, segment.segment_id)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
latest_master = client.projects(latest_master[:master_project_id])
|
|
56
72
|
|
|
57
73
|
# TODO: Check res.first.nil? || res.first[:master_project_id].nil?
|
|
58
74
|
master_pid = latest_master.pid
|
|
@@ -25,7 +25,7 @@ module GoodData
|
|
|
25
25
|
param :domain, instance_of(Type::StringType), required: false
|
|
26
26
|
|
|
27
27
|
description 'ADS Client'
|
|
28
|
-
param :ads_client, instance_of(Type::AdsClientType), required:
|
|
28
|
+
param :ads_client, instance_of(Type::AdsClientType), required: false
|
|
29
29
|
|
|
30
30
|
description 'Segments to manage'
|
|
31
31
|
param :segments, array_of(instance_of(Type::SegmentType)), required: true
|
|
@@ -71,7 +71,7 @@ module GoodData
|
|
|
71
71
|
ads_output_stage_prefix = segment_in.ads_output_stage_prefix
|
|
72
72
|
|
|
73
73
|
# Create master project Postgres
|
|
74
|
-
version = get_project_version(params, segment_id) + 1
|
|
74
|
+
version = get_project_version(params, domain_name, segment_id) + 1
|
|
75
75
|
|
|
76
76
|
master_name = segment_in.master_name.gsub('#{version}', version.to_s)
|
|
77
77
|
|
|
@@ -150,12 +150,16 @@ module GoodData
|
|
|
150
150
|
}
|
|
151
151
|
end
|
|
152
152
|
|
|
153
|
-
def get_project_version(params, segment_id)
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
153
|
+
def get_project_version(params, domain_name, segment_id)
|
|
154
|
+
if params.ads_client
|
|
155
|
+
current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
|
|
156
|
+
params.release_table_name,
|
|
157
|
+
params.ads_client,
|
|
158
|
+
segment_id
|
|
159
|
+
)
|
|
160
|
+
else
|
|
161
|
+
current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, segment_id)
|
|
162
|
+
end
|
|
159
163
|
return 0 unless current_master
|
|
160
164
|
current_master[:version].to_i
|
|
161
165
|
end
|
|
@@ -13,7 +13,7 @@ module GoodData
|
|
|
13
13
|
|
|
14
14
|
PARAMS = define_params(self) do
|
|
15
15
|
description 'ADS Client'
|
|
16
|
-
param :ads_client, instance_of(Type::AdsClientType), required:
|
|
16
|
+
param :ads_client, instance_of(Type::AdsClientType), required: false
|
|
17
17
|
|
|
18
18
|
description 'Table Name'
|
|
19
19
|
param :release_table_name, instance_of(Type::StringType), required: false
|
|
@@ -28,24 +28,32 @@ module GoodData
|
|
|
28
28
|
|
|
29
29
|
class << self
|
|
30
30
|
def call(params)
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
path = File.expand_path('../../data/create_lcm_release.sql.erb', __FILE__)
|
|
36
|
-
query = GoodData::Helpers::ErbHelper.template_file(path, replacements)
|
|
37
|
-
|
|
38
|
-
sql_result = params.ads_client.execute(query)
|
|
39
|
-
|
|
40
|
-
# TODO: Format
|
|
41
|
-
GoodData.logger.info(JSON.pretty_generate(sql_result))
|
|
42
|
-
|
|
43
|
-
[
|
|
44
|
-
{
|
|
45
|
-
table_name: replacements[:table_name],
|
|
46
|
-
status: 'ok'
|
|
31
|
+
if params.ads_client
|
|
32
|
+
replacements = {
|
|
33
|
+
table_name: params.release_table_name || DEFAULT_TABLE_NAME
|
|
47
34
|
}
|
|
48
|
-
|
|
35
|
+
|
|
36
|
+
path = File.expand_path('../data/create_lcm_release.sql.erb', __dir__)
|
|
37
|
+
query = GoodData::Helpers::ErbHelper.template_file(path, replacements)
|
|
38
|
+
|
|
39
|
+
sql_result = params.ads_client.execute(query)
|
|
40
|
+
|
|
41
|
+
# TODO: Format
|
|
42
|
+
GoodData.logger.info(JSON.pretty_generate(sql_result))
|
|
43
|
+
|
|
44
|
+
[
|
|
45
|
+
{
|
|
46
|
+
table_name: replacements[:table_name],
|
|
47
|
+
status: 'ok'
|
|
48
|
+
}
|
|
49
|
+
]
|
|
50
|
+
else
|
|
51
|
+
[
|
|
52
|
+
{
|
|
53
|
+
status: 'ok'
|
|
54
|
+
}
|
|
55
|
+
]
|
|
56
|
+
end
|
|
49
57
|
end
|
|
50
58
|
end
|
|
51
59
|
end
|
|
@@ -31,7 +31,7 @@ module GoodData
|
|
|
31
31
|
param :release_table_name, instance_of(Type::StringType), required: false
|
|
32
32
|
|
|
33
33
|
description 'ADS Client'
|
|
34
|
-
param :ads_client, instance_of(Type::AdsClientType), required:
|
|
34
|
+
param :ads_client, instance_of(Type::AdsClientType), required: false
|
|
35
35
|
|
|
36
36
|
description 'Additional Hidden Parameters'
|
|
37
37
|
param :additional_hidden_params, instance_of(Type::HashType), required: false
|
|
@@ -61,14 +61,18 @@ module GoodData
|
|
|
61
61
|
end
|
|
62
62
|
|
|
63
63
|
results = segments.map do |segment|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
64
|
+
if params.ads_client
|
|
65
|
+
current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
|
|
66
|
+
params.release_table_name,
|
|
67
|
+
params.ads_client,
|
|
68
|
+
segment.segment_id
|
|
69
|
+
)
|
|
70
|
+
else
|
|
71
|
+
current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, segment.segment_id)
|
|
72
|
+
end
|
|
69
73
|
|
|
70
74
|
# TODO: Check res.first.nil? || res.first[:master_project_id].nil?
|
|
71
|
-
master = client.projects(current_master)
|
|
75
|
+
master = client.projects(current_master[:master_project_id])
|
|
72
76
|
|
|
73
77
|
segment.master_project = master
|
|
74
78
|
segment.save
|
|
@@ -13,7 +13,7 @@ module GoodData
|
|
|
13
13
|
|
|
14
14
|
PARAMS = define_params(self) do
|
|
15
15
|
description 'ADS Client'
|
|
16
|
-
param :ads_client, instance_of(Type::AdsClientType), required:
|
|
16
|
+
param :ads_client, instance_of(Type::AdsClientType), required: false
|
|
17
17
|
|
|
18
18
|
description 'Client Used for Connecting to GD'
|
|
19
19
|
param :gdc_gd_client, instance_of(Type::GdClientType), required: true
|
|
@@ -55,7 +55,7 @@ module GoodData
|
|
|
55
55
|
table_name: params.release_table_name || DEFAULT_TABLE_NAME
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
-
update_release_table(params, placeholders)
|
|
58
|
+
update_release_table(params, placeholders, domain_name)
|
|
59
59
|
|
|
60
60
|
{
|
|
61
61
|
segment: segment_id,
|
|
@@ -74,13 +74,17 @@ module GoodData
|
|
|
74
74
|
result
|
|
75
75
|
end
|
|
76
76
|
|
|
77
|
-
def update_release_table(params, placeholders)
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
77
|
+
def update_release_table(params, placeholders, domain_id)
|
|
78
|
+
if params.ads_client
|
|
79
|
+
path = File.expand_path('../data/insert_into_lcm_release.sql.erb', __dir__)
|
|
80
|
+
default_query = GoodData::Helpers::ErbHelper.template_file(path, placeholders)
|
|
81
|
+
temp_query = (params.query && params.query.insert) || default_query
|
|
82
|
+
query = replace_placeholders(temp_query, placeholders)
|
|
82
83
|
|
|
83
|
-
|
|
84
|
+
params.ads_client.execute(query)
|
|
85
|
+
else
|
|
86
|
+
GoodData::LCM2::Helpers.update_latest_master_to_nfs(domain_id, placeholders[:segment_id], placeholders[:master_project_id], placeholders[:version])
|
|
87
|
+
end
|
|
84
88
|
end
|
|
85
89
|
end
|
|
86
90
|
end
|
|
@@ -8,9 +8,10 @@ module GoodData
|
|
|
8
8
|
module LCM2
|
|
9
9
|
class Helpers
|
|
10
10
|
DEFAULT_TABLE_NAME = 'LCM_RELEASE'
|
|
11
|
+
DEFAULT_NFS_DIRECTORY = 'release-tables'
|
|
11
12
|
|
|
12
13
|
class << self
|
|
13
|
-
def
|
|
14
|
+
def latest_master_project_from_ads(release_table_name, ads_client, segment_id)
|
|
14
15
|
replacements = {
|
|
15
16
|
table_name: release_table_name || DEFAULT_TABLE_NAME,
|
|
16
17
|
segment_id: segment_id
|
|
@@ -23,6 +24,24 @@ module GoodData
|
|
|
23
24
|
sorted = res.sort_by { |row| row[:version] }
|
|
24
25
|
sorted.last
|
|
25
26
|
end
|
|
27
|
+
|
|
28
|
+
def latest_master_project_from_nfs(domain_id, segment_id)
|
|
29
|
+
data = GoodData::Helpers::Csv.read_as_hash(path_to_release_table_file(domain_id, segment_id))
|
|
30
|
+
data.sort_by { |master| master[:version] }
|
|
31
|
+
.reverse.first
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def update_latest_master_to_nfs(domain_id, segment_id, master_pid, version)
|
|
35
|
+
GoodData::Helpers::Csv.ammend_line(
|
|
36
|
+
path_to_release_table_file(domain_id, segment_id),
|
|
37
|
+
master_project_id: master_pid,
|
|
38
|
+
version: version
|
|
39
|
+
)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def path_to_release_table_file(domain_id, segment_id)
|
|
43
|
+
[DEFAULT_NFS_DIRECTORY, domain_id, segment_id + '.csv'].join('/')
|
|
44
|
+
end
|
|
26
45
|
end
|
|
27
46
|
end
|
|
28
47
|
end
|
|
@@ -17,6 +17,7 @@ require 'fileutils'
|
|
|
17
17
|
require 'multi_json'
|
|
18
18
|
require 'open-uri'
|
|
19
19
|
require 'zip'
|
|
20
|
+
require 'csv'
|
|
20
21
|
|
|
21
22
|
##
|
|
22
23
|
# Module containing classes that counter-part GoodData server-side meta-data
|
|
@@ -152,11 +153,9 @@ module GoodData
|
|
|
152
153
|
# @return [Hash] Batch upload result
|
|
153
154
|
def upload_multiple_data(data, project_blueprint, options = { :client => GoodData.connection, :project => GoodData.project })
|
|
154
155
|
client, project = GoodData.get_client_and_project(options)
|
|
155
|
-
|
|
156
156
|
project ||= GoodData.project
|
|
157
157
|
|
|
158
158
|
manifest = {
|
|
159
|
-
|
|
160
159
|
'dataSetSLIManifestList' => data.map do |d|
|
|
161
160
|
mode = d[:options] && d[:options][:mode] ? d[:options][:mode] : options[:mode] || 'FULL'
|
|
162
161
|
GoodData::Model::ToManifest.dataset_to_manifest(project_blueprint, d[:dataset], mode)
|
|
@@ -165,7 +164,6 @@ module GoodData
|
|
|
165
164
|
|
|
166
165
|
csv_headers = []
|
|
167
166
|
|
|
168
|
-
# create a temporary zip file
|
|
169
167
|
dir = Dir.mktmpdir
|
|
170
168
|
begin
|
|
171
169
|
Zip::File.open("#{dir}/upload.zip", Zip::File::CREATE) do |zip|
|
|
@@ -173,26 +171,27 @@ module GoodData
|
|
|
173
171
|
zip.get_output_stream('upload_info.json') { |f| f.puts JSON.pretty_generate(manifest) }
|
|
174
172
|
|
|
175
173
|
data.zip(manifest['dataSetSLIManifestList']).each do |item|
|
|
174
|
+
column_mapping = item[0][:options] ? item[0][:options][:column_mapping] : nil
|
|
176
175
|
path = item[0][:data]
|
|
177
176
|
path = item[0][:data].path if item[0][:data].respond_to? :path
|
|
178
177
|
inline_data = !path.is_a?(String)
|
|
179
|
-
|
|
178
|
+
|
|
179
|
+
data_to_upload = inline_data ? path : File.open(path)
|
|
180
180
|
|
|
181
181
|
filename = item[1]['dataSetSLIManifest']['file']
|
|
182
182
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
183
|
+
zip.get_output_stream(filename) do |file|
|
|
184
|
+
data_to_upload.each_with_index do |row, index|
|
|
185
|
+
row = CSV.parse(row).first unless inline_data
|
|
186
|
+
|
|
187
|
+
if index.zero?
|
|
188
|
+
row.map! { |h| column_mapping.key(h) || h } if column_mapping
|
|
189
|
+
csv_headers << row
|
|
188
190
|
end
|
|
191
|
+
|
|
192
|
+
file.puts row.to_csv
|
|
189
193
|
end
|
|
190
|
-
else
|
|
191
|
-
csv_header = File.open(path, &:gets).chomp.split(',')
|
|
192
|
-
zip.add(filename, path)
|
|
193
194
|
end
|
|
194
|
-
|
|
195
|
-
csv_headers << csv_header
|
|
196
195
|
end
|
|
197
196
|
end
|
|
198
197
|
|
|
@@ -240,6 +239,8 @@ module GoodData
|
|
|
240
239
|
manifest_cols = parts.flatten.map { |c| c['columnName'] }
|
|
241
240
|
|
|
242
241
|
# extract some human readable error message from the webdav file
|
|
242
|
+
csv_headers.map!(&:to_sym)
|
|
243
|
+
manifest_cols.map!(&:to_sym)
|
|
243
244
|
manifest_extra = manifest_cols - csv_headers
|
|
244
245
|
csv_extra = csv_headers - manifest_cols
|
|
245
246
|
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: gooddata
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.1.
|
|
4
|
+
version: 2.1.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Pavel Kolesnikov
|
|
@@ -14,7 +14,7 @@ authors:
|
|
|
14
14
|
autorequire:
|
|
15
15
|
bindir: bin
|
|
16
16
|
cert_chain: []
|
|
17
|
-
date: 2019-03
|
|
17
|
+
date: 2019-04-03 00:00:00.000000000 Z
|
|
18
18
|
dependencies:
|
|
19
19
|
- !ruby/object:Gem::Dependency
|
|
20
20
|
name: license_finder
|
|
@@ -993,7 +993,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
993
993
|
- !ruby/object:Gem::Version
|
|
994
994
|
version: '0'
|
|
995
995
|
requirements: []
|
|
996
|
-
|
|
996
|
+
rubyforge_project:
|
|
997
|
+
rubygems_version: 2.6.13
|
|
997
998
|
signing_key:
|
|
998
999
|
specification_version: 4
|
|
999
1000
|
summary: A convenient Ruby wrapper around the GoodData RESTful API
|