gooddata 2.1.19-java → 2.2.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gdc-ii-config.yaml +1 -1
- data/.github/workflows/build.yml +66 -0
- data/.github/workflows/pre-merge.yml +72 -0
- data/CHANGELOG.md +38 -0
- data/Dockerfile +21 -14
- data/Dockerfile.jruby +1 -11
- data/README.md +1 -2
- data/SDK_VERSION +1 -1
- data/VERSION +1 -1
- data/ci/mssql/pom.xml +62 -0
- data/ci/mysql/pom.xml +57 -0
- data/ci/redshift/pom.xml +1 -1
- data/docker-compose.lcm.yml +0 -3
- data/gooddata.gemspec +2 -1
- data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
- data/lcm.rake +2 -8
- data/lib/gooddata/bricks/middleware/aws_middleware.rb +35 -9
- data/lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb +98 -0
- data/lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mssql/mssql_client.rb +122 -0
- data/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme +0 -0
- data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +111 -0
- data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +0 -1
- data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +18 -1
- data/lib/gooddata/helpers/data_helper.rb +9 -4
- data/lib/gooddata/lcm/actions/collect_meta.rb +3 -1
- data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +3 -2
- data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -7
- data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +64 -0
- data/lib/gooddata/lcm/actions/synchronize_ldm.rb +19 -8
- data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +12 -9
- data/lib/gooddata/lcm/actions/update_metric_formats.rb +185 -0
- data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb +5 -0
- data/lib/gooddata/lcm/helpers/release_table_helper.rb +42 -8
- data/lib/gooddata/lcm/lcm2.rb +5 -0
- data/lib/gooddata/mixins/md_object_query.rb +1 -0
- data/lib/gooddata/models/data_source.rb +5 -1
- data/lib/gooddata/models/dataset_mapping.rb +36 -0
- data/lib/gooddata/models/metadata/label.rb +26 -27
- data/lib/gooddata/models/project.rb +34 -9
- data/lib/gooddata/models/schedule.rb +13 -1
- data/lib/gooddata/models/user_filters/user_filter_builder.rb +58 -53
- data/lib/gooddata/rest/phmap.rb +1 -0
- metadata +44 -18
- data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb +0 -37
@@ -124,6 +124,7 @@ module GoodData
|
|
124
124
|
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, number_of_clients=#{all_clients.size}, data_rows=#{user_filters.size}")
|
125
125
|
|
126
126
|
GoodData.logger.info("Synchronizing in mode \"#{mode}\"")
|
127
|
+
results = []
|
127
128
|
case mode
|
128
129
|
when 'sync_project', 'sync_one_project_based_on_pid', 'sync_one_project_based_on_custom_id'
|
129
130
|
if mode == 'sync_one_project_based_on_pid'
|
@@ -134,7 +135,9 @@ module GoodData
|
|
134
135
|
user_filters = user_filters.select { |f| f[:pid] == filter } if filter
|
135
136
|
|
136
137
|
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{user_filters.size}")
|
137
|
-
sync_user_filters(project, user_filters, run_params, symbolized_config)
|
138
|
+
current_results = sync_user_filters(project, user_filters, run_params, symbolized_config)
|
139
|
+
|
140
|
+
results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
|
138
141
|
when 'sync_multiple_projects_based_on_pid', 'sync_multiple_projects_based_on_custom_id'
|
139
142
|
users_by_project = run_params[:users_brick_input].group_by { |u| u[:pid] }
|
140
143
|
user_filters.group_by { |u| u[:pid] }.flat_map.pmap do |id, new_filters|
|
@@ -149,7 +152,9 @@ module GoodData
|
|
149
152
|
end
|
150
153
|
|
151
154
|
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{id}, data_rows=#{new_filters.size}")
|
152
|
-
sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
|
155
|
+
current_results = sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
|
156
|
+
|
157
|
+
results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
|
153
158
|
end
|
154
159
|
when 'sync_domain_client_workspaces'
|
155
160
|
domain_clients = all_clients
|
@@ -161,7 +166,6 @@ module GoodData
|
|
161
166
|
working_client_ids = []
|
162
167
|
|
163
168
|
users_by_project = run_params[:users_brick_input].group_by { |u| u[:pid] }
|
164
|
-
results = []
|
165
169
|
user_filters.group_by { |u| u[multiple_projects_column] }.flat_map.pmap do |client_id, new_filters|
|
166
170
|
users = users_by_project[client_id]
|
167
171
|
fail "Client id cannot be empty" if client_id.blank?
|
@@ -182,7 +186,7 @@ module GoodData
|
|
182
186
|
|
183
187
|
GoodData.gd_logger.info("Synchronizing in mode=#{mode}, client_id=#{client_id}, data_rows=#{new_filters.size}")
|
184
188
|
partial_results = sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
|
185
|
-
results.concat(partial_results[:results])
|
189
|
+
results.concat(partial_results[:results]) unless partial_results.nil? || partial_results[:results].empty?
|
186
190
|
end
|
187
191
|
|
188
192
|
unless run_params[:do_not_touch_filters_that_are_not_mentioned]
|
@@ -197,17 +201,16 @@ module GoodData
|
|
197
201
|
GoodData.gd_logger.info("Delete all filters in project_id=#{current_project.pid}, client_id=#{c.client_id}")
|
198
202
|
current_results = sync_user_filters(current_project, [], run_params.merge(users_brick_input: users), symbolized_config)
|
199
203
|
|
200
|
-
results.concat(current_results[:results])
|
204
|
+
results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
|
201
205
|
rescue StandardError => e
|
202
206
|
params.gdc_logger.error "Failed to clear filters of #{c.client_id} due to: #{e.inspect}"
|
203
207
|
end
|
204
208
|
end
|
205
209
|
end
|
206
|
-
|
207
|
-
{
|
208
|
-
results: results
|
209
|
-
}
|
210
210
|
end
|
211
|
+
{
|
212
|
+
results: results
|
213
|
+
}
|
211
214
|
end
|
212
215
|
|
213
216
|
def sync_user_filters(project, filters, params, filters_config)
|
@@ -0,0 +1,185 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
|
4
|
+
# This source code is licensed under the BSD-style license found in the
|
5
|
+
# LICENSE file in the root directory of this source tree.
|
6
|
+
|
7
|
+
require_relative 'base_action'
|
8
|
+
|
9
|
+
module GoodData
|
10
|
+
module LCM2
|
11
|
+
class UpdateMetricFormats < BaseAction
|
12
|
+
DESCRIPTION = 'Localize Metric Formats'
|
13
|
+
|
14
|
+
PARAMS = define_params(self) do
|
15
|
+
description 'Synchronization Info'
|
16
|
+
param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
|
17
|
+
|
18
|
+
description 'Client Used for Connecting to GD'
|
19
|
+
param :gdc_gd_client, instance_of(Type::GdClientType), required: true
|
20
|
+
|
21
|
+
description 'Organization Name'
|
22
|
+
param :organization, instance_of(Type::StringType), required: false
|
23
|
+
|
24
|
+
description 'DataProduct to manage'
|
25
|
+
param :data_product, instance_of(Type::GDDataProductType), required: false
|
26
|
+
|
27
|
+
description 'Logger'
|
28
|
+
param :gdc_logger, instance_of(Type::GdLogger), required: true
|
29
|
+
|
30
|
+
description 'ADS Client'
|
31
|
+
param :ads_client, instance_of(Type::AdsClientType), required: false
|
32
|
+
|
33
|
+
description 'Input Source'
|
34
|
+
param :input_source, instance_of(Type::HashType), required: false
|
35
|
+
|
36
|
+
description 'Localization query'
|
37
|
+
param :localization_query, instance_of(Type::StringType), required: false
|
38
|
+
end
|
39
|
+
|
40
|
+
RESULT_HEADER = %i[action ok_clients error_clients]
|
41
|
+
|
42
|
+
class << self
|
43
|
+
def load_metric_data(params)
|
44
|
+
if params&.dig(:input_source, :metric_format) && params[:input_source][:metric_format].present?
|
45
|
+
metric_input_source = validate_input_source(params[:input_source])
|
46
|
+
else
|
47
|
+
return nil
|
48
|
+
end
|
49
|
+
|
50
|
+
metric_data_source = GoodData::Helpers::DataSource.new(metric_input_source)
|
51
|
+
begin
|
52
|
+
temp_csv = without_check(PARAMS, params) do
|
53
|
+
File.open(metric_data_source.realize(params), 'r:UTF-8')
|
54
|
+
end
|
55
|
+
rescue StandardError => e
|
56
|
+
GoodData.logger.warn("Unable to get metric input source, skip updating metric formats. Error: #{e.message} - #{e}")
|
57
|
+
return nil
|
58
|
+
end
|
59
|
+
|
60
|
+
metrics_hash = GoodData::Helpers::Csv.read_as_hash temp_csv
|
61
|
+
return nil if metrics_hash.empty?
|
62
|
+
|
63
|
+
expected_keys = %w[tag client_id format]
|
64
|
+
unless expected_keys.map(&:to_sym).all? { |s| metrics_hash.first.key? s }
|
65
|
+
GoodData.logger.warn("The input metric data is incorrect, expecting the following fields: #{expected_keys}")
|
66
|
+
return nil
|
67
|
+
end
|
68
|
+
metrics_hash
|
69
|
+
end
|
70
|
+
|
71
|
+
def validate_input_source(input_source)
|
72
|
+
type = input_source[:type] if input_source&.dig(:type)
|
73
|
+
metric_format = input_source[:metric_format]
|
74
|
+
raise "Incorrect configuration: 'type' of 'input_source' is required" if type.blank?
|
75
|
+
|
76
|
+
modified_input_source = input_source
|
77
|
+
case type
|
78
|
+
when 'ads', 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql'
|
79
|
+
if metric_format[:query].blank?
|
80
|
+
GoodData.logger.warn("The metric input_source '#{type}' is missing property 'query'")
|
81
|
+
return nil
|
82
|
+
end
|
83
|
+
|
84
|
+
modified_input_source[:query] = metric_format[:query]
|
85
|
+
return modified_input_source
|
86
|
+
when 's3'
|
87
|
+
if metric_format[:file].blank?
|
88
|
+
GoodData.logger.warn("The metric input_source '#{type}' is missing property 'file'")
|
89
|
+
return nil
|
90
|
+
end
|
91
|
+
|
92
|
+
if modified_input_source.key?(:key)
|
93
|
+
modified_input_source[:key] = metric_format[:file]
|
94
|
+
else
|
95
|
+
modified_input_source[:file] = metric_format[:file]
|
96
|
+
end
|
97
|
+
return modified_input_source
|
98
|
+
when 'blobStorage'
|
99
|
+
if metric_format[:file].blank?
|
100
|
+
GoodData.logger.warn("The metric input_source '#{type}' is missing property 'file'")
|
101
|
+
return nil
|
102
|
+
end
|
103
|
+
|
104
|
+
modified_input_source[:file] = metric_format[:file]
|
105
|
+
return modified_input_source
|
106
|
+
when 'staging'
|
107
|
+
if metric_format[:file].blank?
|
108
|
+
GoodData.logger.warn("The metric input_source '#{type}' is missing property 'file'")
|
109
|
+
return nil
|
110
|
+
end
|
111
|
+
|
112
|
+
modified_input_source[:path] = metric_format[:file]
|
113
|
+
return modified_input_source
|
114
|
+
when 'web'
|
115
|
+
if metric_format[:url].blank?
|
116
|
+
GoodData.logger.warn("The metric input_source '#{type}' is missing property 'url'")
|
117
|
+
return nil
|
118
|
+
end
|
119
|
+
|
120
|
+
modified_input_source[:url] = metric_format[:url]
|
121
|
+
return modified_input_source
|
122
|
+
else
|
123
|
+
return nil
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def get_clients_metrics(metric_data)
|
128
|
+
return {} if metric_data.nil?
|
129
|
+
|
130
|
+
metric_groups = {}
|
131
|
+
clients = metric_data.map { |row| row[:client_id] }.uniq
|
132
|
+
clients.each do |client|
|
133
|
+
next if client.blank?
|
134
|
+
|
135
|
+
formats = {}
|
136
|
+
metric_data.select { |row| row[:client_id] == client && row[:tag].present? && row[:format].present? }.each { |row| formats[row[:tag]] = row[:format] }
|
137
|
+
metric_groups[client.to_s] ||= formats
|
138
|
+
end
|
139
|
+
metric_groups
|
140
|
+
end
|
141
|
+
|
142
|
+
def call(params)
|
143
|
+
data = load_metric_data(params)
|
144
|
+
result = []
|
145
|
+
return result if data.nil?
|
146
|
+
|
147
|
+
metric_group = get_clients_metrics(data)
|
148
|
+
return result if metric_group.empty?
|
149
|
+
|
150
|
+
GoodData.logger.debug("Clients have metrics which will be modified: #{metric_group.keys}")
|
151
|
+
updated_clients = params.synchronize.map { |segment| segment.to.map { |client| client[:client_id] } }.flatten.uniq
|
152
|
+
GoodData.logger.debug("Updating clients: #{updated_clients}")
|
153
|
+
data_product = params.data_product
|
154
|
+
data_product_clients = data_product.clients
|
155
|
+
number_client_ok = 0
|
156
|
+
number_client_error = 0
|
157
|
+
metric_group.each do |client_id, formats|
|
158
|
+
next unless updated_clients.include?(client_id)
|
159
|
+
|
160
|
+
client = data_product_clients.find { |c| c.id == client_id }
|
161
|
+
begin
|
162
|
+
GoodData.logger.info("Start updating metric format for client: '#{client_id}'")
|
163
|
+
metrics = client.project.metrics.to_a
|
164
|
+
formats.each do |tag, format|
|
165
|
+
next if tag.blank? || format.blank?
|
166
|
+
|
167
|
+
metrics_to_be_updated = metrics.select { |metric| metric.tags.include?(tag) }
|
168
|
+
metrics_to_be_updated.each do |metric|
|
169
|
+
metric.format = format
|
170
|
+
metric.save
|
171
|
+
end
|
172
|
+
end
|
173
|
+
number_client_ok += 1
|
174
|
+
GoodData.logger.info("Finished updating metric format for client: '#{client_id}'")
|
175
|
+
rescue StandardError => e
|
176
|
+
number_client_error += 1
|
177
|
+
GoodData.logger.warn("Failed to update metric format for client: '#{client_id}'. Error: #{e.message} - #{e}")
|
178
|
+
end
|
179
|
+
end
|
180
|
+
[{ :action => 'Update metric format', :ok_clients => number_client_ok, :error_clients => number_client_error }]
|
181
|
+
end
|
182
|
+
end
|
183
|
+
end
|
184
|
+
end
|
185
|
+
end
|
@@ -12,6 +12,21 @@ module GoodData
|
|
12
12
|
|
13
13
|
class << self
|
14
14
|
def latest_master_project_from_ads(release_table_name, ads_client, segment_id)
|
15
|
+
sorted = get_master_project_list_from_ads(release_table_name, ads_client, segment_id)
|
16
|
+
sorted.last
|
17
|
+
end
|
18
|
+
|
19
|
+
def latest_master_project_from_nfs(domain_id, data_product_id, segment_id)
|
20
|
+
file_path = path_to_release_table_file(domain_id, data_product_id, segment_id)
|
21
|
+
sorted = get_master_project_list_from_nfs(domain_id, data_product_id, segment_id)
|
22
|
+
latest_master_project = sorted.last
|
23
|
+
|
24
|
+
version_info = latest_master_project ? "master_pid=#{latest_master_project[:master_project_id]} version=#{latest_master_project[:version]}" : ""
|
25
|
+
GoodData.gd_logger.info "Getting latest master project: file=#{file_path} domain=#{domain_id} data_product=#{data_product_id} segment=#{segment_id} #{version_info}"
|
26
|
+
latest_master_project
|
27
|
+
end
|
28
|
+
|
29
|
+
def get_master_project_list_from_ads(release_table_name, ads_client, segment_id)
|
15
30
|
replacements = {
|
16
31
|
table_name: release_table_name || DEFAULT_TABLE_NAME,
|
17
32
|
segment_id: segment_id
|
@@ -22,18 +37,27 @@ module GoodData
|
|
22
37
|
|
23
38
|
res = ads_client.execute_select(query)
|
24
39
|
sorted = res.sort_by { |row| row[:version] }
|
25
|
-
sorted
|
40
|
+
sorted
|
26
41
|
end
|
27
42
|
|
28
|
-
def
|
43
|
+
def delete_master_project_from_ads(release_table_name, ads_client, segment_id, removal_master_project_ids)
|
44
|
+
replacements = {
|
45
|
+
table_name: release_table_name || DEFAULT_TABLE_NAME,
|
46
|
+
segment_id: segment_id,
|
47
|
+
master_project_ids: removal_master_project_ids.map { |x| "'#{x}'" } * ', '
|
48
|
+
}
|
49
|
+
|
50
|
+
path = File.expand_path('../data/delete_from_lcm_release.sql.erb', __dir__)
|
51
|
+
query = GoodData::Helpers::ErbHelper.template_file(path, replacements)
|
52
|
+
|
53
|
+
ads_client.execute(query)
|
54
|
+
end
|
55
|
+
|
56
|
+
def get_master_project_list_from_nfs(domain_id, data_product_id, segment_id)
|
29
57
|
file_path = path_to_release_table_file(domain_id, data_product_id, segment_id)
|
30
58
|
data = GoodData::Helpers::Csv.read_as_hash(file_path)
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
version_info = latest_master_project ? "master_pid=#{latest_master_project[:master_project_id]} version=#{latest_master_project[:version]}" : ""
|
35
|
-
GoodData.gd_logger.info "Getting latest master project: file=#{file_path} domain=#{domain_id} data_product=#{data_product_id} segment=#{segment_id} #{version_info}"
|
36
|
-
latest_master_project
|
59
|
+
sorted = data.sort_by { |master| master[:version] }
|
60
|
+
sorted
|
37
61
|
end
|
38
62
|
|
39
63
|
def update_latest_master_to_nfs(domain_id, data_product_id, segment_id, master_pid, version)
|
@@ -46,6 +70,16 @@ module GoodData
|
|
46
70
|
)
|
47
71
|
end
|
48
72
|
|
73
|
+
def update_master_project_to_nfs(domain_id, data_product_id, segment_id, data)
|
74
|
+
file_path = path_to_release_table_file(domain_id, data_product_id, segment_id)
|
75
|
+
FileUtils.mkpath(file_path.split('/')[0...-1].join('/'))
|
76
|
+
CSV.open(file_path, 'w', write_headers: true, headers: data.first.keys) do |csv|
|
77
|
+
data.each do |r|
|
78
|
+
csv << r.values
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
49
83
|
def path_to_release_table_file(domain_id, data_prod_id, segment_id)
|
50
84
|
nsf_directory = ENV['RELEASE_TABLE_NFS_DIRECTORY'] || DEFAULT_NFS_DIRECTORY
|
51
85
|
[nsf_directory, domain_id, data_prod_id + '-' + segment_id + '.csv'].join('/')
|
data/lib/gooddata/lcm/lcm2.rb
CHANGED
@@ -96,6 +96,7 @@ module GoodData
|
|
96
96
|
CollectComputedAttributeMetrics,
|
97
97
|
ImportObjectCollections,
|
98
98
|
SynchronizeComputedAttributes,
|
99
|
+
SynchronizeDataSetMapping,
|
99
100
|
SynchronizeProcesses,
|
100
101
|
SynchronizeSchedules,
|
101
102
|
SynchronizeColorPalette,
|
@@ -122,9 +123,11 @@ module GoodData
|
|
122
123
|
AssociateClients,
|
123
124
|
RenameExistingClientProjects,
|
124
125
|
ProvisionClients,
|
126
|
+
UpdateMetricFormats,
|
125
127
|
EnsureTechnicalUsersDomain,
|
126
128
|
EnsureTechnicalUsersProject,
|
127
129
|
CollectDymanicScheduleParams,
|
130
|
+
SynchronizeDataSetMapping,
|
128
131
|
SynchronizeETLsInSegment
|
129
132
|
],
|
130
133
|
|
@@ -136,8 +139,10 @@ module GoodData
|
|
136
139
|
EnsureTechnicalUsersDomain,
|
137
140
|
EnsureTechnicalUsersProject,
|
138
141
|
SynchronizeLdm,
|
142
|
+
SynchronizeDataSetMapping,
|
139
143
|
MigrateGdcDateDimension,
|
140
144
|
SynchronizeClients,
|
145
|
+
UpdateMetricFormats,
|
141
146
|
SynchronizeComputedAttributes,
|
142
147
|
CollectDymanicScheduleParams,
|
143
148
|
SynchronizeETLsInSegment
|
@@ -34,7 +34,7 @@ module GoodData
|
|
34
34
|
c.create(DataSource, ds_data)
|
35
35
|
end
|
36
36
|
else
|
37
|
-
c.create(DataSource, c.get(
|
37
|
+
c.create(DataSource, c.get(DATA_SOURCES_URL + '/' + id))
|
38
38
|
end
|
39
39
|
end
|
40
40
|
|
@@ -177,6 +177,10 @@ module GoodData
|
|
177
177
|
@json['dataSource']['connectionInfo'][type]
|
178
178
|
end
|
179
179
|
|
180
|
+
def type
|
181
|
+
@json['dataSource']['connectionInfo'].first[0].upcase
|
182
|
+
end
|
183
|
+
|
180
184
|
private
|
181
185
|
|
182
186
|
def build_connection_info
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
#
|
4
|
+
# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
|
5
|
+
# This source code is licensed under the BSD-style license found in the
|
6
|
+
# LICENSE file in the root directory of this source tree.
|
7
|
+
|
8
|
+
module GoodData
|
9
|
+
class DatasetMapping
|
10
|
+
DATASET_MAPPING_GET_URI = '/gdc/dataload/projects/%<project_id>s/modelMapping/datasets'
|
11
|
+
DATASET_MAPPING_UPDATE_URI = '/gdc/dataload/projects/%<project_id>s/modelMapping/datasets/bulk/upsert'
|
12
|
+
|
13
|
+
class << self
|
14
|
+
def [](opts = { :client => GoodData.connection, :project => GoodData.project })
|
15
|
+
client, project = GoodData.get_client_and_project(opts)
|
16
|
+
get_uri = DATASET_MAPPING_GET_URI % { project_id: project.pid }
|
17
|
+
res = client.get(get_uri)
|
18
|
+
res
|
19
|
+
end
|
20
|
+
|
21
|
+
alias_method :get, :[]
|
22
|
+
end
|
23
|
+
|
24
|
+
def initialize(data)
|
25
|
+
@data = data
|
26
|
+
end
|
27
|
+
|
28
|
+
def save(opts)
|
29
|
+
client, project = GoodData.get_client_and_project(opts)
|
30
|
+
|
31
|
+
post_uri = DATASET_MAPPING_UPDATE_URI % { project_id: project.pid }
|
32
|
+
res = client.post(post_uri, @data, opts)
|
33
|
+
res
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -40,22 +40,20 @@ module GoodData
|
|
40
40
|
end
|
41
41
|
end
|
42
42
|
|
43
|
-
# Gets valid elements
|
43
|
+
# Gets valid elements of a label for a specific paging (:offset and :limit) or get validElements of a specific value (:filter).
|
44
|
+
# In the case filter a specific value, because the API /validElements only filter by partial match, we need to filter again at client side for exact match.
|
44
45
|
# @return [Array] Results
|
45
46
|
def get_valid_elements(*args)
|
46
|
-
|
47
|
-
|
48
|
-
# so we do a preliminary first request to check and then increase the limit if needed
|
49
|
-
if results['validElements']['paging']['total'].to_i != params[:limit]
|
47
|
+
if args && !args.empty? && args.first[:filter]
|
48
|
+
params = args.first
|
50
49
|
params[:limit] = 100_000
|
51
50
|
results, = valid_elements params
|
52
|
-
|
53
|
-
|
54
|
-
i['element']['title'] != params[:filter]
|
55
|
-
end
|
51
|
+
results['validElements']['items'] = results['validElements']['items'].select do |i|
|
52
|
+
i['element']['title'] == params[:filter]
|
56
53
|
end
|
54
|
+
else
|
55
|
+
results, = valid_elements(*args)
|
57
56
|
end
|
58
|
-
|
59
57
|
results
|
60
58
|
end
|
61
59
|
|
@@ -74,24 +72,25 @@ module GoodData
|
|
74
72
|
# @option options [Number] :limit limits the number of values to certain number. Default is 100
|
75
73
|
# @return [Array]
|
76
74
|
def values(options = {})
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
end
|
91
|
-
break if elements['items'].count < page_limit
|
92
|
-
offset += page_limit
|
75
|
+
all_values = []
|
76
|
+
offset = options[:offset] || 0
|
77
|
+
page_limit = options[:limit] || 100
|
78
|
+
loop do
|
79
|
+
results = get_valid_elements(limit: page_limit, offset: offset)
|
80
|
+
|
81
|
+
elements = results['validElements']
|
82
|
+
elements['items'].map do |el|
|
83
|
+
v = el['element']
|
84
|
+
all_values << {
|
85
|
+
:value => v['title'],
|
86
|
+
:uri => v['uri']
|
87
|
+
}
|
93
88
|
end
|
89
|
+
break if elements['items'].count < page_limit
|
90
|
+
|
91
|
+
offset += page_limit
|
94
92
|
end
|
93
|
+
all_values
|
95
94
|
end
|
96
95
|
|
97
96
|
def values_count
|
@@ -136,7 +135,7 @@ module GoodData
|
|
136
135
|
if status_url
|
137
136
|
results = client.poll_on_response(status_url) do |body|
|
138
137
|
status = body['taskState'] && body['taskState']['status']
|
139
|
-
status == 'RUNNING' || status == 'PREPARED'
|
138
|
+
status == 'RUNNING' || status == 'PREPARED' || body['uri']
|
140
139
|
end
|
141
140
|
end
|
142
141
|
|
@@ -30,6 +30,7 @@ require_relative 'process'
|
|
30
30
|
require_relative 'project_log_formatter'
|
31
31
|
require_relative 'project_role'
|
32
32
|
require_relative 'blueprint/blueprint'
|
33
|
+
require_relative 'dataset_mapping'
|
33
34
|
|
34
35
|
require_relative 'metadata/scheduled_mail'
|
35
36
|
require_relative 'metadata/scheduled_mail/dashboard_attachment'
|
@@ -255,6 +256,22 @@ module GoodData
|
|
255
256
|
transfer_schedules(from_project, to_project)
|
256
257
|
end
|
257
258
|
|
259
|
+
def get_dataset_mapping(from_project)
|
260
|
+
GoodData::DatasetMapping.get(:client => from_project.client, :project => from_project)
|
261
|
+
end
|
262
|
+
|
263
|
+
def update_dataset_mapping(model_mapping_json, to_project)
|
264
|
+
dataset_mapping = GoodData::DatasetMapping.new(model_mapping_json)
|
265
|
+
res = dataset_mapping.save(:client => to_project.client, :project => to_project)
|
266
|
+
status = res&.dig('datasetMappings', 'items').nil? ? "Failed" : "OK"
|
267
|
+
count = "OK".eql?(status) ? res['datasetMappings']['items'].length : 0
|
268
|
+
{
|
269
|
+
to: to_project.pid,
|
270
|
+
count: count,
|
271
|
+
status: status
|
272
|
+
}
|
273
|
+
end
|
274
|
+
|
258
275
|
# @param from_project The source project
|
259
276
|
# @param to_project The target project
|
260
277
|
# @param options Optional parameters
|
@@ -337,20 +354,16 @@ module GoodData
|
|
337
354
|
def get_data_source_alias(data_source_id, client, aliases)
|
338
355
|
unless aliases[data_source_id]
|
339
356
|
data_source = GoodData::DataSource.from_id(data_source_id, client: client)
|
340
|
-
if data_source&.
|
357
|
+
if data_source&.alias
|
341
358
|
aliases[data_source_id] = {
|
342
|
-
:type =>
|
343
|
-
:alias => data_source
|
359
|
+
:type => data_source.type,
|
360
|
+
:alias => data_source.alias
|
344
361
|
}
|
345
362
|
end
|
346
363
|
end
|
347
364
|
aliases[data_source_id]
|
348
365
|
end
|
349
366
|
|
350
|
-
def get_data_source_type(data_source_data)
|
351
|
-
data_source_data&.dig('dataSource', 'connectionInfo') ? data_source_data['dataSource']['connectionInfo'].first[0].upcase : ""
|
352
|
-
end
|
353
|
-
|
354
367
|
def replace_process_data_source_ids(process_data, client, aliases)
|
355
368
|
component = process_data.dig(:process, :component)
|
356
369
|
if component&.dig(:configLocation, :dataSourceConfig)
|
@@ -460,7 +473,9 @@ module GoodData
|
|
460
473
|
local_stuff = local_schedules.map do |s|
|
461
474
|
v = s.to_hash
|
462
475
|
after_schedule = local_schedules.find { |s2| s.trigger_id == s2.obj_id }
|
463
|
-
|
476
|
+
after_process_schedule = from_project_processes.find { |p| after_schedule && p.obj_id == after_schedule.process_id }
|
477
|
+
v[:after] = s.trigger_id && after_process_schedule && after_schedule && after_schedule.name
|
478
|
+
v[:trigger_execution_status] = s.trigger_execution_status
|
464
479
|
v[:remote_schedule] = s
|
465
480
|
v[:params] = v[:params].except("EXECUTABLE", "PROCESS_ID")
|
466
481
|
v.compact
|
@@ -529,6 +544,7 @@ module GoodData
|
|
529
544
|
schedule.params = (schedule_spec[:params] || {})
|
530
545
|
schedule.cron = schedule_spec[:cron] if schedule_spec[:cron]
|
531
546
|
schedule.after = schedule_cache[schedule_spec[:after]] if schedule_spec[:after]
|
547
|
+
schedule.trigger_execution_status = schedule_cache[schedule_spec[:trigger_execution_status]] if schedule_spec[:after]
|
532
548
|
schedule.hidden_params = schedule_spec[:hidden_params] || {}
|
533
549
|
if process_spec.type != :dataload
|
534
550
|
schedule.executable = schedule_spec[:executable] || (process_spec.type == :ruby ? 'main.rb' : 'main.grf')
|
@@ -589,7 +605,8 @@ module GoodData
|
|
589
605
|
hidden_params: schedule_spec[:hidden_params],
|
590
606
|
name: schedule_spec[:name],
|
591
607
|
reschedule: schedule_spec[:reschedule],
|
592
|
-
state: schedule_spec[:state]
|
608
|
+
state: schedule_spec[:state],
|
609
|
+
trigger_execution_status: schedule_spec[:trigger_execution_status]
|
593
610
|
}
|
594
611
|
end
|
595
612
|
end
|
@@ -2022,6 +2039,14 @@ module GoodData
|
|
2022
2039
|
GoodData::Project.transfer_etl(client, self, target)
|
2023
2040
|
end
|
2024
2041
|
|
2042
|
+
def dataset_mapping
|
2043
|
+
GoodData::Project.get_dataset_mapping(self)
|
2044
|
+
end
|
2045
|
+
|
2046
|
+
def update_dataset_mapping(model_mapping_json)
|
2047
|
+
GoodData::Project.update_dataset_mapping(model_mapping_json, self)
|
2048
|
+
end
|
2049
|
+
|
2025
2050
|
def transfer_processes(target)
|
2026
2051
|
GoodData::Project.transfer_processes(self, target)
|
2027
2052
|
end
|
@@ -101,6 +101,7 @@ module GoodData
|
|
101
101
|
|
102
102
|
schedule.name = options[:name]
|
103
103
|
schedule.set_trigger(trigger)
|
104
|
+
schedule.trigger_execution_status = options[:trigger_execution_status]
|
104
105
|
schedule.params = default_opts[:params].merge(options[:params] || {})
|
105
106
|
schedule.hidden_params = options[:hidden_params] || {}
|
106
107
|
schedule.timezone = options[:timezone] || default_opts[:timezone]
|
@@ -468,6 +469,7 @@ module GoodData
|
|
468
469
|
hidden_params: hidden_params,
|
469
470
|
cron: cron,
|
470
471
|
trigger_id: trigger_id,
|
472
|
+
trigger_execution_status: trigger_execution_status,
|
471
473
|
timezone: timezone,
|
472
474
|
uri: uri,
|
473
475
|
reschedule: reschedule,
|
@@ -486,6 +488,16 @@ module GoodData
|
|
486
488
|
self
|
487
489
|
end
|
488
490
|
|
491
|
+
def trigger_execution_status
|
492
|
+
json['schedule']['triggerExecutionStatus']
|
493
|
+
end
|
494
|
+
|
495
|
+
def trigger_execution_status=(trigger_execution_status)
|
496
|
+
json['schedule']['triggerExecutionStatus'] = trigger_execution_status
|
497
|
+
@dirty = true
|
498
|
+
self # rubocop:disable Lint/Void
|
499
|
+
end
|
500
|
+
|
489
501
|
def name
|
490
502
|
json['schedule']['name']
|
491
503
|
end
|
@@ -530,7 +542,7 @@ module GoodData
|
|
530
542
|
'hiddenParams' => GoodData::Helpers.encode_hidden_params(hidden_params)
|
531
543
|
}
|
532
544
|
}
|
533
|
-
|
545
|
+
res['schedule']['triggerExecutionStatus'] = trigger_execution_status if trigger_execution_status
|
534
546
|
res['schedule']['reschedule'] = reschedule if reschedule
|
535
547
|
|
536
548
|
res
|