gooddata 0.6.51 → 0.6.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -1
- data/CHANGELOG.md +13 -1
- data/CONTRIBUTING.md +25 -0
- data/PULL_REQUEST_TEMPLATE.md +5 -0
- data/README.md +7 -4
- data/gooddata.gemspec +2 -3
- data/lib/gooddata.rb +1 -0
- data/lib/gooddata/bricks/base_downloader.rb +6 -6
- data/lib/gooddata/bricks/middleware/aws_middleware.rb +15 -5
- data/lib/gooddata/bricks/middleware/dwh_middleware.rb +15 -3
- data/lib/gooddata/bricks/middleware/gooddata_middleware.rb +13 -4
- data/lib/gooddata/bricks/middleware/logger_middleware.rb +3 -0
- data/lib/gooddata/exceptions/no_project_error.rb +5 -1
- data/lib/gooddata/goodzilla/goodzilla.rb +7 -6
- data/lib/gooddata/helpers/data_helper.rb +4 -4
- data/lib/gooddata/helpers/global_helpers_params.rb +61 -39
- data/lib/gooddata/lcm/actions/apply_custom_maql.rb +9 -0
- data/lib/gooddata/lcm/actions/associate_clients.rb +23 -4
- data/lib/gooddata/lcm/actions/collect_attrs.rb +56 -0
- data/lib/gooddata/lcm/actions/collect_ca_metrics.rb +53 -0
- data/lib/gooddata/lcm/actions/collect_clients.rb +25 -3
- data/lib/gooddata/lcm/actions/collect_meta.rb +83 -0
- data/lib/gooddata/lcm/actions/collect_segment_clients.rb +12 -4
- data/lib/gooddata/lcm/actions/collect_segments.rb +4 -4
- data/lib/gooddata/lcm/actions/collect_tagged_objects.rb +74 -0
- data/lib/gooddata/lcm/actions/create_segment_masters.rb +16 -30
- data/lib/gooddata/lcm/actions/ensure_release_table.rb +0 -3
- data/lib/gooddata/lcm/actions/ensure_segments.rb +1 -4
- data/lib/gooddata/lcm/actions/ensure_technical_users_domain.rb +5 -5
- data/lib/gooddata/lcm/actions/ensure_technical_users_project.rb +8 -5
- data/lib/gooddata/lcm/actions/hello_world.rb +0 -3
- data/lib/gooddata/lcm/actions/import_object_collections.rb +60 -0
- data/lib/gooddata/lcm/actions/print_actions.rb +0 -3
- data/lib/gooddata/lcm/actions/print_modes.rb +0 -3
- data/lib/gooddata/lcm/actions/print_types.rb +1 -4
- data/lib/gooddata/lcm/actions/provision_clients.rb +5 -5
- data/lib/gooddata/lcm/actions/purge_clients.rb +4 -10
- data/lib/gooddata/lcm/actions/segments_filter.rb +0 -6
- data/lib/gooddata/lcm/actions/synchronize_attribute_drillpaths.rb +8 -4
- data/lib/gooddata/lcm/actions/synchronize_cas.rb +61 -0
- data/lib/gooddata/lcm/actions/synchronize_clients.rb +9 -3
- data/lib/gooddata/lcm/actions/synchronize_color_palette.rb +13 -5
- data/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb +71 -17
- data/lib/gooddata/lcm/actions/synchronize_label_types.rb +8 -5
- data/lib/gooddata/lcm/actions/synchronize_ldm.rb +17 -8
- data/lib/gooddata/lcm/actions/synchronize_meta.rb +0 -3
- data/lib/gooddata/lcm/actions/synchronize_new_segments.rb +9 -4
- data/lib/gooddata/lcm/actions/synchronize_processes.rb +9 -5
- data/lib/gooddata/lcm/actions/synchronize_schedules.rb +15 -5
- data/lib/gooddata/lcm/actions/synchronize_tag_objects.rb +61 -0
- data/lib/gooddata/lcm/actions/update_release_table.rb +0 -3
- data/lib/gooddata/lcm/helpers/tags_helper.rb +35 -0
- data/lib/gooddata/lcm/lcm.rb +22 -4
- data/lib/gooddata/lcm/lcm2.rb +66 -13
- data/lib/gooddata/lcm/types/complex/update_preference.rb +1 -1
- data/lib/gooddata/mixins/md_finders.rb +4 -2
- data/lib/gooddata/mixins/md_object_indexer.rb +13 -3
- data/lib/gooddata/mixins/md_object_query.rb +8 -2
- data/lib/gooddata/models/blueprint/date_dimension.rb +6 -0
- data/lib/gooddata/models/blueprint/project_blueprint.rb +41 -11
- data/lib/gooddata/models/blueprint/project_builder.rb +20 -0
- data/lib/gooddata/models/blueprint/to_wire.rb +7 -0
- data/lib/gooddata/models/client.rb +6 -0
- data/lib/gooddata/models/domain.rb +6 -6
- data/lib/gooddata/models/from_wire.rb +5 -1
- data/lib/gooddata/models/metadata.rb +55 -9
- data/lib/gooddata/models/metadata/attribute.rb +19 -4
- data/lib/gooddata/models/metadata/dashboard.rb +15 -3
- data/lib/gooddata/models/metadata/dataset.rb +5 -2
- data/lib/gooddata/models/metadata/dimension.rb +4 -1
- data/lib/gooddata/models/metadata/fact.rb +9 -2
- data/lib/gooddata/models/metadata/folder.rb +4 -1
- data/lib/gooddata/models/metadata/metric.rb +11 -3
- data/lib/gooddata/models/metadata/report.rb +7 -2
- data/lib/gooddata/models/metadata/report_definition.rb +11 -4
- data/lib/gooddata/models/metadata/scheduled_mail.rb +4 -1
- data/lib/gooddata/models/metadata/variable.rb +7 -2
- data/lib/gooddata/models/model.rb +14 -3
- data/lib/gooddata/models/process.rb +10 -9
- data/lib/gooddata/models/project.rb +134 -36
- data/lib/gooddata/models/project_creator.rb +43 -20
- data/lib/gooddata/models/report_data_result.rb +6 -2
- data/lib/gooddata/models/schedule.rb +6 -3
- data/lib/gooddata/models/subscription.rb +8 -1
- data/lib/gooddata/models/user_filters/user_filter.rb +1 -0
- data/lib/gooddata/models/user_filters/user_filter_builder.rb +18 -4
- data/lib/gooddata/models/user_filters/variable_user_filter.rb +3 -1
- data/lib/gooddata/rest/client.rb +4 -6
- data/lib/gooddata/rest/connection.rb +10 -2
- data/lib/gooddata/version.rb +1 -1
- data/spec/data/blueprints/test_blueprint.json +1 -0
- data/spec/data/wire_models/test_blueprint.json +3 -0
- data/spec/data/workspace_table.csv +3 -0
- data/spec/environment/development.rb +4 -1
- data/spec/environment/environment.rb +1 -1
- data/spec/environment/staging.rb +5 -1
- data/spec/environment/testing.rb +5 -2
- data/spec/integration/blueprint_with_ca_spec.rb +56 -0
- data/spec/integration/clients_spec.rb +21 -0
- data/spec/integration/command_datawarehouse_spec.rb +7 -1
- data/spec/integration/create_from_template_spec.rb +9 -3
- data/spec/integration/project_spec.rb +7 -0
- data/spec/integration/segments_spec.rb +0 -53
- data/spec/integration/subscription_spec.rb +29 -4
- data/spec/integration/urn_date_dim_spec.rb +53 -0
- data/spec/integration/user_filters_spec.rb +6 -0
- data/spec/integration/variables_spec.rb +1 -2
- data/spec/spec_helper.rb +5 -30
- data/spec/unit/actions/collect_clients_spec.rb +38 -0
- data/spec/unit/actions/collect_meta_spec.rb +87 -0
- data/spec/unit/actions/collect_segment_clients_spec.rb +40 -0
- data/spec/unit/actions/collect_tagged_objects_spec.rb +110 -0
- data/spec/unit/actions/synchronize_etls_in_segment_spec.rb +51 -0
- data/spec/unit/bricks/middleware/aws_middelware_spec.rb +55 -1
- data/spec/unit/bricks/middleware/logger_middleware_spec.rb +15 -0
- data/spec/unit/helpers/data_helper_spec.rb +3 -5
- data/spec/unit/helpers/global_helpers_spec.rb +29 -0
- data/spec/unit/helpers_spec.rb +18 -1
- data/spec/unit/models/blueprint/project_blueprint_spec.rb +1 -23
- data/spec/unit/models/domain_spec.rb +19 -0
- data/spec/unit/models/metadata_spec.rb +34 -0
- data/spec/unit/models/schedule_spec.rb +31 -0
- data/spec/unit/models/to_manifest_spec.rb +10 -2
- data/spec/unit/models/unit_project_spec.rb +6 -1
- data/spec/unit/rest/polling_spec.rb +13 -1
- metadata +49 -31
|
@@ -48,7 +48,10 @@ module GoodData
|
|
|
48
48
|
# Method intended to get all objects of that type in a specified project
|
|
49
49
|
#
|
|
50
50
|
# @param options [Hash] the options hash
|
|
51
|
-
# @option options [Boolean] :full if passed true the subclass can decide
|
|
51
|
+
# @option options [Boolean] :full if passed true the subclass can decide
|
|
52
|
+
# to pull in full objects. This is desirable from the usability POV
|
|
53
|
+
# but unfortunately has negative impact on performance so it is not the
|
|
54
|
+
# default
|
|
52
55
|
# @return [Array<GoodData::MdObject> | Array<Hash>] Return the appropriate metadata objects or their representation
|
|
53
56
|
def all(options = { :client => GoodData.connection, :project => GoodData.project })
|
|
54
57
|
query('scheduledMail', ScheduledMail, options)
|
|
@@ -14,7 +14,10 @@ module GoodData
|
|
|
14
14
|
# Method intended to get all objects of that type in a specified project
|
|
15
15
|
#
|
|
16
16
|
# @param options [Hash] the options hash
|
|
17
|
-
# @option options [Boolean] :full if passed true the subclass can decide
|
|
17
|
+
# @option options [Boolean] :full if passed true the subclass can decide
|
|
18
|
+
# to pull in full objects. This is desirable from the usability
|
|
19
|
+
# POV but unfortunately has negative impact on performance so it
|
|
20
|
+
# is not the default
|
|
18
21
|
# @return [Array<GoodData::MdObject> | Array<Hash>] Return the appropriate metadata objects or their representation
|
|
19
22
|
def all(options = { :client => GoodData.connection, :project => GoodData.project })
|
|
20
23
|
query('prompt', Variable, options)
|
|
@@ -67,7 +70,9 @@ module GoodData
|
|
|
67
70
|
values.select { |x| x.level == :user }
|
|
68
71
|
end
|
|
69
72
|
|
|
70
|
-
# Method used for replacing values in their state according to mapping.
|
|
73
|
+
# Method used for replacing values in their state according to mapping.
|
|
74
|
+
# Can be used to replace any values but it is typically used to replace
|
|
75
|
+
# the URIs. Returns a new object of the same type.
|
|
71
76
|
#
|
|
72
77
|
# @param [Array<Array>]Mapping specifying what should be exchanged for what. As mapping should be used output of GoodData::Helpers.prepare_mapping.
|
|
73
78
|
# @return [GoodData::Variable]
|
|
@@ -267,13 +267,24 @@ module GoodData
|
|
|
267
267
|
d = GoodData::Helpers.deep_dup(a_schema_blueprint)
|
|
268
268
|
d[:columns] = d[:columns] + b_schema_blueprint[:columns]
|
|
269
269
|
d[:columns].uniq!
|
|
270
|
-
columns_that_failed_to_merge = d[:columns]
|
|
270
|
+
columns_that_failed_to_merge = d[:columns]
|
|
271
|
+
.group_by { |x| [:reference, :date].include?(x[:type]) ? x[:dataset] : x[:id] }
|
|
272
|
+
.map { |k, v| [k, v.count, v] }.select { |x| x[1] > 1 }
|
|
271
273
|
unless columns_that_failed_to_merge.empty?
|
|
272
274
|
columns_that_failed_to_merge.each do |error|
|
|
273
|
-
|
|
275
|
+
message = "Columns #{error[0]} failed to merge. There are " \
|
|
276
|
+
"#{error[1]} conflicting columns. When merging columns " \
|
|
277
|
+
"with the same name they have to be identical."
|
|
278
|
+
GoodData.logger.error message
|
|
274
279
|
GoodData.logger.error error[2]
|
|
275
280
|
end
|
|
276
|
-
|
|
281
|
+
unless columns_that_failed_to_merge.empty?
|
|
282
|
+
fail "Columns #{columns_that_failed_to_merge.first} failed to " \
|
|
283
|
+
"merge. There are #{columns_that_failed_to_merge[1]} " \
|
|
284
|
+
"conflicting columns. #{columns_that_failed_to_merge[2]} " \
|
|
285
|
+
"When merging columns with the same name they have to be " \
|
|
286
|
+
"identical."
|
|
287
|
+
end
|
|
277
288
|
end
|
|
278
289
|
d
|
|
279
290
|
end
|
|
@@ -123,6 +123,8 @@ module GoodData
|
|
|
123
123
|
client.put("/gdc/projects/#{project.pid}/dataload/processes/#{process_id}", data)
|
|
124
124
|
end
|
|
125
125
|
|
|
126
|
+
File.delete(deployed_path) if File.exist?(deployed_path)
|
|
127
|
+
|
|
126
128
|
process = client.create(Process, res, project: project)
|
|
127
129
|
puts HighLine.color("Deploy DONE #{path}", HighLine::GREEN) if verbose
|
|
128
130
|
process
|
|
@@ -207,16 +209,15 @@ module GoodData
|
|
|
207
209
|
|
|
208
210
|
def with_zip(opts = {})
|
|
209
211
|
client = opts[:client]
|
|
210
|
-
Tempfile.
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
end
|
|
217
|
-
client.upload_to_user_webdav(temp.path, opts)
|
|
218
|
-
temp.path
|
|
212
|
+
temp = Tempfile.new(['deploy-graph-archive', '.zip'])
|
|
213
|
+
zip_filename = temp.path
|
|
214
|
+
|
|
215
|
+
temp.close!
|
|
216
|
+
Zip::File.open(zip_filename, Zip::File::CREATE) do |zipfile|
|
|
217
|
+
yield zipfile
|
|
219
218
|
end
|
|
219
|
+
client.upload_to_user_webdav(zip_filename, opts)
|
|
220
|
+
zip_filename
|
|
220
221
|
end
|
|
221
222
|
|
|
222
223
|
def zip_and_upload(path, files_to_exclude, opts = {})
|
|
@@ -140,7 +140,9 @@ module GoodData
|
|
|
140
140
|
|
|
141
141
|
project = create_object(opts)
|
|
142
142
|
project.save
|
|
143
|
-
# until it is enabled or deleted, recur. This should still end if there
|
|
143
|
+
# until it is enabled or deleted, recur. This should still end if there
|
|
144
|
+
# is a exception thrown out from RESTClient. This sometimes happens from
|
|
145
|
+
# WebApp when request is too long
|
|
144
146
|
while project.state.to_s != 'enabled'
|
|
145
147
|
if project.deleted?
|
|
146
148
|
# if project is switched to deleted state, fail. This is usually problem of creating a template which is invalid.
|
|
@@ -220,7 +222,10 @@ module GoodData
|
|
|
220
222
|
# Clones project along with etl and schedules.
|
|
221
223
|
#
|
|
222
224
|
# @param client [GoodData::Rest::Client] GoodData client to be used for connection
|
|
223
|
-
# @param from_project [GoodData::Project | GoodData::Segment | GoodData:Client | String]
|
|
225
|
+
# @param from_project [GoodData::Project | GoodData::Segment | GoodData:Client | String]
|
|
226
|
+
# Object to be cloned from. Can be either segment in which case we
|
|
227
|
+
# take the master, client in which case we take its project, string
|
|
228
|
+
# in which case we treat is as an project object or directly project
|
|
224
229
|
# @param to_project [GoodData::Project | GoodData::Segment | GoodData:Client | String]
|
|
225
230
|
def transfer_etl(client, from_project, to_project)
|
|
226
231
|
from_project = case from_project
|
|
@@ -304,7 +309,10 @@ module GoodData
|
|
|
304
309
|
# Clones project along with etl and schedules.
|
|
305
310
|
#
|
|
306
311
|
# @param client [GoodData::Rest::Client] GoodData client to be used for connection
|
|
307
|
-
# @param from_project [GoodData::Project | GoodData::Segment | GoodData:Client | String]
|
|
312
|
+
# @param from_project [GoodData::Project | GoodData::Segment | GoodData:Client | String]
|
|
313
|
+
# Object to be cloned from. Can be either segment in which case we take
|
|
314
|
+
# the master, client in which case we take its project, string in which
|
|
315
|
+
# case we treat is as an project object or directly project.
|
|
308
316
|
def transfer_schedules(from_project, to_project)
|
|
309
317
|
cache = to_project
|
|
310
318
|
.processes.sort_by(&:name)
|
|
@@ -335,7 +343,7 @@ module GoodData
|
|
|
335
343
|
v.compact
|
|
336
344
|
end
|
|
337
345
|
|
|
338
|
-
diff = GoodData::Helpers.diff(remote_stuff, local_stuff, key: :name, fields: [:name, :cron, :after, :params, :hidden_params, :reschedule])
|
|
346
|
+
diff = GoodData::Helpers.diff(remote_stuff, local_stuff, key: :name, fields: [:name, :cron, :after, :params, :hidden_params, :reschedule, :state])
|
|
339
347
|
stack = diff[:added].map do |x|
|
|
340
348
|
[:added, x]
|
|
341
349
|
end
|
|
@@ -369,7 +377,7 @@ module GoodData
|
|
|
369
377
|
if process_spec.type != :dataload
|
|
370
378
|
executable = schedule_spec[:executable] || (process_spec.type == :ruby ? 'main.rb' : 'main.grf')
|
|
371
379
|
end
|
|
372
|
-
params = schedule_parameters(
|
|
380
|
+
params = schedule_parameters(schedule_spec)
|
|
373
381
|
created_schedule = remote_process.create_schedule(schedule_spec[:cron] || schedule_cache[schedule_spec[:after]], executable, params)
|
|
374
382
|
schedule_cache[created_schedule.name] = created_schedule
|
|
375
383
|
|
|
@@ -403,6 +411,7 @@ module GoodData
|
|
|
403
411
|
|
|
404
412
|
schedule.reschedule = schedule_spec[:reschedule]
|
|
405
413
|
schedule.name = schedule_spec[:name]
|
|
414
|
+
schedule.state = schedule_spec[:state]
|
|
406
415
|
schedule.save
|
|
407
416
|
schedule_cache[schedule.name] = schedule
|
|
408
417
|
|
|
@@ -449,12 +458,13 @@ module GoodData
|
|
|
449
458
|
|
|
450
459
|
private
|
|
451
460
|
|
|
452
|
-
def schedule_parameters(
|
|
461
|
+
def schedule_parameters(schedule_spec)
|
|
453
462
|
{
|
|
454
|
-
params: schedule_spec[:params]
|
|
463
|
+
params: schedule_spec[:params],
|
|
455
464
|
hidden_params: schedule_spec[:hidden_params],
|
|
456
465
|
name: schedule_spec[:name],
|
|
457
|
-
reschedule: schedule_spec[:reschedule]
|
|
466
|
+
reschedule: schedule_spec[:reschedule],
|
|
467
|
+
state: schedule_spec[:state]
|
|
458
468
|
}
|
|
459
469
|
end
|
|
460
470
|
end
|
|
@@ -535,6 +545,15 @@ module GoodData
|
|
|
535
545
|
GoodData::Attribute[id, project: self, client: client]
|
|
536
546
|
end
|
|
537
547
|
|
|
548
|
+
def computed_attributes(id = :all)
|
|
549
|
+
attrs = attributes(id)
|
|
550
|
+
if attrs.is_a?(GoodData::Attribute)
|
|
551
|
+
attrs.computed_attribute? ? attrs : nil
|
|
552
|
+
else
|
|
553
|
+
attrs.select(&:computed_attribute?)
|
|
554
|
+
end
|
|
555
|
+
end
|
|
556
|
+
|
|
538
557
|
def attribute_by_identifier(identifier)
|
|
539
558
|
GoodData::Attribute.find_first_by_identifier(identifier, project: self, client: client)
|
|
540
559
|
end
|
|
@@ -555,10 +574,10 @@ module GoodData
|
|
|
555
574
|
#
|
|
556
575
|
# @return [GoodData::ProjectRole] Project role if found
|
|
557
576
|
def blueprint(options = {})
|
|
558
|
-
result = client.get("/gdc/projects/#{pid}/model/view", params: { includeDeprecated: true, includeGrain: true })
|
|
577
|
+
result = client.get("/gdc/projects/#{pid}/model/view", params: { includeDeprecated: true, includeGrain: true, includeCA: true })
|
|
559
578
|
polling_url = result['asyncTask']['link']['poll']
|
|
560
579
|
model = client.poll_on_code(polling_url, options)
|
|
561
|
-
bp = GoodData::Model::FromWire.from_wire(model)
|
|
580
|
+
bp = GoodData::Model::FromWire.from_wire(model, { include_ca: true }.merge(options))
|
|
562
581
|
bp.title = title
|
|
563
582
|
bp
|
|
564
583
|
end
|
|
@@ -1119,10 +1138,18 @@ module GoodData
|
|
|
1119
1138
|
def objects_export(objs, options = {})
|
|
1120
1139
|
fail 'Nothing to migrate. You have to pass list of objects, ids or uris that you would like to migrate' if objs.nil?
|
|
1121
1140
|
objs = Array(objs)
|
|
1122
|
-
|
|
1141
|
+
if objs.empty?
|
|
1142
|
+
GoodData.logger.warn 'Nothing to migrate.'
|
|
1143
|
+
return
|
|
1144
|
+
end
|
|
1123
1145
|
|
|
1124
1146
|
objs = objs.pmap { |obj| [obj, objects(obj)] }
|
|
1125
|
-
|
|
1147
|
+
if objs.any? { |_, obj| obj.nil? }
|
|
1148
|
+
object = objs.select { |_, obj| obj.nil? }.map { |o, _| o }.join(', ')
|
|
1149
|
+
error_message = "Exporting objects failed with messages. " \
|
|
1150
|
+
"Object #{object} could not be found."
|
|
1151
|
+
fail ObjectsExportError, error_message
|
|
1152
|
+
end
|
|
1126
1153
|
export_payload = {
|
|
1127
1154
|
:partialMDExport => {
|
|
1128
1155
|
:uris => objs.map { |_, obj| obj.uri },
|
|
@@ -1158,7 +1185,8 @@ module GoodData
|
|
|
1158
1185
|
:partialMDImport => {
|
|
1159
1186
|
:token => token,
|
|
1160
1187
|
:overwriteNewer => '1',
|
|
1161
|
-
:updateLDMObjects => '0'
|
|
1188
|
+
:updateLDMObjects => '0',
|
|
1189
|
+
:importAttributeProperties => '1'
|
|
1162
1190
|
}
|
|
1163
1191
|
}
|
|
1164
1192
|
|
|
@@ -1183,11 +1211,15 @@ module GoodData
|
|
|
1183
1211
|
# @option options [GoodData::Project | String | Array<String> | Array<GoodData::Project>] :project Project(s) to migrate to
|
|
1184
1212
|
# @option options [Number] :batch_size Number of projects that are migrated at the same time. Default is 10
|
|
1185
1213
|
#
|
|
1186
|
-
# @return [Boolean | Array<Hash>] Return either true or throws exception
|
|
1214
|
+
# @return [Boolean | Array<Hash>] Return either true or throws exception
|
|
1215
|
+
# if you passed only one project. If you provided an array returns list
|
|
1216
|
+
# of hashes signifying sucees or failure. Take note that in case of list
|
|
1217
|
+
# of projects it does not throw exception.
|
|
1187
1218
|
def partial_md_export(objects, options = {})
|
|
1188
1219
|
projects = options[:project]
|
|
1189
1220
|
batch_size = options[:batch_size] || 10
|
|
1190
1221
|
token = objects_export(objects)
|
|
1222
|
+
return if token.nil?
|
|
1191
1223
|
|
|
1192
1224
|
if projects.is_a?(Array)
|
|
1193
1225
|
projects.each_slice(batch_size).flat_map do |batch|
|
|
@@ -1269,7 +1301,9 @@ module GoodData
|
|
|
1269
1301
|
self
|
|
1270
1302
|
end
|
|
1271
1303
|
|
|
1272
|
-
# Method used for walking through objects in project and trying to
|
|
1304
|
+
# Method used for walking through objects in project and trying to
|
|
1305
|
+
# replace all occurences of some object for another object. This is
|
|
1306
|
+
# typically used as a means for exchanging Date dimensions.
|
|
1273
1307
|
#
|
|
1274
1308
|
# @param mapping [Array<Array>] Mapping specifying what should be exchanged for what. As mapping should be used output of GoodData::Helpers.prepare_mapping.
|
|
1275
1309
|
def replace_from_mapping(mapping, opts = {})
|
|
@@ -1328,12 +1362,50 @@ module GoodData
|
|
|
1328
1362
|
end
|
|
1329
1363
|
end
|
|
1330
1364
|
|
|
1365
|
+
GoodData.logger.info 'Replacing dashboard saved views'
|
|
1366
|
+
contexts = mapping.map { |a, _| a }.pmapcat { |a| a.usedby('executionContext') }.map { |a| GoodData::MdObject[a['link'], client: client, project: self] }
|
|
1367
|
+
puts "Found #{contexts.count} dashboard saved views"
|
|
1368
|
+
contexts.peach do |item|
|
|
1369
|
+
new_item = GoodData::MdObject.replace_quoted(item, mapping)
|
|
1370
|
+
if new_item.json != item.json
|
|
1371
|
+
if dry_run
|
|
1372
|
+
GoodData.logger.info "Would save #{new_item.uri}. Running in dry run mode"
|
|
1373
|
+
else
|
|
1374
|
+
GoodData.logger.info "Saving #{new_item.uri}"
|
|
1375
|
+
new_item.save
|
|
1376
|
+
end
|
|
1377
|
+
else
|
|
1378
|
+
GoodData.logger.info "Ignore #{item.uri}"
|
|
1379
|
+
end
|
|
1380
|
+
end
|
|
1381
|
+
|
|
1331
1382
|
GoodData.logger.info 'Replacing variable values'
|
|
1332
1383
|
variables.each do |var|
|
|
1333
1384
|
var.values.peach do |val|
|
|
1334
1385
|
val.replace(mapping).save unless dry_run
|
|
1335
1386
|
end
|
|
1336
1387
|
end
|
|
1388
|
+
|
|
1389
|
+
{
|
|
1390
|
+
visualizations: MdObject.query('visualization', MdObject, client: client, project: self),
|
|
1391
|
+
visualization_widgets: MdObject.query('visualizationWidget', MdObject, client: client, project: self),
|
|
1392
|
+
kpis: MdObject.query('kpi', MdObject, client: client, project: self)
|
|
1393
|
+
}.each do |key, collection|
|
|
1394
|
+
GoodData.logger.info "Replacing #{key}"
|
|
1395
|
+
collection.each do |item|
|
|
1396
|
+
new_item = MdObject.replace_quoted(item, mapping)
|
|
1397
|
+
if new_item.json != item.json
|
|
1398
|
+
if dry_run
|
|
1399
|
+
GoodData.logger.info "Would save #{new_item.uri}. Running in dry run mode"
|
|
1400
|
+
else
|
|
1401
|
+
GoodData.logger.info "Saving #{new_item.uri}"
|
|
1402
|
+
new_item.save
|
|
1403
|
+
end
|
|
1404
|
+
else
|
|
1405
|
+
GoodData.logger.info "Ignore #{item.uri}"
|
|
1406
|
+
end
|
|
1407
|
+
end
|
|
1408
|
+
end
|
|
1337
1409
|
nil
|
|
1338
1410
|
end
|
|
1339
1411
|
|
|
@@ -1516,7 +1588,7 @@ module GoodData
|
|
|
1516
1588
|
def import_users(new_users, options = {})
|
|
1517
1589
|
role_list = roles
|
|
1518
1590
|
users_list = users
|
|
1519
|
-
new_users = new_users.map { |x| (x.is_a?(Hash) && x[:user] && x[:user].to_hash.merge(role: x[:role])) || x.to_hash }
|
|
1591
|
+
new_users = new_users.map { |x| ((x.is_a?(Hash) && x[:user] && x[:user].to_hash.merge(role: x[:role])) || x.to_hash).tap { |u| u[:login].downcase! } }
|
|
1520
1592
|
|
|
1521
1593
|
GoodData.logger.warn("Importing users to project (#{pid})")
|
|
1522
1594
|
|
|
@@ -1587,22 +1659,24 @@ module GoodData
|
|
|
1587
1659
|
@log_formatter.log_updated_users(updated_users_result, diff[:changed], role_list)
|
|
1588
1660
|
results.concat(updated_users_result)
|
|
1589
1661
|
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
user
|
|
1662
|
+
unless options[:do_not_touch_users_that_are_not_mentioned]
|
|
1663
|
+
# Remove old users
|
|
1664
|
+
to_disable = diff[:removed].reject { |user| user[:status] == 'DISABLED' || user[:status] == :disabled }
|
|
1665
|
+
GoodData.logger.warn("Disabling #{to_disable.count} users from project (#{pid})")
|
|
1666
|
+
disabled_users_result = disable_users(to_disable, roles: role_list, project_users: whitelisted_users)
|
|
1667
|
+
@log_formatter.log_disabled_users(disabled_users_result)
|
|
1668
|
+
results.concat(disabled_users_result)
|
|
1669
|
+
|
|
1670
|
+
# Remove old users completely
|
|
1671
|
+
if options[:remove_users_from_project]
|
|
1672
|
+
to_remove = (to_disable + users(disabled: true).to_a).map(&:to_hash).uniq do |user|
|
|
1673
|
+
user[:uri]
|
|
1674
|
+
end
|
|
1675
|
+
GoodData.logger.warn("Removing #{to_remove.count} users from project (#{pid})")
|
|
1676
|
+
removed_users_result = remove_users(to_remove)
|
|
1677
|
+
@log_formatter.log_removed_users(removed_users_result)
|
|
1678
|
+
results.concat(removed_users_result)
|
|
1601
1679
|
end
|
|
1602
|
-
GoodData.logger.warn("Removing #{to_remove.count} users from project (#{pid})")
|
|
1603
|
-
removed_users_result = remove_users(to_remove)
|
|
1604
|
-
@log_formatter.log_removed_users(removed_users_result)
|
|
1605
|
-
results.concat(removed_users_result)
|
|
1606
1680
|
end
|
|
1607
1681
|
|
|
1608
1682
|
# reassign to groups
|
|
@@ -1667,7 +1741,11 @@ module GoodData
|
|
|
1667
1741
|
create_group(name: g, description: g)
|
|
1668
1742
|
end
|
|
1669
1743
|
else
|
|
1670
|
-
|
|
1744
|
+
unless missing_groups.empty?
|
|
1745
|
+
fail 'All groups have to be specified before you try to import ' \
|
|
1746
|
+
'users. Groups that are currently in project are ' \
|
|
1747
|
+
"#{groups.join(',')} and you asked for #{missing_groups.join(',')}"
|
|
1748
|
+
end
|
|
1671
1749
|
end
|
|
1672
1750
|
end
|
|
1673
1751
|
|
|
@@ -1716,7 +1794,11 @@ module GoodData
|
|
|
1716
1794
|
client.post("#{uri}/users", 'users' => payload)
|
|
1717
1795
|
end
|
|
1718
1796
|
# this ugly line turns the hash of errors into list of errors with types so we can process them easily
|
|
1719
|
-
typed_results = results.flat_map
|
|
1797
|
+
typed_results = results.flat_map do |x|
|
|
1798
|
+
x['projectUsersUpdateResult'].flat_map do |k, v|
|
|
1799
|
+
v.map { |v2| v2.is_a?(String) ? { type: k.to_sym, user: v2 } : GoodData::Helpers.symbolize_keys(v2).merge(type: k.to_sym) }
|
|
1800
|
+
end
|
|
1801
|
+
end
|
|
1720
1802
|
# we have to concat errors from role resolution and API result
|
|
1721
1803
|
typed_results + (users_by_type[:failed] || [])
|
|
1722
1804
|
end
|
|
@@ -1823,6 +1905,22 @@ module GoodData
|
|
|
1823
1905
|
GoodData::StyleSetting.reset(client: client, project: self)
|
|
1824
1906
|
end
|
|
1825
1907
|
|
|
1908
|
+
# get maql diff from another project or blueprint to current project
|
|
1909
|
+
#
|
|
1910
|
+
# @param options [Hash] options
|
|
1911
|
+
# @option options [GoodData::Project] :project source project
|
|
1912
|
+
# @option options [GoodData::Model::ProjectBlueprint] :blueprint blueprint of source project
|
|
1913
|
+
# @option options [Array] :params additional parameters for diff api
|
|
1914
|
+
# @return [Hash] project model diff
|
|
1915
|
+
def maql_diff(options = {})
|
|
1916
|
+
fail "No :project or :blueprint specified" unless options[:blueprint] || options[:project]
|
|
1917
|
+
bp = options[:blueprint] || options[:project].blueprint
|
|
1918
|
+
uri = "/gdc/projects/#{pid}/model/diff"
|
|
1919
|
+
params = Hash[(options[:params] || []).map { |i| [i, true] }]
|
|
1920
|
+
result = client.post(uri, bp.to_wire, params: params)
|
|
1921
|
+
client.poll_on_code(result['asyncTask']['link']['poll'])
|
|
1922
|
+
end
|
|
1923
|
+
|
|
1826
1924
|
private
|
|
1827
1925
|
|
|
1828
1926
|
def send_mail_to_new_users(users, email_options)
|
|
@@ -1859,14 +1957,14 @@ module GoodData
|
|
|
1859
1957
|
server_side_encryption = options['email_server_side_encryption'] || false
|
|
1860
1958
|
args['s3_server_side_encryption'] = :aes256 if server_side_encryption
|
|
1861
1959
|
|
|
1862
|
-
s3 =
|
|
1863
|
-
bucket = s3.
|
|
1960
|
+
s3 = Aws::S3::Resource.new(args)
|
|
1961
|
+
bucket = s3.bucket(bucket)
|
|
1864
1962
|
process_email_template(bucket, path)
|
|
1865
1963
|
end
|
|
1866
1964
|
|
|
1867
1965
|
def process_email_template(bucket, path)
|
|
1868
1966
|
type = path.split('/').last.include?('.html') ? 'html' : 'txt'
|
|
1869
|
-
body = bucket.
|
|
1967
|
+
body = bucket.object(path).read
|
|
1870
1968
|
body.prepend("MIME-Version: 1.0\nContent-type: text/html\n") if type == 'html'
|
|
1871
1969
|
body
|
|
1872
1970
|
end
|
|
@@ -14,7 +14,7 @@ module GoodData
|
|
|
14
14
|
class ProjectCreator
|
|
15
15
|
class << self
|
|
16
16
|
def migrate(opts = {})
|
|
17
|
-
opts = { client: GoodData.connection }.merge(opts)
|
|
17
|
+
opts = { client: GoodData.connection, execute_ca_scripts: true }.merge(opts)
|
|
18
18
|
client = opts[:client]
|
|
19
19
|
fail ArgumentError, 'No :client specified' if client.nil?
|
|
20
20
|
|
|
@@ -25,15 +25,13 @@ module GoodData
|
|
|
25
25
|
|
|
26
26
|
project = opts[:project] || client.create_project(opts.merge(:title => opts[:title] || spec[:title], :client => client, :environment => opts[:environment]))
|
|
27
27
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
project
|
|
36
|
-
end
|
|
28
|
+
maqls = migrate_datasets(spec, opts.merge(project: project, client: client))
|
|
29
|
+
load(p, spec)
|
|
30
|
+
migrate_metrics(p, spec[:metrics] || [])
|
|
31
|
+
migrate_reports(p, spec[:reports] || [])
|
|
32
|
+
migrate_dashboards(p, spec[:dashboards] || [])
|
|
33
|
+
execute_tests(p, spec[:assert_tests] || [])
|
|
34
|
+
opts[:execute_ca_scripts] ? project : maqls.find { |maql| maql.key?('maqlDdlChunks') }
|
|
37
35
|
end
|
|
38
36
|
|
|
39
37
|
def migrate_datasets(spec, opts = {})
|
|
@@ -41,17 +39,18 @@ module GoodData
|
|
|
41
39
|
dry_run = opts[:dry_run]
|
|
42
40
|
replacements = opts['maql_replacements'] || opts[:maql_replacements] || {}
|
|
43
41
|
|
|
44
|
-
|
|
42
|
+
_, project = GoodData.get_client_and_project(opts)
|
|
45
43
|
|
|
46
44
|
bp = ProjectBlueprint.new(spec)
|
|
45
|
+
response = project.maql_diff(blueprint: bp, params: [:includeGrain])
|
|
47
46
|
|
|
48
|
-
|
|
49
|
-
result = client.post(uri, bp.to_wire)
|
|
50
|
-
response = client.poll_on_code(result['asyncTask']['link']['poll'])
|
|
51
|
-
|
|
47
|
+
GoodData.logger.debug("projectModelDiff") { response.pretty_inspect }
|
|
52
48
|
chunks = response['projectModelDiff']['updateScripts']
|
|
53
49
|
return [] if chunks.empty?
|
|
54
50
|
|
|
51
|
+
ca_maql = response['projectModelDiff']['computedAttributesScript'] if response['projectModelDiff']['computedAttributesScript']
|
|
52
|
+
ca_chunks = ca_maql && ca_maql['maqlDdlChunks']
|
|
53
|
+
|
|
55
54
|
maqls = pick_correct_chunks(chunks, opts)
|
|
56
55
|
replaced_maqls = apply_replacements_on_maql(maqls, replacements)
|
|
57
56
|
|
|
@@ -59,19 +58,32 @@ module GoodData
|
|
|
59
58
|
errors = []
|
|
60
59
|
replaced_maqls.each do |replaced_maql_chunks|
|
|
61
60
|
begin
|
|
62
|
-
replaced_maql_chunks['updateScript']['maqlDdlChunks'].each
|
|
61
|
+
replaced_maql_chunks['updateScript']['maqlDdlChunks'].each do |chunk|
|
|
62
|
+
GoodData.logger.debug(chunk)
|
|
63
|
+
project.execute_maql(chunk)
|
|
64
|
+
end
|
|
63
65
|
rescue => e
|
|
64
66
|
puts "Error occured when executing MAQL, project: \"#{project.title}\" reason: \"#{e.message}\", chunks: #{replaced_maql_chunks.inspect}"
|
|
65
67
|
errors << e
|
|
66
68
|
next
|
|
67
69
|
end
|
|
68
70
|
end
|
|
71
|
+
|
|
72
|
+
if ca_chunks && opts[:execute_ca_scripts]
|
|
73
|
+
begin
|
|
74
|
+
ca_chunks.each { |chunk| project.execute_maql(chunk) }
|
|
75
|
+
rescue => e
|
|
76
|
+
puts "Error occured when executing MAQL, project: \"#{project.title}\" reason: \"#{e.message}\", chunks: #{ca_chunks.inspect}"
|
|
77
|
+
errors << e
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
69
81
|
if (!errors.empty?) && (errors.length == replaced_maqls.length)
|
|
70
|
-
messages = errors.map { |
|
|
82
|
+
messages = errors.map { |err| GoodData::Helpers.interpolate_error_messages(err.data['wTaskStatus']['messages']) }
|
|
71
83
|
fail "Unable to migrate LDM, reason(s): \n #{messages.join("\n")}"
|
|
72
84
|
end
|
|
73
85
|
end
|
|
74
|
-
replaced_maqls
|
|
86
|
+
replaced_maqls + (ca_maql ? [ca_maql] : [])
|
|
75
87
|
end
|
|
76
88
|
|
|
77
89
|
def migrate_reports(project, spec)
|
|
@@ -111,7 +123,9 @@ module GoodData
|
|
|
111
123
|
end
|
|
112
124
|
|
|
113
125
|
def pick_correct_chunks(chunks, opts = {})
|
|
126
|
+
GoodData.logger.debug("update_preference") { opts[:update_preference].pretty_inspect }
|
|
114
127
|
preference = GoodData::Helpers.symbolize_keys(opts[:update_preference] || {})
|
|
128
|
+
preference = Hash[preference.map { |k, v| [k, GoodData::Helpers.to_boolean(v)] }]
|
|
115
129
|
|
|
116
130
|
# first is cascadeDrops, second is preserveData
|
|
117
131
|
rules = [
|
|
@@ -126,10 +140,19 @@ module GoodData
|
|
|
126
140
|
end
|
|
127
141
|
|
|
128
142
|
stuff = stuff.map do |chunk|
|
|
129
|
-
{ cascade_drops: chunk['updateScript']['cascadeDrops'],
|
|
143
|
+
{ cascade_drops: chunk['updateScript']['cascadeDrops'],
|
|
144
|
+
preserve_data: chunk['updateScript']['preserveData'],
|
|
145
|
+
maql: chunk['updateScript']['maqlDdlChunks'],
|
|
146
|
+
orig: chunk }
|
|
130
147
|
end
|
|
131
148
|
|
|
132
|
-
results_from_api = GoodData::Helpers.join(
|
|
149
|
+
results_from_api = GoodData::Helpers.join(
|
|
150
|
+
rules,
|
|
151
|
+
stuff,
|
|
152
|
+
[:cascade_drops, :preserve_data],
|
|
153
|
+
[:cascade_drops, :preserve_data],
|
|
154
|
+
inner: true
|
|
155
|
+
).sort_by { |l| l[:priority] } || []
|
|
133
156
|
|
|
134
157
|
if preference.empty?
|
|
135
158
|
[results_from_api.first[:orig]]
|