gooddata 0.6.51 → 0.6.52

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +1 -1
  3. data/CHANGELOG.md +13 -1
  4. data/CONTRIBUTING.md +25 -0
  5. data/PULL_REQUEST_TEMPLATE.md +5 -0
  6. data/README.md +7 -4
  7. data/gooddata.gemspec +2 -3
  8. data/lib/gooddata.rb +1 -0
  9. data/lib/gooddata/bricks/base_downloader.rb +6 -6
  10. data/lib/gooddata/bricks/middleware/aws_middleware.rb +15 -5
  11. data/lib/gooddata/bricks/middleware/dwh_middleware.rb +15 -3
  12. data/lib/gooddata/bricks/middleware/gooddata_middleware.rb +13 -4
  13. data/lib/gooddata/bricks/middleware/logger_middleware.rb +3 -0
  14. data/lib/gooddata/exceptions/no_project_error.rb +5 -1
  15. data/lib/gooddata/goodzilla/goodzilla.rb +7 -6
  16. data/lib/gooddata/helpers/data_helper.rb +4 -4
  17. data/lib/gooddata/helpers/global_helpers_params.rb +61 -39
  18. data/lib/gooddata/lcm/actions/apply_custom_maql.rb +9 -0
  19. data/lib/gooddata/lcm/actions/associate_clients.rb +23 -4
  20. data/lib/gooddata/lcm/actions/collect_attrs.rb +56 -0
  21. data/lib/gooddata/lcm/actions/collect_ca_metrics.rb +53 -0
  22. data/lib/gooddata/lcm/actions/collect_clients.rb +25 -3
  23. data/lib/gooddata/lcm/actions/collect_meta.rb +83 -0
  24. data/lib/gooddata/lcm/actions/collect_segment_clients.rb +12 -4
  25. data/lib/gooddata/lcm/actions/collect_segments.rb +4 -4
  26. data/lib/gooddata/lcm/actions/collect_tagged_objects.rb +74 -0
  27. data/lib/gooddata/lcm/actions/create_segment_masters.rb +16 -30
  28. data/lib/gooddata/lcm/actions/ensure_release_table.rb +0 -3
  29. data/lib/gooddata/lcm/actions/ensure_segments.rb +1 -4
  30. data/lib/gooddata/lcm/actions/ensure_technical_users_domain.rb +5 -5
  31. data/lib/gooddata/lcm/actions/ensure_technical_users_project.rb +8 -5
  32. data/lib/gooddata/lcm/actions/hello_world.rb +0 -3
  33. data/lib/gooddata/lcm/actions/import_object_collections.rb +60 -0
  34. data/lib/gooddata/lcm/actions/print_actions.rb +0 -3
  35. data/lib/gooddata/lcm/actions/print_modes.rb +0 -3
  36. data/lib/gooddata/lcm/actions/print_types.rb +1 -4
  37. data/lib/gooddata/lcm/actions/provision_clients.rb +5 -5
  38. data/lib/gooddata/lcm/actions/purge_clients.rb +4 -10
  39. data/lib/gooddata/lcm/actions/segments_filter.rb +0 -6
  40. data/lib/gooddata/lcm/actions/synchronize_attribute_drillpaths.rb +8 -4
  41. data/lib/gooddata/lcm/actions/synchronize_cas.rb +61 -0
  42. data/lib/gooddata/lcm/actions/synchronize_clients.rb +9 -3
  43. data/lib/gooddata/lcm/actions/synchronize_color_palette.rb +13 -5
  44. data/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb +71 -17
  45. data/lib/gooddata/lcm/actions/synchronize_label_types.rb +8 -5
  46. data/lib/gooddata/lcm/actions/synchronize_ldm.rb +17 -8
  47. data/lib/gooddata/lcm/actions/synchronize_meta.rb +0 -3
  48. data/lib/gooddata/lcm/actions/synchronize_new_segments.rb +9 -4
  49. data/lib/gooddata/lcm/actions/synchronize_processes.rb +9 -5
  50. data/lib/gooddata/lcm/actions/synchronize_schedules.rb +15 -5
  51. data/lib/gooddata/lcm/actions/synchronize_tag_objects.rb +61 -0
  52. data/lib/gooddata/lcm/actions/update_release_table.rb +0 -3
  53. data/lib/gooddata/lcm/helpers/tags_helper.rb +35 -0
  54. data/lib/gooddata/lcm/lcm.rb +22 -4
  55. data/lib/gooddata/lcm/lcm2.rb +66 -13
  56. data/lib/gooddata/lcm/types/complex/update_preference.rb +1 -1
  57. data/lib/gooddata/mixins/md_finders.rb +4 -2
  58. data/lib/gooddata/mixins/md_object_indexer.rb +13 -3
  59. data/lib/gooddata/mixins/md_object_query.rb +8 -2
  60. data/lib/gooddata/models/blueprint/date_dimension.rb +6 -0
  61. data/lib/gooddata/models/blueprint/project_blueprint.rb +41 -11
  62. data/lib/gooddata/models/blueprint/project_builder.rb +20 -0
  63. data/lib/gooddata/models/blueprint/to_wire.rb +7 -0
  64. data/lib/gooddata/models/client.rb +6 -0
  65. data/lib/gooddata/models/domain.rb +6 -6
  66. data/lib/gooddata/models/from_wire.rb +5 -1
  67. data/lib/gooddata/models/metadata.rb +55 -9
  68. data/lib/gooddata/models/metadata/attribute.rb +19 -4
  69. data/lib/gooddata/models/metadata/dashboard.rb +15 -3
  70. data/lib/gooddata/models/metadata/dataset.rb +5 -2
  71. data/lib/gooddata/models/metadata/dimension.rb +4 -1
  72. data/lib/gooddata/models/metadata/fact.rb +9 -2
  73. data/lib/gooddata/models/metadata/folder.rb +4 -1
  74. data/lib/gooddata/models/metadata/metric.rb +11 -3
  75. data/lib/gooddata/models/metadata/report.rb +7 -2
  76. data/lib/gooddata/models/metadata/report_definition.rb +11 -4
  77. data/lib/gooddata/models/metadata/scheduled_mail.rb +4 -1
  78. data/lib/gooddata/models/metadata/variable.rb +7 -2
  79. data/lib/gooddata/models/model.rb +14 -3
  80. data/lib/gooddata/models/process.rb +10 -9
  81. data/lib/gooddata/models/project.rb +134 -36
  82. data/lib/gooddata/models/project_creator.rb +43 -20
  83. data/lib/gooddata/models/report_data_result.rb +6 -2
  84. data/lib/gooddata/models/schedule.rb +6 -3
  85. data/lib/gooddata/models/subscription.rb +8 -1
  86. data/lib/gooddata/models/user_filters/user_filter.rb +1 -0
  87. data/lib/gooddata/models/user_filters/user_filter_builder.rb +18 -4
  88. data/lib/gooddata/models/user_filters/variable_user_filter.rb +3 -1
  89. data/lib/gooddata/rest/client.rb +4 -6
  90. data/lib/gooddata/rest/connection.rb +10 -2
  91. data/lib/gooddata/version.rb +1 -1
  92. data/spec/data/blueprints/test_blueprint.json +1 -0
  93. data/spec/data/wire_models/test_blueprint.json +3 -0
  94. data/spec/data/workspace_table.csv +3 -0
  95. data/spec/environment/development.rb +4 -1
  96. data/spec/environment/environment.rb +1 -1
  97. data/spec/environment/staging.rb +5 -1
  98. data/spec/environment/testing.rb +5 -2
  99. data/spec/integration/blueprint_with_ca_spec.rb +56 -0
  100. data/spec/integration/clients_spec.rb +21 -0
  101. data/spec/integration/command_datawarehouse_spec.rb +7 -1
  102. data/spec/integration/create_from_template_spec.rb +9 -3
  103. data/spec/integration/project_spec.rb +7 -0
  104. data/spec/integration/segments_spec.rb +0 -53
  105. data/spec/integration/subscription_spec.rb +29 -4
  106. data/spec/integration/urn_date_dim_spec.rb +53 -0
  107. data/spec/integration/user_filters_spec.rb +6 -0
  108. data/spec/integration/variables_spec.rb +1 -2
  109. data/spec/spec_helper.rb +5 -30
  110. data/spec/unit/actions/collect_clients_spec.rb +38 -0
  111. data/spec/unit/actions/collect_meta_spec.rb +87 -0
  112. data/spec/unit/actions/collect_segment_clients_spec.rb +40 -0
  113. data/spec/unit/actions/collect_tagged_objects_spec.rb +110 -0
  114. data/spec/unit/actions/synchronize_etls_in_segment_spec.rb +51 -0
  115. data/spec/unit/bricks/middleware/aws_middelware_spec.rb +55 -1
  116. data/spec/unit/bricks/middleware/logger_middleware_spec.rb +15 -0
  117. data/spec/unit/helpers/data_helper_spec.rb +3 -5
  118. data/spec/unit/helpers/global_helpers_spec.rb +29 -0
  119. data/spec/unit/helpers_spec.rb +18 -1
  120. data/spec/unit/models/blueprint/project_blueprint_spec.rb +1 -23
  121. data/spec/unit/models/domain_spec.rb +19 -0
  122. data/spec/unit/models/metadata_spec.rb +34 -0
  123. data/spec/unit/models/schedule_spec.rb +31 -0
  124. data/spec/unit/models/to_manifest_spec.rb +10 -2
  125. data/spec/unit/models/unit_project_spec.rb +6 -1
  126. data/spec/unit/rest/polling_spec.rb +13 -1
  127. metadata +49 -31
@@ -48,7 +48,10 @@ module GoodData
48
48
  # Method intended to get all objects of that type in a specified project
49
49
  #
50
50
  # @param options [Hash] the options hash
51
- # @option options [Boolean] :full if passed true the subclass can decide to pull in full objects. This is desirable from the usability POV but unfortunately has negative impact on performance so it is not the default
51
+ # @option options [Boolean] :full if passed true the subclass can decide
52
+ # to pull in full objects. This is desirable from the usability POV
53
+ # but unfortunately has negative impact on performance so it is not the
54
+ # default
52
55
  # @return [Array<GoodData::MdObject> | Array<Hash>] Return the appropriate metadata objects or their representation
53
56
  def all(options = { :client => GoodData.connection, :project => GoodData.project })
54
57
  query('scheduledMail', ScheduledMail, options)
@@ -14,7 +14,10 @@ module GoodData
14
14
  # Method intended to get all objects of that type in a specified project
15
15
  #
16
16
  # @param options [Hash] the options hash
17
- # @option options [Boolean] :full if passed true the subclass can decide to pull in full objects. This is desirable from the usability POV but unfortunately has negative impact on performance so it is not the default
17
+ # @option options [Boolean] :full if passed true the subclass can decide
18
+ # to pull in full objects. This is desirable from the usability
19
+ # POV but unfortunately has negative impact on performance so it
20
+ # is not the default
18
21
  # @return [Array<GoodData::MdObject> | Array<Hash>] Return the appropriate metadata objects or their representation
19
22
  def all(options = { :client => GoodData.connection, :project => GoodData.project })
20
23
  query('prompt', Variable, options)
@@ -67,7 +70,9 @@ module GoodData
67
70
  values.select { |x| x.level == :user }
68
71
  end
69
72
 
70
- # Method used for replacing values in their state according to mapping. Can be used to replace any values but it is typically used to replace the URIs. Returns a new object of the same type.
73
+ # Method used for replacing values in their state according to mapping.
74
+ # Can be used to replace any values but it is typically used to replace
75
+ # the URIs. Returns a new object of the same type.
71
76
  #
72
77
  # @param [Array<Array>]Mapping specifying what should be exchanged for what. As mapping should be used output of GoodData::Helpers.prepare_mapping.
73
78
  # @return [GoodData::Variable]
@@ -267,13 +267,24 @@ module GoodData
267
267
  d = GoodData::Helpers.deep_dup(a_schema_blueprint)
268
268
  d[:columns] = d[:columns] + b_schema_blueprint[:columns]
269
269
  d[:columns].uniq!
270
- columns_that_failed_to_merge = d[:columns].group_by { |x| [:reference, :date].include?(x[:type]) ? x[:dataset] : x[:id] }.map { |k, v| [k, v.count, v] }.select { |x| x[1] > 1 }
270
+ columns_that_failed_to_merge = d[:columns]
271
+ .group_by { |x| [:reference, :date].include?(x[:type]) ? x[:dataset] : x[:id] }
272
+ .map { |k, v| [k, v.count, v] }.select { |x| x[1] > 1 }
271
273
  unless columns_that_failed_to_merge.empty?
272
274
  columns_that_failed_to_merge.each do |error|
273
- GoodData.logger.error "Columns #{error[0]} failed to merge. There are #{error[1]} conflicting columns. When merging columns with the same name they have to be identical."
275
+ message = "Columns #{error[0]} failed to merge. There are " \
276
+ "#{error[1]} conflicting columns. When merging columns " \
277
+ "with the same name they have to be identical."
278
+ GoodData.logger.error message
274
279
  GoodData.logger.error error[2]
275
280
  end
276
- fail "Columns #{columns_that_failed_to_merge.first} failed to merge. There are #{columns_that_failed_to_merge[1]} conflicting columns. #{columns_that_failed_to_merge[2]} When merging columns with the same name they have to be identical." unless columns_that_failed_to_merge.empty?
281
+ unless columns_that_failed_to_merge.empty?
282
+ fail "Columns #{columns_that_failed_to_merge.first} failed to " \
283
+ "merge. There are #{columns_that_failed_to_merge[1]} " \
284
+ "conflicting columns. #{columns_that_failed_to_merge[2]} " \
285
+ "When merging columns with the same name they have to be " \
286
+ "identical."
287
+ end
277
288
  end
278
289
  d
279
290
  end
@@ -123,6 +123,8 @@ module GoodData
123
123
  client.put("/gdc/projects/#{project.pid}/dataload/processes/#{process_id}", data)
124
124
  end
125
125
 
126
+ File.delete(deployed_path) if File.exist?(deployed_path)
127
+
126
128
  process = client.create(Process, res, project: project)
127
129
  puts HighLine.color("Deploy DONE #{path}", HighLine::GREEN) if verbose
128
130
  process
@@ -207,16 +209,15 @@ module GoodData
207
209
 
208
210
  def with_zip(opts = {})
209
211
  client = opts[:client]
210
- Tempfile.open('deploy-graph-archive') do |temp|
211
- zip_filename = temp.path
212
- File.open(zip_filename, 'w') do |zip|
213
- Zip::File.open(zip.path, Zip::File::CREATE) do |zipfile|
214
- yield zipfile
215
- end
216
- end
217
- client.upload_to_user_webdav(temp.path, opts)
218
- temp.path
212
+ temp = Tempfile.new(['deploy-graph-archive', '.zip'])
213
+ zip_filename = temp.path
214
+
215
+ temp.close!
216
+ Zip::File.open(zip_filename, Zip::File::CREATE) do |zipfile|
217
+ yield zipfile
219
218
  end
219
+ client.upload_to_user_webdav(zip_filename, opts)
220
+ zip_filename
220
221
  end
221
222
 
222
223
  def zip_and_upload(path, files_to_exclude, opts = {})
@@ -140,7 +140,9 @@ module GoodData
140
140
 
141
141
  project = create_object(opts)
142
142
  project.save
143
- # until it is enabled or deleted, recur. This should still end if there is a exception thrown out from RESTClient. This sometimes happens from WebApp when request is too long
143
+ # until it is enabled or deleted, recur. This should still end if there
144
+ # is a exception thrown out from RESTClient. This sometimes happens from
145
+ # WebApp when request is too long
144
146
  while project.state.to_s != 'enabled'
145
147
  if project.deleted?
146
148
  # if project is switched to deleted state, fail. This is usually problem of creating a template which is invalid.
@@ -220,7 +222,10 @@ module GoodData
220
222
  # Clones project along with etl and schedules.
221
223
  #
222
224
  # @param client [GoodData::Rest::Client] GoodData client to be used for connection
223
- # @param from_project [GoodData::Project | GoodData::Segment | GoodData:Client | String] Object to be cloned from. Can be either segment in which case we take the master, client in which case we take its project, string in which case we treat is as an project object or directly project
225
+ # @param from_project [GoodData::Project | GoodData::Segment | GoodData:Client | String]
226
+ # Object to be cloned from. Can be either segment in which case we
227
+ # take the master, client in which case we take its project, string
228
+ # in which case we treat is as an project object or directly project
224
229
  # @param to_project [GoodData::Project | GoodData::Segment | GoodData:Client | String]
225
230
  def transfer_etl(client, from_project, to_project)
226
231
  from_project = case from_project
@@ -304,7 +309,10 @@ module GoodData
304
309
  # Clones project along with etl and schedules.
305
310
  #
306
311
  # @param client [GoodData::Rest::Client] GoodData client to be used for connection
307
- # @param from_project [GoodData::Project | GoodData::Segment | GoodData:Client | String] Object to be cloned from. Can be either segment in which case we take the master, client in which case we take its project, string in which case we treat is as an project object or directly project
312
+ # @param from_project [GoodData::Project | GoodData::Segment | GoodData:Client | String]
313
+ # Object to be cloned from. Can be either segment in which case we take
314
+ # the master, client in which case we take its project, string in which
315
+ # case we treat is as an project object or directly project.
308
316
  def transfer_schedules(from_project, to_project)
309
317
  cache = to_project
310
318
  .processes.sort_by(&:name)
@@ -335,7 +343,7 @@ module GoodData
335
343
  v.compact
336
344
  end
337
345
 
338
- diff = GoodData::Helpers.diff(remote_stuff, local_stuff, key: :name, fields: [:name, :cron, :after, :params, :hidden_params, :reschedule])
346
+ diff = GoodData::Helpers.diff(remote_stuff, local_stuff, key: :name, fields: [:name, :cron, :after, :params, :hidden_params, :reschedule, :state])
339
347
  stack = diff[:added].map do |x|
340
348
  [:added, x]
341
349
  end
@@ -369,7 +377,7 @@ module GoodData
369
377
  if process_spec.type != :dataload
370
378
  executable = schedule_spec[:executable] || (process_spec.type == :ruby ? 'main.rb' : 'main.grf')
371
379
  end
372
- params = schedule_parameters(to_project.pid, schedule_spec)
380
+ params = schedule_parameters(schedule_spec)
373
381
  created_schedule = remote_process.create_schedule(schedule_spec[:cron] || schedule_cache[schedule_spec[:after]], executable, params)
374
382
  schedule_cache[created_schedule.name] = created_schedule
375
383
 
@@ -403,6 +411,7 @@ module GoodData
403
411
 
404
412
  schedule.reschedule = schedule_spec[:reschedule]
405
413
  schedule.name = schedule_spec[:name]
414
+ schedule.state = schedule_spec[:state]
406
415
  schedule.save
407
416
  schedule_cache[schedule.name] = schedule
408
417
 
@@ -449,12 +458,13 @@ module GoodData
449
458
 
450
459
  private
451
460
 
452
- def schedule_parameters(id, schedule_spec)
461
+ def schedule_parameters(schedule_spec)
453
462
  {
454
- params: schedule_spec[:params].merge('PROJECT_ID' => id),
463
+ params: schedule_spec[:params],
455
464
  hidden_params: schedule_spec[:hidden_params],
456
465
  name: schedule_spec[:name],
457
- reschedule: schedule_spec[:reschedule]
466
+ reschedule: schedule_spec[:reschedule],
467
+ state: schedule_spec[:state]
458
468
  }
459
469
  end
460
470
  end
@@ -535,6 +545,15 @@ module GoodData
535
545
  GoodData::Attribute[id, project: self, client: client]
536
546
  end
537
547
 
548
+ def computed_attributes(id = :all)
549
+ attrs = attributes(id)
550
+ if attrs.is_a?(GoodData::Attribute)
551
+ attrs.computed_attribute? ? attrs : nil
552
+ else
553
+ attrs.select(&:computed_attribute?)
554
+ end
555
+ end
556
+
538
557
  def attribute_by_identifier(identifier)
539
558
  GoodData::Attribute.find_first_by_identifier(identifier, project: self, client: client)
540
559
  end
@@ -555,10 +574,10 @@ module GoodData
555
574
  #
556
575
  # @return [GoodData::ProjectRole] Project role if found
557
576
  def blueprint(options = {})
558
- result = client.get("/gdc/projects/#{pid}/model/view", params: { includeDeprecated: true, includeGrain: true })
577
+ result = client.get("/gdc/projects/#{pid}/model/view", params: { includeDeprecated: true, includeGrain: true, includeCA: true })
559
578
  polling_url = result['asyncTask']['link']['poll']
560
579
  model = client.poll_on_code(polling_url, options)
561
- bp = GoodData::Model::FromWire.from_wire(model)
580
+ bp = GoodData::Model::FromWire.from_wire(model, { include_ca: true }.merge(options))
562
581
  bp.title = title
563
582
  bp
564
583
  end
@@ -1119,10 +1138,18 @@ module GoodData
1119
1138
  def objects_export(objs, options = {})
1120
1139
  fail 'Nothing to migrate. You have to pass list of objects, ids or uris that you would like to migrate' if objs.nil?
1121
1140
  objs = Array(objs)
1122
- fail 'Nothing to migrate. The list you provided is empty' if objs.empty?
1141
+ if objs.empty?
1142
+ GoodData.logger.warn 'Nothing to migrate.'
1143
+ return
1144
+ end
1123
1145
 
1124
1146
  objs = objs.pmap { |obj| [obj, objects(obj)] }
1125
- fail ObjectsExportError, "Exporting objects failed with messages. Object #{objs.select { |_, obj| obj.nil? }.map { |o, _| o }.join(', ')} could not be found." if objs.any? { |_, obj| obj.nil? }
1147
+ if objs.any? { |_, obj| obj.nil? }
1148
+ object = objs.select { |_, obj| obj.nil? }.map { |o, _| o }.join(', ')
1149
+ error_message = "Exporting objects failed with messages. " \
1150
+ "Object #{object} could not be found."
1151
+ fail ObjectsExportError, error_message
1152
+ end
1126
1153
  export_payload = {
1127
1154
  :partialMDExport => {
1128
1155
  :uris => objs.map { |_, obj| obj.uri },
@@ -1158,7 +1185,8 @@ module GoodData
1158
1185
  :partialMDImport => {
1159
1186
  :token => token,
1160
1187
  :overwriteNewer => '1',
1161
- :updateLDMObjects => '0'
1188
+ :updateLDMObjects => '0',
1189
+ :importAttributeProperties => '1'
1162
1190
  }
1163
1191
  }
1164
1192
 
@@ -1183,11 +1211,15 @@ module GoodData
1183
1211
  # @option options [GoodData::Project | String | Array<String> | Array<GoodData::Project>] :project Project(s) to migrate to
1184
1212
  # @option options [Number] :batch_size Number of projects that are migrated at the same time. Default is 10
1185
1213
  #
1186
- # @return [Boolean | Array<Hash>] Return either true or throws exception if you passed only one project. If you provided an array returns list of hashes signifying sucees or failure. Take note that in case of list of projects it does not throw exception
1214
+ # @return [Boolean | Array<Hash>] Return either true or throws exception
1215
+ # if you passed only one project. If you provided an array returns list
1216
+ # of hashes signifying sucees or failure. Take note that in case of list
1217
+ # of projects it does not throw exception.
1187
1218
  def partial_md_export(objects, options = {})
1188
1219
  projects = options[:project]
1189
1220
  batch_size = options[:batch_size] || 10
1190
1221
  token = objects_export(objects)
1222
+ return if token.nil?
1191
1223
 
1192
1224
  if projects.is_a?(Array)
1193
1225
  projects.each_slice(batch_size).flat_map do |batch|
@@ -1269,7 +1301,9 @@ module GoodData
1269
1301
  self
1270
1302
  end
1271
1303
 
1272
- # Method used for walking through objects in project and trying to replace all occurences of some object for another object. This is typically used as a means for exchanging Date dimensions.
1304
+ # Method used for walking through objects in project and trying to
1305
+ # replace all occurences of some object for another object. This is
1306
+ # typically used as a means for exchanging Date dimensions.
1273
1307
  #
1274
1308
  # @param mapping [Array<Array>] Mapping specifying what should be exchanged for what. As mapping should be used output of GoodData::Helpers.prepare_mapping.
1275
1309
  def replace_from_mapping(mapping, opts = {})
@@ -1328,12 +1362,50 @@ module GoodData
1328
1362
  end
1329
1363
  end
1330
1364
 
1365
+ GoodData.logger.info 'Replacing dashboard saved views'
1366
+ contexts = mapping.map { |a, _| a }.pmapcat { |a| a.usedby('executionContext') }.map { |a| GoodData::MdObject[a['link'], client: client, project: self] }
1367
+ puts "Found #{contexts.count} dashboard saved views"
1368
+ contexts.peach do |item|
1369
+ new_item = GoodData::MdObject.replace_quoted(item, mapping)
1370
+ if new_item.json != item.json
1371
+ if dry_run
1372
+ GoodData.logger.info "Would save #{new_item.uri}. Running in dry run mode"
1373
+ else
1374
+ GoodData.logger.info "Saving #{new_item.uri}"
1375
+ new_item.save
1376
+ end
1377
+ else
1378
+ GoodData.logger.info "Ignore #{item.uri}"
1379
+ end
1380
+ end
1381
+
1331
1382
  GoodData.logger.info 'Replacing variable values'
1332
1383
  variables.each do |var|
1333
1384
  var.values.peach do |val|
1334
1385
  val.replace(mapping).save unless dry_run
1335
1386
  end
1336
1387
  end
1388
+
1389
+ {
1390
+ visualizations: MdObject.query('visualization', MdObject, client: client, project: self),
1391
+ visualization_widgets: MdObject.query('visualizationWidget', MdObject, client: client, project: self),
1392
+ kpis: MdObject.query('kpi', MdObject, client: client, project: self)
1393
+ }.each do |key, collection|
1394
+ GoodData.logger.info "Replacing #{key}"
1395
+ collection.each do |item|
1396
+ new_item = MdObject.replace_quoted(item, mapping)
1397
+ if new_item.json != item.json
1398
+ if dry_run
1399
+ GoodData.logger.info "Would save #{new_item.uri}. Running in dry run mode"
1400
+ else
1401
+ GoodData.logger.info "Saving #{new_item.uri}"
1402
+ new_item.save
1403
+ end
1404
+ else
1405
+ GoodData.logger.info "Ignore #{item.uri}"
1406
+ end
1407
+ end
1408
+ end
1337
1409
  nil
1338
1410
  end
1339
1411
 
@@ -1516,7 +1588,7 @@ module GoodData
1516
1588
  def import_users(new_users, options = {})
1517
1589
  role_list = roles
1518
1590
  users_list = users
1519
- new_users = new_users.map { |x| (x.is_a?(Hash) && x[:user] && x[:user].to_hash.merge(role: x[:role])) || x.to_hash }
1591
+ new_users = new_users.map { |x| ((x.is_a?(Hash) && x[:user] && x[:user].to_hash.merge(role: x[:role])) || x.to_hash).tap { |u| u[:login].downcase! } }
1520
1592
 
1521
1593
  GoodData.logger.warn("Importing users to project (#{pid})")
1522
1594
 
@@ -1587,22 +1659,24 @@ module GoodData
1587
1659
  @log_formatter.log_updated_users(updated_users_result, diff[:changed], role_list)
1588
1660
  results.concat(updated_users_result)
1589
1661
 
1590
- # Remove old users
1591
- to_disable = diff[:removed].reject { |user| user[:status] == 'DISABLED' || user[:status] == :disabled }
1592
- GoodData.logger.warn("Disabling #{to_disable.count} users from project (#{pid})")
1593
- disabled_users_result = disable_users(to_disable, roles: role_list, project_users: whitelisted_users)
1594
- @log_formatter.log_disabled_users(disabled_users_result)
1595
- results.concat(disabled_users_result)
1596
-
1597
- # Remove old users completely
1598
- if options[:remove_users_from_project]
1599
- to_remove = (to_disable + users(disabled: true).to_a).map(&:to_hash).uniq do |user|
1600
- user[:uri]
1662
+ unless options[:do_not_touch_users_that_are_not_mentioned]
1663
+ # Remove old users
1664
+ to_disable = diff[:removed].reject { |user| user[:status] == 'DISABLED' || user[:status] == :disabled }
1665
+ GoodData.logger.warn("Disabling #{to_disable.count} users from project (#{pid})")
1666
+ disabled_users_result = disable_users(to_disable, roles: role_list, project_users: whitelisted_users)
1667
+ @log_formatter.log_disabled_users(disabled_users_result)
1668
+ results.concat(disabled_users_result)
1669
+
1670
+ # Remove old users completely
1671
+ if options[:remove_users_from_project]
1672
+ to_remove = (to_disable + users(disabled: true).to_a).map(&:to_hash).uniq do |user|
1673
+ user[:uri]
1674
+ end
1675
+ GoodData.logger.warn("Removing #{to_remove.count} users from project (#{pid})")
1676
+ removed_users_result = remove_users(to_remove)
1677
+ @log_formatter.log_removed_users(removed_users_result)
1678
+ results.concat(removed_users_result)
1601
1679
  end
1602
- GoodData.logger.warn("Removing #{to_remove.count} users from project (#{pid})")
1603
- removed_users_result = remove_users(to_remove)
1604
- @log_formatter.log_removed_users(removed_users_result)
1605
- results.concat(removed_users_result)
1606
1680
  end
1607
1681
 
1608
1682
  # reassign to groups
@@ -1667,7 +1741,11 @@ module GoodData
1667
1741
  create_group(name: g, description: g)
1668
1742
  end
1669
1743
  else
1670
- fail "All groups have to be specified before you try to import users. Groups that are currently in project are #{groups.join(',')} and you asked for #{missing_groups.join(',')}" unless missing_groups.empty?
1744
+ unless missing_groups.empty?
1745
+ fail 'All groups have to be specified before you try to import ' \
1746
+ 'users. Groups that are currently in project are ' \
1747
+ "#{groups.join(',')} and you asked for #{missing_groups.join(',')}"
1748
+ end
1671
1749
  end
1672
1750
  end
1673
1751
 
@@ -1716,7 +1794,11 @@ module GoodData
1716
1794
  client.post("#{uri}/users", 'users' => payload)
1717
1795
  end
1718
1796
  # this ugly line turns the hash of errors into list of errors with types so we can process them easily
1719
- typed_results = results.flat_map { |x| x['projectUsersUpdateResult'].flat_map { |k, v| v.map { |v2| v2.is_a?(String) ? { type: k.to_sym, user: v2 } : GoodData::Helpers.symbolize_keys(v2).merge(type: k.to_sym) } } }
1797
+ typed_results = results.flat_map do |x|
1798
+ x['projectUsersUpdateResult'].flat_map do |k, v|
1799
+ v.map { |v2| v2.is_a?(String) ? { type: k.to_sym, user: v2 } : GoodData::Helpers.symbolize_keys(v2).merge(type: k.to_sym) }
1800
+ end
1801
+ end
1720
1802
  # we have to concat errors from role resolution and API result
1721
1803
  typed_results + (users_by_type[:failed] || [])
1722
1804
  end
@@ -1823,6 +1905,22 @@ module GoodData
1823
1905
  GoodData::StyleSetting.reset(client: client, project: self)
1824
1906
  end
1825
1907
 
1908
+ # get maql diff from another project or blueprint to current project
1909
+ #
1910
+ # @param options [Hash] options
1911
+ # @option options [GoodData::Project] :project source project
1912
+ # @option options [GoodData::Model::ProjectBlueprint] :blueprint blueprint of source project
1913
+ # @option options [Array] :params additional parameters for diff api
1914
+ # @return [Hash] project model diff
1915
+ def maql_diff(options = {})
1916
+ fail "No :project or :blueprint specified" unless options[:blueprint] || options[:project]
1917
+ bp = options[:blueprint] || options[:project].blueprint
1918
+ uri = "/gdc/projects/#{pid}/model/diff"
1919
+ params = Hash[(options[:params] || []).map { |i| [i, true] }]
1920
+ result = client.post(uri, bp.to_wire, params: params)
1921
+ client.poll_on_code(result['asyncTask']['link']['poll'])
1922
+ end
1923
+
1826
1924
  private
1827
1925
 
1828
1926
  def send_mail_to_new_users(users, email_options)
@@ -1859,14 +1957,14 @@ module GoodData
1859
1957
  server_side_encryption = options['email_server_side_encryption'] || false
1860
1958
  args['s3_server_side_encryption'] = :aes256 if server_side_encryption
1861
1959
 
1862
- s3 = AWS::S3.new(args)
1863
- bucket = s3.buckets[bucket]
1960
+ s3 = Aws::S3::Resource.new(args)
1961
+ bucket = s3.bucket(bucket)
1864
1962
  process_email_template(bucket, path)
1865
1963
  end
1866
1964
 
1867
1965
  def process_email_template(bucket, path)
1868
1966
  type = path.split('/').last.include?('.html') ? 'html' : 'txt'
1869
- body = bucket.objects[path].read
1967
+ body = bucket.object(path).read
1870
1968
  body.prepend("MIME-Version: 1.0\nContent-type: text/html\n") if type == 'html'
1871
1969
  body
1872
1970
  end
@@ -14,7 +14,7 @@ module GoodData
14
14
  class ProjectCreator
15
15
  class << self
16
16
  def migrate(opts = {})
17
- opts = { client: GoodData.connection }.merge(opts)
17
+ opts = { client: GoodData.connection, execute_ca_scripts: true }.merge(opts)
18
18
  client = opts[:client]
19
19
  fail ArgumentError, 'No :client specified' if client.nil?
20
20
 
@@ -25,15 +25,13 @@ module GoodData
25
25
 
26
26
  project = opts[:project] || client.create_project(opts.merge(:title => opts[:title] || spec[:title], :client => client, :environment => opts[:environment]))
27
27
 
28
- begin
29
- migrate_datasets(spec, opts.merge(project: project, client: client))
30
- load(p, spec)
31
- migrate_metrics(p, spec[:metrics] || [])
32
- migrate_reports(p, spec[:reports] || [])
33
- migrate_dashboards(p, spec[:dashboards] || [])
34
- execute_tests(p, spec[:assert_tests] || [])
35
- project
36
- end
28
+ maqls = migrate_datasets(spec, opts.merge(project: project, client: client))
29
+ load(p, spec)
30
+ migrate_metrics(p, spec[:metrics] || [])
31
+ migrate_reports(p, spec[:reports] || [])
32
+ migrate_dashboards(p, spec[:dashboards] || [])
33
+ execute_tests(p, spec[:assert_tests] || [])
34
+ opts[:execute_ca_scripts] ? project : maqls.find { |maql| maql.key?('maqlDdlChunks') }
37
35
  end
38
36
 
39
37
  def migrate_datasets(spec, opts = {})
@@ -41,17 +39,18 @@ module GoodData
41
39
  dry_run = opts[:dry_run]
42
40
  replacements = opts['maql_replacements'] || opts[:maql_replacements] || {}
43
41
 
44
- client, project = GoodData.get_client_and_project(opts)
42
+ _, project = GoodData.get_client_and_project(opts)
45
43
 
46
44
  bp = ProjectBlueprint.new(spec)
45
+ response = project.maql_diff(blueprint: bp, params: [:includeGrain])
47
46
 
48
- uri = "/gdc/projects/#{project.pid}/model/diff?includeGrain=true"
49
- result = client.post(uri, bp.to_wire)
50
- response = client.poll_on_code(result['asyncTask']['link']['poll'])
51
-
47
+ GoodData.logger.debug("projectModelDiff") { response.pretty_inspect }
52
48
  chunks = response['projectModelDiff']['updateScripts']
53
49
  return [] if chunks.empty?
54
50
 
51
+ ca_maql = response['projectModelDiff']['computedAttributesScript'] if response['projectModelDiff']['computedAttributesScript']
52
+ ca_chunks = ca_maql && ca_maql['maqlDdlChunks']
53
+
55
54
  maqls = pick_correct_chunks(chunks, opts)
56
55
  replaced_maqls = apply_replacements_on_maql(maqls, replacements)
57
56
 
@@ -59,19 +58,32 @@ module GoodData
59
58
  errors = []
60
59
  replaced_maqls.each do |replaced_maql_chunks|
61
60
  begin
62
- replaced_maql_chunks['updateScript']['maqlDdlChunks'].each { |chunk| project.execute_maql(chunk) }
61
+ replaced_maql_chunks['updateScript']['maqlDdlChunks'].each do |chunk|
62
+ GoodData.logger.debug(chunk)
63
+ project.execute_maql(chunk)
64
+ end
63
65
  rescue => e
64
66
  puts "Error occured when executing MAQL, project: \"#{project.title}\" reason: \"#{e.message}\", chunks: #{replaced_maql_chunks.inspect}"
65
67
  errors << e
66
68
  next
67
69
  end
68
70
  end
71
+
72
+ if ca_chunks && opts[:execute_ca_scripts]
73
+ begin
74
+ ca_chunks.each { |chunk| project.execute_maql(chunk) }
75
+ rescue => e
76
+ puts "Error occured when executing MAQL, project: \"#{project.title}\" reason: \"#{e.message}\", chunks: #{ca_chunks.inspect}"
77
+ errors << e
78
+ end
79
+ end
80
+
69
81
  if (!errors.empty?) && (errors.length == replaced_maqls.length)
70
- messages = errors.map { |e| GoodData::Helpers.interpolate_error_messages(e.data['wTaskStatus']['messages']) }
82
+ messages = errors.map { |err| GoodData::Helpers.interpolate_error_messages(err.data['wTaskStatus']['messages']) }
71
83
  fail "Unable to migrate LDM, reason(s): \n #{messages.join("\n")}"
72
84
  end
73
85
  end
74
- replaced_maqls
86
+ replaced_maqls + (ca_maql ? [ca_maql] : [])
75
87
  end
76
88
 
77
89
  def migrate_reports(project, spec)
@@ -111,7 +123,9 @@ module GoodData
111
123
  end
112
124
 
113
125
  def pick_correct_chunks(chunks, opts = {})
126
+ GoodData.logger.debug("update_preference") { opts[:update_preference].pretty_inspect }
114
127
  preference = GoodData::Helpers.symbolize_keys(opts[:update_preference] || {})
128
+ preference = Hash[preference.map { |k, v| [k, GoodData::Helpers.to_boolean(v)] }]
115
129
 
116
130
  # first is cascadeDrops, second is preserveData
117
131
  rules = [
@@ -126,10 +140,19 @@ module GoodData
126
140
  end
127
141
 
128
142
  stuff = stuff.map do |chunk|
129
- { cascade_drops: chunk['updateScript']['cascadeDrops'], preserve_data: chunk['updateScript']['preserveData'], maql: chunk['updateScript']['maqlDdlChunks'], orig: chunk }
143
+ { cascade_drops: chunk['updateScript']['cascadeDrops'],
144
+ preserve_data: chunk['updateScript']['preserveData'],
145
+ maql: chunk['updateScript']['maqlDdlChunks'],
146
+ orig: chunk }
130
147
  end
131
148
 
132
- results_from_api = GoodData::Helpers.join(rules, stuff, [:cascade_drops, :preserve_data], [:cascade_drops, :preserve_data], inner: true).sort_by { |l| l[:priority] } || []
149
+ results_from_api = GoodData::Helpers.join(
150
+ rules,
151
+ stuff,
152
+ [:cascade_drops, :preserve_data],
153
+ [:cascade_drops, :preserve_data],
154
+ inner: true
155
+ ).sort_by { |l| l[:priority] } || []
133
156
 
134
157
  if preference.empty?
135
158
  [results_from_api.first[:orig]]