gooddata 0.6.50 → 0.6.51

data/lib/gooddata/lcm/actions/purge_clients.rb

@@ -55,10 +55,10 @@ module GoodData
               project = segment_client.project
               res = {
                 client_id: segment_client.client_id,
-                project: project.pid
+                project: project && project.pid
               }
 
-              if project.deleted?
+              if project.nil? || project.deleted?
                 segment_client.delete
                 res[:status] = 'purged'
               else

data/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb

@@ -0,0 +1,81 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_action'
+
+module GoodData
+  module LCM2
+    class SynchronizeETLsInSegment < BaseAction
+      DESCRIPTION = 'Synchronize ETLs (CC/Ruby) In Segment'
+
+      PARAMS = define_params(self) do
+        description 'Client Used for Connecting to GD'
+        param :gdc_gd_client, instance_of(Type::GdClientType), required: true
+      end
+
+      # will be updated later based on the way etl synchronization
+      RESULT_HEADER = [
+        :segment,
+        :master_project,
+        :client_id,
+        :client_project,
+        :status
+      ]
+
+      class << self
+        def call(params)
+          # Check if all required parameters were passed
+          BaseAction.check_params(PARAMS, params)
+
+          client = params.gdc_gd_client
+          domain_name = params.organization || params.domain
+          domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
+          synchronize_segments = params.synchronize.group_by do |info|
+            info[:segment_id]
+          end
+
+          results = synchronize_segments.flat_map do |segment_id, synchronize|
+            segment = domain.segments(segment_id)
+            res = segment.synchronize_processes(
+              synchronize.flat_map do |info|
+                info[:to].flat_map do |to|
+                  to[:pid]
+                end
+              end
+            )
+
+            res = GoodData::Helpers.symbolize_keys(res)
+            res[:syncedResult][:clients].flat_map do |item|
+              item = item[:client]
+              {
+                segment: segment_id,
+                master_project: segment.master_project_id,
+                client_id: item[:id],
+                client_project: item[:project].split('/').last,
+                status: 'ok'
+              }
+            end
+          end
+
+          params.synchronize.each do |info|
+            to_projects = info.to
+            to_projects.each do |entry|
+              pid = entry[:pid]
+              to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
+              to_project.schedules.each do |schedule|
+                schedule.update_params(params.additional_params || {})
+                schedule.update_hidden_params(params.additional_hidden_params || {})
+                schedule.save
+              end
+            end
+          end
+
+          results
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/lcm/actions/synchronize_label_types.rb

@@ -37,8 +37,8 @@ module GoodData
             to = info.to
 
             from_project = development_client.projects(from) || fail("Invalid 'from' project specified - '#{from}'")
-            to_projects = to.map do |pid|
-              client.projects(pid)
+            to_projects = to.map do |project|
+              client.projects(project.pid)
             end
 
             GoodData::LCM.transfer_label_types(from_project, to_projects)

data/lib/gooddata/lcm/actions/synchronize_processes.rb

@@ -14,6 +14,8 @@ module GoodData
       PARAMS = define_params(self) do
         description 'Client Used for Connecting to GD'
         param :gdc_gd_client, instance_of(Type::GdClientType), required: true
+        description 'Uri of the source output stage. It must be in the same domain as the target project.'
+        param :ads_output_stage_uri, instance_of(Type::StringType), required: false
       end
 
       RESULT_HEADER = [
@@ -45,7 +47,7 @@ module GoodData
               to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
 
               params.gdc_logger.info "Transferring processes, from project: '#{from.title}', PID: '#{from.pid}', to project: '#{to_project.title}', PID: '#{to_project.pid}'"
-              GoodData::Project.transfer_processes(from, to_project)
+              GoodData::Project.transfer_processes(from, to_project, ads_output_stage_uri: params.ads_output_stage_uri)
 
               to_project.add.output_stage.client_id = client_id if client_id && to_project.add.output_stage
 

data/lib/gooddata/lcm/data/create_lcm_release.sql.erb

@@ -1,5 +1,6 @@
 CREATE TABLE IF NOT EXISTS "<%= table_name || 'LCM_RELEASE' %>" (
   segment_id        VARCHAR(255) NOT NULL,
   master_project_id VARCHAR(255) NOT NULL,
-  version           INTEGER NOT NULL
+  version           INTEGER NOT NULL,
+  maql_ddl          VARCHAR(65000)
 );

data/lib/gooddata/lcm/helpers/check_helper.rb

@@ -12,7 +12,7 @@ module GoodData
       class << self
         def check_params(specification, params)
           specification.keys.each do |param_name|
-            value = params[param_name]
+            value = params.send(param_name)
             type = specification[param_name][:type]
 
             if value.nil?

data/lib/gooddata/lcm/lcm.rb

@@ -146,6 +146,15 @@ module GoodData
             end
           end
         end
+
+        # User groups must be migrated after dashboards
+        puts 'Migrating User Groups'
+        domain.clients.peach do |c|
+          segment = c.segment
+          segment_master = segment.master_project
+          project = c.project
+          GoodData::Project.transfer_user_groups(segment_master, project)
+        end
       end
 
       def transfer_label_types(source_project, targets)
@@ -251,30 +260,39 @@ module GoodData
 
         # Generate transfer table
         drill_paths = attributes.map do |attribute|
-          [attribute.meta['identifier'], attribute.content['drillDownStepAttributeDF']]
+          drill_label_uri = attribute.content['drillDownStepAttributeDF']
+          if drill_label_uri
+            drill_label = source_project.labels(drill_label_uri)
+            [attribute.meta['identifier'], drill_label.identifier]
+          else
+            []
+          end
         end
         transfer = Hash[*drill_paths.flatten].compact
 
         # Transfer to target projects
         targets.peach do |target|
-          transfer.peach do |identifier, drill_path|
-            uri = GoodData::MdObject.identifier_to_uri({ project: target, client: target.client }, identifier)
-            next unless uri
+          transfer.peach do |attr_identifier, drill_path_identifier|
+            attr_uri = GoodData::MdObject.identifier_to_uri({ project: target, client: target.client }, attr_identifier)
+            next unless attr_uri
 
-            obj = GoodData::MdObject[uri, { project: target, client: target.client }]
+            attribute = GoodData::MdObject[attr_uri, project: target, client: target.client]
 
-            if obj
-              if !obj.content['drillDownStepAttributeDF'] || obj.content['drillDownStepAttributeDF'] != drill_path
+            if attribute
+              drill_path = GoodData::MdObject.identifier_to_uri({ project: target, client: target.client }, drill_path_identifier)
+              next unless drill_path
+
+              if !attribute.content['drillDownStepAttributeDF'] || attribute.content['drillDownStepAttributeDF'] != drill_path
                 semaphore.synchronize do
-                  GoodData.logger.info "Updating drill path of #{identifier} -> #{drill_path} in '#{target.title}'"
+                  GoodData.logger.info "Updating drill path of #{attr_identifier} -> #{drill_path} in '#{target.title}'"
                 end
 
-                obj.content['drillDownStepAttributeDF'] = drill_path
-                obj.save
+                attribute.content['drillDownStepAttributeDF'] = drill_path
+                attribute.save
               end
             else
               semaphore.synchronize do
-                GoodData.logger.warn "Unable to find #{identifier} in '#{target.title}'"
+                GoodData.logger.warn "Unable to find #{attr_identifier} in '#{target.title}'"
               end
             end
 

data/lib/gooddata/lcm/lcm2.rb

@@ -16,8 +16,16 @@ module GoodData
       def method_missing(name, *_args)
         key = name.to_s.downcase.to_sym
 
-        if key?(key)
-          self[key]
+        value = nil
+        keys.each do |k|
+          if k.to_s.downcase.to_sym == key
+            value = self[k]
+            break
+          end
+        end
+
+        if value
+          value
         else
           begin
             super
@@ -27,6 +35,16 @@ module GoodData
         end
       end
 
+      def key?(key)
+        return true if super
+
+        keys.each do |k|
+          return true if k.to_s.downcase.to_sym == key.to_s.downcase.to_sym
+        end
+
+        false
+      end
+
       def respond_to_missing?(name, *_args)
         key = name.to_s.downcase.to_sym
         key?(key)
@@ -68,6 +86,7 @@ module GoodData
         EnsureTechnicalUsersProject,
         SynchronizeLdm,
         SynchronizeMeta,
+        SynchronizeLabelTypes,
         SynchronizeAttributeDrillpath,
         SynchronizeProcesses,
         SynchronizeSchedules,
@@ -86,10 +105,9 @@ module GoodData
         ProvisionClients,
         EnsureTechnicalUsersDomain,
         EnsureTechnicalUsersProject,
-        EnsureTitles,
         SynchronizeAttributeDrillpath,
-        SynchronizeProcesses,
-        SynchronizeSchedules
+        SynchronizeETLsInSegment,
+        SynchronizeColorPalette
       ],
 
       rollout: [
@@ -102,9 +120,10 @@ module GoodData
         SynchronizeLdm,
         # SynchronizeLabelTypes,
         SynchronizeAttributeDrillpath,
-        SynchronizeProcesses,
-        SynchronizeSchedules,
-        SynchronizeClients
+        ApplyCustomMaql,
+        SynchronizeColorPalette,
+        SynchronizeClients,
+        SynchronizeETLsInSegment
       ]
     }
 
@@ -122,9 +141,9 @@ module GoodData
           res = SmartHash.new
           params.each_pair do |k, v|
             if v.is_a?(Hash) || v.is_a?(Array)
-              res[k.downcase] = convert_to_smart_hash(v)
+              res[k] = convert_to_smart_hash(v)
             else
-              res[k.downcase] = v
+              res[k] = v
             end
           end
           res
@@ -138,7 +157,13 @@ module GoodData
       end
 
       def get_mode_actions(mode)
-        MODES[mode.to_sym] || fail("Invalid mode specified '#{mode}', supported modes are: '#{MODE_NAMES.join(', ')}'")
+        mode = mode.to_sym
+        actions = MODES[mode]
+        if mode == :generic_lifecycle
+          []
+        else
+          actions || fail("Invalid mode specified '#{mode}', supported modes are: '#{MODE_NAMES.join(', ')}'")
+        end
       end
 
       def print_action_names(mode, actions)
@@ -166,12 +191,13 @@ module GoodData
         keys = if action.const_defined?('RESULT_HEADER')
                  action.const_get('RESULT_HEADER')
                else
+                 GoodData.logger.warn("Action #{action.name} does not have RESULT_HEADERS, inferring headers from results.")
                  (messages.first && messages.first.keys) || []
                end
 
         headings = keys.map(&:upcase)
 
-        rows = messages.map do |message|
+        rows = messages && messages.map do |message|
           row = []
           keys.each do |heading|
             row << message[heading]
@@ -179,6 +205,8 @@ module GoodData
           row
         end
 
+        rows ||= []
+
         table = Terminal::Table.new :title => title, :headings => headings do |t|
           rows.each_with_index do |row, index|
             t << row
@@ -202,6 +230,17 @@ module GoodData
 
         # Get actions for mode specified
         actions = get_mode_actions(mode)
+        if params.actions
+          actions = params.actions.map do |action|
+            "GoodData::LCM2::#{action}".split('::').inject(Object) do |o, c|
+              begin
+                o.const_get(c)
+              rescue NameError
+                fail NameError, "Cannot find action 'GoodData::LCM2::#{action}'"
+              end
+            end
+          end
+        end
 
         # Print name of actions to be performed for debug purposes
         print_action_names(mode, actions)
@@ -210,12 +249,34 @@ module GoodData
 
         new_params = params
 
+        fail_early = if params.key?(:fail_early)
+                       params.fail_early.to_b
+                     else
+                       true
+                     end
+
+        strict_mode = if params.key?(:strict)
+                        params.strict.to_b
+                      else
+                        true
+                      end
+
         # Run actions
-        results = actions.map do |action|
+        errors = []
+        results = []
+        actions.each do |action|
           puts
 
           # Invoke action
-          out = action.send(:call, params)
+          begin
+            out = action.send(:call, params)
+          rescue => e
+            errors << {
+              action: action,
+              err: e
+            }
+            break if fail_early
+          end
 
           # Handle output results and params
           res = out.is_a?(Array) ? out : out[:results]
@@ -229,10 +290,13 @@ module GoodData
           puts
           print_action_result(action, res)
 
-          # Return result for final summary
-          res
+          # Store result for final summary
+          results << res
         end
 
+        # Fail whole execution if there is any failed action
+        fail(JSON.pretty_generate(errors)) if strict_mode && errors.any?
+
         if actions.length > 1
           puts
           puts 'SUMMARY'
@@ -244,13 +308,11 @@ module GoodData
 
         brick_results = {}
         actions.each_with_index do |action, index|
-          brick_results[action.class.short_name] = results[index]
+          brick_results[action.short_name] = results[index]
         end
 
         {
-          actions: actions.map do |action|
-            action.class.short_name
-          end,
+          actions: actions.map(&:short_name),
           results: brick_results,
           params: params
         }

data/lib/gooddata/models/domain.rb

@@ -241,19 +241,30 @@ module GoodData
 
         list.pmapcat do |user|
           begin
-            user_data = user.to_hash.tap { |uh| uh[:login].downcase }
+            user_data = user.to_hash.tap { |uh| uh[:login].downcase! }
             domain_user = domain_users_cache[user_data[:login]]
+            user_login = user_data[:login]
             if !domain_user
               added_user = domain.add_user(user_data, opts)
+              GoodData.logger.info("Added new user=#{user_login} to domain=#{default_domain_name}.")
               [{ type: :successful, :action => :user_added_to_domain, user: added_user }]
             else
+              domain_user_data = domain_user.to_hash
               fields_to_check = opts[:fields_to_check] || user_data.keys
               diff = GoodData::Helpers.diff([domain_user.to_hash], [user_data], key: :login, fields: fields_to_check)
               next [] if diff[:changed].empty?
               updated_user = domain.update_user(domain_user.to_hash.merge(user_data.compact), opts)
+              GoodData.logger.debug "Updated user=#{user_login} from old properties \
+(email=#{domain_user_data[:email]}, sso_provider=#{domain_user_data[:sso_provider]}) \
+to new properties (email=#{user_data[:email]}, sso_provider=#{user_data[:sso_provider]}) in domain=#{default_domain_name}."
               [{ type: :successful, :action => :user_changed_in_domain, user: updated_user }]
             end
           rescue RuntimeError => e
+            if !domain_user
+              GoodData.logger.error("Failed to add user=#{user_login} to domain=#{default_domain_name}. Error: #{e.message}")
+            else
+              GoodData.logger.error("Failed to update user=#{user_login} in domain=#{default_domain_name}. Error: #{e.message}")
+            end
             [{ type: :failed, :user => user, message: e }]
           end
         end
@@ -431,6 +442,10 @@ module GoodData
     alias_method :add_clients_settings, :update_clients_settings
 
     def update_clients(data, options = {})
+      if options[:delete_extra] && options[:delete_extra_in_segments]
+        fail 'Options delete_extra and delete_extra_in_segments are mutually exclusive.'
+      end
+
       delete_projects = options[:delete_projects] == false ? false : true
       payload = data.map do |datum|
         {
@@ -444,6 +459,12 @@ module GoodData
       end
       if options[:delete_extra] == true
         res = client.post(segments_uri + '/updateClients?deleteExtra=true', updateClients: { items: payload })
+      elsif options[:delete_extra_in_segments]
+        segments_to_delete_in = options[:delete_extra_in_segments]
+          .map { |segment| CGI.escape(segment) }
+          .join(',')
+        uri = segments_uri + "/updateClients?deleteExtraInSegments=#{segments_to_delete_in}"
+        res = client.post(uri, updateClients: { items: payload })
       else
         res = client.post(segments_uri + '/updateClients', updateClients: { items: payload })
       end