gooddata 0.6.50 → 0.6.51

data/lib/gooddata/models/metadata.rb

@@ -87,17 +87,22 @@ module GoodData
           from_attribute, to_attribute = mapping.find { |k, _| k.uri == a_uri }
           vals = from_attribute.values_for(id)
           labels = to_attribute.labels
-          results = labels.to_enum.mapcat do |l|
-            vals.map do |v|
-              begin
-                l.find_value_uri(v)
-              rescue
-                nil
+
+          result = nil
+          catch :found_value do
+            labels.each do |l|
+              vals.each do |v|
+                throw :found_value if result
+                result = begin
+                           l.find_value_uri(v)
+                         rescue
+                           nil
+                         end
               end
             end
           end
-          fail "Unable to find replacement for #{a_uri}" if results.compact.empty?
-          [a_uri, id, results.compact.first]
+          fail "Unable to find replacement for #{a_uri}" unless result
+          [a_uri, id, result]
         end
         replaceable_vals.map { |a, id, r| ["#{a}/elements?id=#{id}", r] }
       end

data/lib/gooddata/models/metadata/attribute.rb

@@ -116,7 +116,7 @@ module GoodData
     # @return [Array] list of values for certain element. Returned in the same order as is the order of labels
     def values_for(element_id)
       # element_id = element_id.is_a?(String) ? element_id.match(/\?id=(\d)/)[1] : element_id
-      labels.map do |label|
+      labels.pmap do |label|
         label.find_element_value(element_id)
       end
     end

data/lib/gooddata/models/metadata/report_definition.rb

@@ -251,6 +251,7 @@ module GoodData
       x = GoodData::MdObject.replace_quoted(self, mapping)
       x = GoodData::MdObject.replace_bracketed(x, mapping)
       vals = GoodData::MdObject.find_replaceable_values(self, mapping)
+      GoodData::MdObject.replace_quoted(x, vals)
       GoodData::MdObject.replace_bracketed(x, vals)
     end
 

data/lib/gooddata/models/profile.rb

@@ -60,7 +60,7 @@ module GoodData
         return id if id.instance_of?(GoodData::Profile) || id.respond_to?(:profile?) && id.profile?
 
         if id.to_s !~ %r{^(\/gdc\/account\/profile\/)?[a-zA-Z\d]+$}
-          fail(ArgumentError, 'wrong type of argument. Should be either project ID or path')
+          fail(ArgumentError, 'wrong type of argument. Should be either profile ID or path')
         end
 
         id = id.match(/[a-zA-Z\d]+$/)[0] if id =~ %r{/}

data/lib/gooddata/models/project.rb

@@ -10,6 +10,7 @@ require 'fileutils'
 require 'multi_json'
 require 'pmap'
 require 'zip'
+require 'net/smtp'
 
 require_relative '../exceptions/no_project_error'
 
@@ -23,6 +24,7 @@ require_relative '../mixins/uri_getter'
 
 require_relative 'membership'
 require_relative 'process'
+require_relative 'project_log_formatter'
 require_relative 'project_role'
 require_relative 'blueprint/blueprint'
 
@@ -78,9 +80,7 @@ module GoodData
         if id == :all
           Project.all({ client: GoodData.connection }.merge(opts))
         else
-          if id.to_s !~ %r{^(\/gdc\/(projects|md)\/)?[a-zA-Z\d]+$}
-            fail(ArgumentError, 'wrong type of argument. Should be either project ID or path')
-          end
+          fail(ArgumentError, 'wrong type of argument. Should be either project ID or path') unless project_id_or_path?(id)
 
           id = id.match(/[a-zA-Z\d]+$/)[0] if id =~ %r{/}
 
@@ -90,6 +90,10 @@ module GoodData
         end
       end
 
+      def project_id_or_path?(id)
+        id.to_s =~ %r{^(\/gdc\/(projects|md)\/)?[a-zA-Z\d]+$}
+      end
+
       # Clones project along with etl and schedules
       #
       # @param project [Project] Project to be cloned from
@@ -184,10 +188,40 @@ module GoodData
         }
       end
 
+      def transfer_output_stage(from_project, to_project, options)
+        if from_project.processes.any? { |p| p.type == :dataload }
+          if to_project.processes.any? { |p| p.type == :dataload }
+            to_project.add.output_stage.schema = from_project.add.output_stage.schema
+            to_project.add.output_stage.output_stage_prefix = from_project.add.output_stage.output_stage_prefix
+            to_project.add.output_stage.save
+          else
+            from_prj_output_stage = from_project.add.output_stage
+            from_server = from_project.client.connection.server.url
+            to_server = to_project.client.connection.server.url
+            if from_server != to_server && options[:ads_output_stage_uri].nil?
+              raise "Cannot transfer output stage from #{from_server} to #{to_server}. " \
+                    'It is not possible to transfer output stages between ' \
+                    'different domains. Please specify an address of an output ' \
+                    'stage that is in the same domain as the target project ' \
+                    'using the "ads_output_stage_uri" parameter.'
+            end
+
+            to_project.add.output_stage = GoodData::AdsOutputStage.create(
+              client: to_project.client,
+              ads: options[:ads_output_stage_uri] || from_prj_output_stage.schema,
+              client_id: from_prj_output_stage.client_id,
+              output_stage_prefix: from_prj_output_stage.output_stage_prefix,
+              project: to_project
+            )
+          end
+        end
+      end
+
       # Clones project along with etl and schedules.
       #
       # @param client [GoodData::Rest::Client] GoodData client to be used for connection
       # @param from_project [GoodData::Project | GoodData::Segment | GoodData:Client | String] Object to be cloned from. Can be either segment in which case we take the master, client in which case we take its project, string in which case we treat is as an project object or directly project
+      # @param to_project [GoodData::Project | GoodData::Segment | GoodData:Client | String]
       def transfer_etl(client, from_project, to_project)
         from_project = case from_project
                        when GoodData::Client
@@ -210,7 +244,12 @@ module GoodData
         transfer_schedules(from_project, to_project)
       end
 
-      def transfer_processes(from_project, to_project)
+      # @param from_project The source project
+      # @param to_project The target project
+      # @param options Optional parameters
+      # @option ads_output_stage_uri Uri of the source output stage. It must be in the same domain as the target project.
+      def transfer_processes(from_project, to_project, options = {})
+        options = GoodData::Helpers.symbolize_keys(options)
         to_project_processes = to_project.processes
         from_project.processes.uniq(&:name).each do |process|
           fail "The process name #{process.name} must be unique in transfered project #{to_project}" if to_project_processes.count { |p| p.name == process.name } > 1
@@ -232,16 +271,8 @@ module GoodData
           end
         end
 
-        if from_project.processes.any? { |p| p.type == :dataload }
-          if to_project_processes.any? { |p| p.type == :dataload }
-            to_project.add.output_stage.schema = from_project.add.output_stage.schema
-            to_project.add.output_stage.output_stage_prefix = from_project.add.output_stage.output_stage_prefix
-            to_project.add.output_stage.save
-          else
-            from_prj_output_stage = from_project.add.output_stage
-            to_project.add.output_stage = GoodData::AdsOutputStage.create(client: to_project.client, ads: from_prj_output_stage.schema, client_id: from_prj_output_stage.client_id, output_stage_prefix: from_prj_output_stage.output_stage_prefix, project: to_project)
-          end
-        end
+        transfer_output_stage(from_project, to_project, options)
+
         res = (from_project.processes + to_project.processes).map { |p| [p, p.name, p.type] }
         res.group_by { |x| [x[1], x[2]] }
           .select { |_, procs| procs.length == 1 && procs[2] != :dataload }
@@ -249,6 +280,27 @@ module GoodData
           .peach(&:delete)
       end
 
+      def transfer_user_groups(from_project, to_project)
+        from_project.user_groups.each do |ug|
+          # migrate groups
+          new_group = to_project.user_groups.select { |group| group.name == ug.name }.first
+          new_group ||= UserGroup.create(:name => ug.name, :description => ug.description, :project => to_project)
+          new_group.project = to_project
+          new_group.description = ug.description
+          new_group.save
+          # migrate dashboard "grantees"
+          dashboards = from_project.dashboards
+          dashboards.each do |dashboard|
+            new_dashboard = to_project.dashboards.select { |dash| dash.title == dashboard.title }.first
+            next unless new_dashboard
+            grantee = dashboard.grantees['granteeURIs']['items'].select { |item| item['aclEntryURI']['grantee'].split('/').last == ug.links['self'].split('/').last }.first
+            next unless grantee
+            permission = grantee['aclEntryURI']['permission']
+            new_dashboard.grant(:member => new_group, :permission => permission)
+          end
+        end
+      end
+
       # Clones project along with etl and schedules.
       #
       # @param client [GoodData::Rest::Client] GoodData client to be used for connection
@@ -680,7 +732,7 @@ module GoodData
     def delete_all_data(options = {})
       return false unless options[:force]
       begin
-        datasets.pmap(&:delete_data)
+        datasets.reject(&:date_dimension?).pmap(&:delete_data)
       rescue MaqlExecutionError => e
         # This is here so that we do not throw out exceptions on synchornizing date dimensions
         # Currently there is no reliable way how to tell it is a date dimension
@@ -916,6 +968,7 @@ module GoodData
     def initialize(json)
       super
       @json = json
+      @log_formatter = GoodData::ProjectLogFormatter.new(self)
     end
 
     # Invites new user to project
@@ -1252,14 +1305,16 @@ module GoodData
               GoodData.logger.info "Saving #{new_item.uri}"
               new_item.save
             end
+          else
+            GoodData.logger.info "Ignore #{item.uri}"
           end
         end
       end
 
       GoodData.logger.info 'Replacing hidden metrics'
-      local_metrics = rds.pmapcat { |rd| rd.using('metric') }.select { |m| m['deprecated'] == '1' }
+      local_metrics = mapping.map { |a, _| a }.pmapcat { |a| a.usedby('metric') }.select { |m| m['deprecated'] == '1' }.map { |m| m['link'] }.uniq
       puts "Found #{local_metrics.count} metrics"
-      local_metrics.pmap { |m| metrics(m['link']) }.peach do |item|
+      local_metrics.pmap { |m| metrics(m) }.peach do |item|
         new_item = item.replace(mapping)
         if new_item.json != item.json
           if dry_run
@@ -1268,6 +1323,8 @@ module GoodData
             GoodData.logger.info "Saving #{new_item.uri}"
             new_item.save
           end
+        else
+          GoodData.logger.info "Ignore #{item.uri}"
         end
       end
 
@@ -1466,7 +1523,7 @@ module GoodData
       whitelisted_new_users, whitelisted_users = whitelist_users(new_users.map(&:to_hash), users_list, options[:whitelists])
 
       # First check that if groups are provided we have them set up
-      check_groups(new_users.map(&:to_hash).flat_map { |u| u[:user_group] || [] }.uniq)
+      check_groups(new_users.map(&:to_hash).flat_map { |u| u[:user_group] || [] }.uniq, options)
 
       # conform the role on list of new users so we can diff them with the users coming from the project
       diffable_new_with_default_role = whitelisted_new_users.map do |u|
@@ -1477,12 +1534,12 @@ module GoodData
       intermediate_new = diffable_new_with_default_role.map do |u|
         u[:role] = u[:role].map do |r|
           role = get_role(r, role_list)
-          role && role.uri
+          role ? role.uri : r
         end
 
         u[:role_title] = u[:role].map do |r|
           role = get_role(r, role_list)
-          role && role.title
+          role ? role.title : r
         end
 
         if u[:role].all?(&:nil?)
@@ -1515,29 +1572,37 @@ module GoodData
       return diff_results if options[:dry_run]
 
       # Create new users
-      u = diff[:added].map { |x| { user: x, role: x[:role] } }
-
       results = []
       GoodData.logger.warn("Creating #{diff[:added].count} users in project (#{pid})")
-      results.concat(create_users(u, roles: role_list, project_users: whitelisted_users))
+      to_create = diff[:added].map { |x| { user: x, role: x[:role] } }
+      created_users_result = create_users(to_create, roles: role_list, project_users: whitelisted_users)
+      @log_formatter.log_created_users(created_users_result, diff[:added])
+      results.concat(created_users_result)
+      send_mail_to_new_users(diff[:added], options[:email_options]) if options[:email_options] && !options[:email_options].empty? && !diff[:added].empty?
 
       # # Update existing users
       GoodData.logger.warn("Updating #{diff[:changed].count} users in project (#{pid})")
-      list = diff[:changed].map { |x| { user: x[:new_obj], role: x[:new_obj][:role] || x[:new_obj][:roles] } }
-      results.concat(set_users_roles(list, roles: role_list, project_users: whitelisted_users))
+      to_update = diff[:changed].map { |x| { user: x[:new_obj], role: x[:new_obj][:role] || x[:new_obj][:roles] } }
+      updated_users_result = set_users_roles(to_update, roles: role_list, project_users: whitelisted_users)
+      @log_formatter.log_updated_users(updated_users_result, diff[:changed], role_list)
+      results.concat(updated_users_result)
 
       # Remove old users
-      to_remove = diff[:removed].reject { |user| user[:status] == 'DISABLED' || user[:status] == :disabled }
-      GoodData.logger.warn("Disabling #{to_remove.count} users from project (#{pid})")
-      results.concat(disable_users(to_remove, roles: role_list, project_users: whitelisted_users))
+      to_disable = diff[:removed].reject { |user| user[:status] == 'DISABLED' || user[:status] == :disabled }
+      GoodData.logger.warn("Disabling #{to_disable.count} users from project (#{pid})")
+      disabled_users_result = disable_users(to_disable, roles: role_list, project_users: whitelisted_users)
+      @log_formatter.log_disabled_users(disabled_users_result)
+      results.concat(disabled_users_result)
 
       # Remove old users completely
       if options[:remove_users_from_project]
-        to_remove = (to_remove + users(disabled: true).to_a).map(&:to_hash).uniq do |user|
+        to_remove = (to_disable + users(disabled: true).to_a).map(&:to_hash).uniq do |user|
           user[:uri]
         end
         GoodData.logger.warn("Removing #{to_remove.count} users from project (#{pid})")
-        results.concat(remove_users(to_remove))
+        removed_users_result = remove_users(to_remove)
+        @log_formatter.log_removed_users(removed_users_result)
+        results.concat(removed_users_result)
       end
 
       # reassign to groups
@@ -1551,9 +1616,9 @@ module GoodData
           a
         end
         mappings.group_by { |_, g| g }.each do |g, mapping|
-          # find group + set users
-          # CARE YOU DO NOT KNOW URI
-          user_groups(g).set_members(mapping.map { |user, _| user }.map { |login| users_lookup[login] && users_lookup[login].uri })
+          remote_users = mapping.map { |user, _| user }.map { |login| users_lookup[login] && users_lookup[login].uri }.reject(&:nil?)
+          next if remote_users.empty?
+          user_groups(g).set_members(remote_users)
         end
         mentioned_groups = mappings.map(&:last).uniq
         groups_to_cleanup = user_groups.reject { |g| mentioned_groups.include?(g.name) }
@@ -1589,15 +1654,21 @@ module GoodData
           client.delete(url)
           [{ type: :successful, operation: :user_deleted_from_project, user: u }]
         rescue => e
-          [{ type: :failed, message: e.message }]
+          [{ type: :failed, message: e.message, user: u }]
         end
       end
     end
 
-    def check_groups(specified_groups)
+    def check_groups(specified_groups, options = {})
       groups = user_groups.map(&:name)
       missing_groups = specified_groups - groups
-      fail "All groups have to be specified before you try to import users. Groups that are currently in project are #{groups.join(',')} and you asked for #{missing_groups.join(',')}" unless missing_groups.empty?
+      if options[:create_non_existing_user_groups]
+        missing_groups.each do |g|
+          create_group(name: g, description: g)
+        end
+      else
+        fail "All groups have to be specified before you try to import users. Groups that are currently in project are #{groups.join(',')} and you asked for #{missing_groups.join(',')}" unless missing_groups.empty?
+      end
     end
 
     # Update user
@@ -1632,7 +1703,7 @@ module GoodData
           login, roles = resolve_roles(user, desired_roles, options.merge(project_users: project_users, roles: role_list))
           [{ :type => :successful, user: login, roles: roles }]
         rescue => e
-          [{ :type => :failed, :reason => e.message, user: login, roles: roles }]
+          [{ :type => :failed, :reason => e.message, user: user, roles: desired_roles }]
         end
       end
 
@@ -1705,7 +1776,7 @@ module GoodData
       desired_roles = Array(desired_roles)
       roles = desired_roles.map do |role_name|
         role = get_role(role_name, role_list)
-        fail ArgumentError, "Invalid role '#{role_name}' specified for user '#{user.email}'" if role.nil?
+        fail ArgumentError, "Invalid role '#{role_name}' specified for user '#{GoodData::Helpers.last_uri_part(user)}'" if role.nil?
         role.uri
       end
       [user, roles]
@@ -1754,6 +1825,59 @@ module GoodData
 
     private
 
+    def send_mail_to_new_users(users, email_options)
+      password = email_options[:email_password]
+      from = email_options[:email_from]
+      raise 'Missing sender email, please specify parameter "email_from"' unless from
+      raise 'Missing authentication password, please specify parameter "email_password"' unless password
+      template = get_email_template(email_options)
+      smtp = Net::SMTP.new('relay1.na.intgdc.com', 25)
+      smtp.enable_starttls OpenSSL::SSL::SSLContext.new("TLSv1_2_client")
+      smtp.start('notifications.gooddata.com', 'gdc', password, :plain)
+      users.each do |user|
+        smtp.send_mail(get_email_body(template, user), from, user[:login])
+      end
+    end
+
+    def get_email_template(options)
+      bucket = options[:email_template_bucket]
+      path = options[:email_template_path]
+      access_key = options[:email_template_access_key]
+      secret_key = options[:email_template_secret_key]
+      raise "Unable to connect to AWS. Parameter \"email_template_bucket\" seems to be empty" unless bucket
+      raise "Unable to connect to AWS. Parameter \"email_template_path\" is missing" unless path
+      raise "Unable to connect to AWS. Parameter \"email_template_access_key\" is missing" unless access_key
+      raise "Unable to connect to AWS. Parameter \"email_template_secret_key\" is missing" unless secret_key
+      args = {
+        access_key_id: access_key,
+        secret_access_key: secret_key,
+        max_retries: 15,
+        http_read_timeout: 120,
+        http_open_timeout: 120
+      }
+
+      server_side_encryption = options['email_server_side_encryption'] || false
+      args['s3_server_side_encryption'] = :aes256 if server_side_encryption
+
+      s3 = AWS::S3.new(args)
+      bucket = s3.buckets[bucket]
+      process_email_template(bucket, path)
+    end
+
+    def process_email_template(bucket, path)
+      type = path.split('/').last.include?('.html') ? 'html' : 'txt'
+      body = bucket.objects[path].read
+      body.prepend("MIME-Version: 1.0\nContent-type: text/html\n") if type == 'html'
+      body
+    end
+
+    def get_email_body(template, user)
+      template.gsub('${name}', "#{user[:first_name]} #{user[:last_name]}")
+              .gsub('${role}', user[:role_title].count == 1 ? user[:role_title].first : user[:role_title].to_s)
+              .gsub('${user_group}', user[:user_group].count == 1 ? user[:user_group].first : user[:user_group].to_s)
+              .gsub('${project}', Project[user[:pid]].title)
+    end
+
     def generate_user_payload(user_uri, status = 'ENABLED', roles_uri = nil)
       payload = {
         'user' => {

data/lib/gooddata/models/project_creator.rb

@@ -49,7 +49,10 @@ module GoodData
           result = client.post(uri, bp.to_wire)
           response = client.poll_on_code(result['asyncTask']['link']['poll'])
 
-          maqls = pick_correct_chunks(response['projectModelDiff']['updateScripts'], opts)
+          chunks = response['projectModelDiff']['updateScripts']
+          return [] if chunks.empty?
+
+          maqls = pick_correct_chunks(chunks, opts)
           replaced_maqls = apply_replacements_on_maql(maqls, replacements)
 
           unless dry_run
@@ -126,14 +129,31 @@ module GoodData
             { cascade_drops: chunk['updateScript']['cascadeDrops'], preserve_data: chunk['updateScript']['preserveData'], maql: chunk['updateScript']['maqlDdlChunks'], orig: chunk }
           end
 
-          results = GoodData::Helpers.join(rules, stuff, [:cascade_drops, :preserve_data], [:cascade_drops, :preserve_data], inner: true).sort_by { |l| l[:priority] } || []
+          results_from_api = GoodData::Helpers.join(rules, stuff, [:cascade_drops, :preserve_data], [:cascade_drops, :preserve_data], inner: true).sort_by { |l| l[:priority] } || []
 
-          preference.each do |k, v|
-            results = results.find_all do |result|
-              result[k] == v
+          if preference.empty?
+            [results_from_api.first[:orig]]
+          else
+            results = results_from_api.dup
+            preference.each do |k, v|
+              results = results.select do |result|
+                result[k] == v
+              end
+            end
+            if results.empty?
+              available_chunks = results_from_api
+                                    .map do |result|
+                                      {
+                                        cascade_drops: result[:cascade_drops],
+                                        preserve_data: result[:preserve_data]
+                                      }
+                                    end
+                                    .map(&:to_s)
+                                    .join(', ')
+              fail "Synchronize LDM cannot proceed. Adjust your update_preferences and try again. Available chunks with preference: #{available_chunks}"
             end
+            results.map { |result| result[:orig] }
           end
-          (preference.empty? ? [results.first].compact : results).map { |result| result[:orig] }
         end
 
         private