gooddata 1.0.0-java

data/lib/gooddata/models/project_creator.rb

@@ -0,0 +1,209 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'project'
+require_relative 'blueprint/project_blueprint'
+
+require 'open-uri'
+
+module GoodData
+  module Model
+    class ProjectCreator
+      class << self
+        def migrate(opts = {})
+          opts = { client: GoodData.connection, execute_ca_scripts: true }.merge(opts)
+          client = opts[:client]
+          fail ArgumentError, 'No :client specified' if client.nil?
+
+          spec = opts[:spec] || fail('You need to provide spec for migration')
+          bp = ProjectBlueprint.new(spec)
+          fail GoodData::ValidationError, "Blueprint is invalid #{bp.validate.inspect}" unless bp.valid?
+          spec = bp.to_hash
+
+          project = opts[:project] || client.create_project(opts.merge(:title => opts[:title] || spec[:title], :client => client, :environment => opts[:environment]))
+
+          maqls = migrate_datasets(spec, opts.merge(project: project, client: client))
+          load(p, spec)
+          migrate_metrics(p, spec[:metrics] || [])
+          migrate_reports(p, spec[:reports] || [])
+          migrate_dashboards(p, spec[:dashboards] || [])
+          execute_tests(p, spec[:assert_tests] || [])
+          opts[:execute_ca_scripts] ? project : maqls.find { |maql| maql.key?('maqlDdlChunks') }
+        end
+
+        def migrate_datasets(spec, opts = {})
+          opts = { client: GoodData.connection }.merge(opts)
+          dry_run = opts[:dry_run]
+          replacements = opts['maql_replacements'] || opts[:maql_replacements] || {}
+
+          _, project = GoodData.get_client_and_project(opts)
+
+          bp = ProjectBlueprint.new(spec)
+          response = project.maql_diff(blueprint: bp, params: [:includeGrain])
+
+          GoodData.logger.debug("projectModelDiff") { response.pretty_inspect }
+          chunks = response['projectModelDiff']['updateScripts']
+          return [] if chunks.empty?
+
+          ca_maql = response['projectModelDiff']['computedAttributesScript'] if response['projectModelDiff']['computedAttributesScript']
+          ca_chunks = ca_maql && ca_maql['maqlDdlChunks']
+
+          maqls = pick_correct_chunks(chunks, opts)
+          replaced_maqls = apply_replacements_on_maql(maqls, replacements)
+
+          unless dry_run
+            errors = []
+            replaced_maqls.each do |replaced_maql_chunks|
+              begin
+                replaced_maql_chunks['updateScript']['maqlDdlChunks'].each do |chunk|
+                  GoodData.logger.debug(chunk)
+                  project.execute_maql(chunk)
+                end
+              rescue => e
+                puts "Error occured when executing MAQL, project: \"#{project.title}\" reason: \"#{e.message}\", chunks: #{replaced_maql_chunks.inspect}"
+                errors << e
+                next
+              end
+            end
+
+            if ca_chunks && opts[:execute_ca_scripts]
+              begin
+                ca_chunks.each { |chunk| project.execute_maql(chunk) }
+              rescue => e
+                puts "Error occured when executing MAQL, project: \"#{project.title}\" reason: \"#{e.message}\", chunks: #{ca_chunks.inspect}"
+                errors << e
+              end
+            end
+
+            if (!errors.empty?) && (errors.length == replaced_maqls.length)
+              messages = errors.map { |err| GoodData::Helpers.interpolate_error_messages(err.data['wTaskStatus']['messages']) }
+              fail "Unable to migrate LDM, reason(s): \n #{messages.join("\n")}"
+            end
+          end
+          replaced_maqls + (ca_maql ? [ca_maql] : [])
+        end
+
+        def migrate_reports(project, spec)
+          spec.each do |report|
+            project.add_report(report)
+          end
+        end
+
+        def migrate_dashboards(project, spec)
+          spec.each do |dash|
+            project.add_dashboard(dash)
+          end
+        end
+
+        def migrate_metrics(project, spec)
+          spec.each do |metric|
+            project.add_metric(metric)
+          end
+        end
+
+        alias_method :migrate_measures, :migrate_metrics
+
+        def load(project, spec)
+          if spec.key?(:uploads) # rubocop:disable Style/GuardClause
+            spec[:uploads].each do |load|
+              schema = GoodData::Model::Schema.new(spec[:datasets].find { |d| d[:name] == load[:dataset] })
+              project.upload(load[:source], schema, load[:mode])
+            end
+          end
+        end
+
+        def execute_tests(_project, spec)
+          spec.each do |assert|
+            result = GoodData::ReportDefinition.execute(assert[:report])
+            fail "Test did not pass. Got #{result.table.inspect}, expected #{assert[:result].inspect}" if result.table != assert[:result]
+          end
+        end
+
+        def pick_correct_chunks(chunks, opts = {})
+          GoodData.logger.debug("update_preference") { opts[:update_preference].pretty_inspect }
+          preference = GoodData::Helpers.symbolize_keys(opts[:update_preference] || {})
+          preference = Hash[preference.map { |k, v| [k, GoodData::Helpers.to_boolean(v)] }]
+
+          # will use new parameters instead of the old ones
+          if preference.empty? || [:allow_cascade_drops, :keep_data].any? { |k| preference.key?(k) }
+            if [:cascade_drops, :preserve_data].any? { |k| preference.key?(k) }
+              fail "Please do not mix old parameters (:cascade_drops, :preserve_data) with the new ones (:allow_cascade_drops, :keep_data)."
+            end
+            preference = { allow_cascade_drops: false, keep_data: true }.merge(preference)
+
+            new_preference = {}
+            new_preference[:cascade_drops] = false unless preference[:allow_cascade_drops]
+            new_preference[:preserve_data] = true if preference[:keep_data]
+            preference = new_preference
+          end
+
+          # first is cascadeDrops, second is preserveData
+          rules = [
+            { priority: 1, cascade_drops: false, preserve_data: true },
+            { priority: 2, cascade_drops: false, preserve_data: false },
+            { priority: 3, cascade_drops: true, preserve_data: true },
+            { priority: 4, cascade_drops: true, preserve_data: false }
+          ]
+
+          stuff = chunks.select do |chunk|
+            chunk['updateScript']['maqlDdlChunks']
+          end
+
+          stuff = stuff.map do |chunk|
+            { cascade_drops: chunk['updateScript']['cascadeDrops'],
+              preserve_data: chunk['updateScript']['preserveData'],
+              maql: chunk['updateScript']['maqlDdlChunks'],
+              orig: chunk }
+          end
+
+          results_from_api = GoodData::Helpers.join(
+            rules,
+            stuff,
+            [:cascade_drops, :preserve_data],
+            [:cascade_drops, :preserve_data],
+            inner: true
+          ).sort_by { |l| l[:priority] } || []
+
+          if preference.empty?
+            [results_from_api.first[:orig]]
+          else
+            results = results_from_api.dup
+            preference.each do |k, v|
+              results = results.select do |result|
+                result[k] == v
+              end
+            end
+            if results.empty?
+              available_chunks = results_from_api
+                                    .map do |result|
+                                      {
+                                        cascade_drops: result[:cascade_drops],
+                                        preserve_data: result[:preserve_data]
+                                      }
+                                    end
+                                    .map(&:to_s)
+                                    .join(', ')
+              fail "Synchronize LDM cannot proceed. Adjust your update_preferences and try again. Available chunks with preference: #{available_chunks}"
+            end
+            results.map { |result| result[:orig] }
+          end
+        end
+
+        private
+
+        def apply_replacements_on_maql(maqls, replacements = {})
+          maqls.map do |maql|
+            GoodData::Helpers.deep_dup(maql).tap do |m|
+              m['updateScript']['maqlDdlChunks'] = m['updateScript']['maqlDdlChunks'].map do |chunk|
+                replacements.reduce(chunk) { |a, (k, v)| a.gsub(Regexp.new(k), v) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/models/project_log_formatter.rb

@@ -0,0 +1,204 @@
+require_relative 'project'
+
+module GoodData
+  class ProjectLogFormatter
+    def initialize(project)
+      @project = project
+      @users_cache = nil
+    end
+
+    # Log created users
+    #
+    # @param created_users [Array<Hash>] collection of created user result, e.g:
+    # [
+    #   {
+    #     type => :successful || :failed,
+    #     user => '/gdc/account/profile/abc@gooddata.com',
+    #     message => error_message,
+    #     reason: error_message
+    #   },
+    #   ...
+    # ]
+    # @param new_users [Array<Hash>] collection of new users to be created
+    #   [
+    #     {
+    #       login => 'xxx@gooddata.com',
+    #       role_title => 'Editor' || 'Admin' || ...
+    #     },
+    #     ...
+    #   ]
+    # @return nil
+    def log_created_users(created_users, new_users)
+      created_users.each do |created_user|
+        user_login = to_user_login(created_user[:user])
+        if created_user[:type] == :successful
+          user_data = new_users.find { |new_user| new_user[:login] == user_login }
+          GoodData.logger.info("Added new user=#{user_login}, roles=#{user_data[:role_title]} to project=#{@project.pid}.")
+        elsif created_user[:type] == :failed
+          error_message = created_user[:message] || created_user[:reason]
+          GoodData.logger.error("Failed to add user=#{user_login} to project=#{@project.pid}. Error: #{error_message}")
+        end
+      end
+    end
+
+    # Log updated users
+    #
+    # @param updated_users [Array<Hash>] collection of updated user result, e.g:
+    #   [
+    #     {
+    #       type => :successful || :failed,
+    #       user => '/gdc/account/profile/abc@gooddata.com',
+    #       message => error_message,
+    #       reason: error_message
+    #     },
+    #     ...
+    #   ]
+    # @param changed_users [Array<Hash>] collection of changed users to be updated
+    #   [
+    #     {
+    #       old_obj: {
+    #         :login => '/gdc/account/profile/abc@gooddata.com',
+    #         :role => '/gdc/projects/clp4z1qw60o0t048tov909b1xi4qztay/roles/5'
+    #       },
+    #       new_obj: {
+    #         :role_title => 'Editor' || 'Admin' || ...
+    #       }
+    #     },
+    #     ...
+    #   ]
+    # @param role_list [Array<ProjectRole>] project roles
+    # @return nil
+    def log_updated_users(updated_users, changed_users, role_list)
+      updated_users.each do |updated_user|
+        user_login = to_user_login(updated_user[:user])
+        if updated_user[:type] == :successful
+          changed_user = changed_users.find { |user| user[:old_obj][:login] == user_login }
+          old_user_data = changed_user[:old_obj]
+          old_role_uris = old_user_data[:role] || old_user_data[:roles]
+          old_role_titles = old_role_uris.map do |old_role_uri|
+            old_role = @project.get_role(old_role_uri, role_list)
+            old_role && old_role.title
+          end
+          new_role_titles = changed_user[:new_obj][:role_title]
+          GoodData.logger.info("Update user=#{user_login} from old_roles=#{old_role_titles} to new_roles=#{new_role_titles} in project=#{@project.pid}.")
+        elsif updated_user[:type] == :failed
+          error_message = updated_user[:message] || updated_user[:reason]
+          GoodData.logger.error("Failed to update user=#{user_login} to project=#{@project.pid}. Error: #{error_message}")
+        end
+      end
+    end
+
+    # Log disabled users
+    #
+    # @param disabled_users [Array<Hash>] collection of disabled user result, e.g:
+    #   [
+    #     {
+    #       type => :successful || :failed,
+    #       user => '/gdc/account/profile/abc@gooddata.com',
+    #       message => error_message,
+    #       reason: error_message
+    #     },
+    #     ...
+    #   ]
+    # @return nil
+    def log_disabled_users(disabled_users)
+      disabled_users.each do |disabled_user|
+        user_login = to_user_login(disabled_user[:user])
+        if disabled_user[:type] == :successful
+          GoodData.logger.warn("Disable user=#{user_login} in project=#{@project.pid}")
+        elsif disabled_user[:type] == :failed
+          error_message = disabled_user[:message] || disabled_user[:reason]
+          GoodData.logger.error("Failed to disable user=#{user_login} in project=#{@project.pid}. Error: #{error_message}")
+        end
+      end
+    end
+
+    # Log removed users
+    #
+    # @param removed_users [Array<Hash>] collection of removed user result, e.g:
+    #   [
+    #     {
+    #       type => :successful || :failed,
+    #       user => {
+    #         login => 'abc@gooddata.com'
+    #       },
+    #       message => error_message
+    #     },
+    #     ...
+    #   ]
+    # @return nil
+    def log_removed_users(removed_users)
+      removed_users.each do |removed_user|
+        user_login = to_user_login(removed_user[:user])
+        if removed_user[:type] == :successful
+          GoodData.logger.warn("Remove user=#{user_login} out of project=#{@project.pid}")
+        elsif removed_user[:type] == :failed
+          error_message = removed_user[:message]
+          GoodData.logger.error("Failed to remove user=#{user_login} out of project=#{@project.pid}. Error: #{error_message}")
+        end
+      end
+    end
+
+    # Log user filters results
+    #
+    # @param results [Array<Hash>] user-filter results
+    # [
+    #   {
+    #     status: :successful || :failed,
+    #     type: :create || delete,
+    #     user: user_profile_url
+    #   },
+    #   ...
+    # ]
+    # @param user_filters [Hash] user-filters data
+    # {
+    #   user_profile_url => MandatoryUserFilter,
+    #   ...
+    # }
+    # @return nil
+    def log_user_filter_results(results, user_filters)
+      results ||= []
+      results.each do |result|
+        user_profile_url = result[:user]
+        status = result[:status]
+        operator = result[:type]
+        if status == :successful
+          filter_uris = user_filters[user_profile_url].map(&:uri)
+          if operator == :create && GoodData.logger.info?
+            readable_user_login = users_cache[user_profile_url] || user_profile_url
+            GoodData.logger.info "Created user-filter=#{filter_uris} for user=#{readable_user_login} in project=#{@project.pid}"
+          elsif operator == :delete && GoodData.logger.warn?
+            readable_user_login = users_cache[user_profile_url] || user_profile_url
+            GoodData.logger.warn "Deleted user-filter=#{filter_uris} of user=#{readable_user_login} in project=#{@project.pid}"
+          end
+        else
+          error_message = result[:message]
+          if operator == :create && GoodData.logger.error?
+            GoodData.logger.error "Failed to create user-filters for user=#{user_profile_url} in project=#{@project.pid}. Error: #{error_message}"
+          elsif operator == :delete && GoodData.logger.error?
+            GoodData.logger.error "Failed to delete user-filters from user=#{user_profile_url} in project=#{@project.pid}. Error: #{error_message}"
+          end
+        end
+      end
+    end
+
+    private
+
+    def users_cache
+      if @users_cache.nil?
+        @users_cache = Hash[@project.users.map { |user| [user.profile_url, user.login] }]
+      end
+      @users_cache
+    end
+
+    def to_user_login(user)
+      if user.is_a?(String) && user.start_with?('/gdc/account/profile/')
+        GoodData::Helpers.last_uri_part(user)
+      elsif user.is_a?(Hash) && user[:login]
+        user[:login]
+      else
+        user
+      end
+    end
+  end
+end

data/lib/gooddata/models/project_metadata.rb

@@ -0,0 +1,67 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+module GoodData
+  class ProjectMetadata
+    class << self
+      def keys(opts = { :client => GoodData.connection, :project => GoodData.project })
+        ProjectMetadata[:all, opts].keys
+      end
+
+      def [](key, opts = { :client => GoodData.connection, :project => GoodData.project })
+        client, project = GoodData.get_client_and_project(opts)
+
+        get_opts = {
+          do_not_log: [
+            RestClient::ResourceNotFound
+          ]
+        }
+
+        if key == :all
+          uri = "/gdc/projects/#{project.pid}/dataload/metadata"
+          res = client.get(uri, get_opts)
+          res['metadataItems']['items'].reduce({}) do |memo, i|
+            memo[i['metadataItem']['key']] = i['metadataItem']['value']
+            memo
+          end
+        else
+          uri = "/gdc/projects/#{project.pid}/dataload/metadata/#{key}"
+          res = client.get(uri, get_opts)
+          res['metadataItem']['value']
+        end
+      end
+
+      alias_method :get, :[]
+      alias_method :get_key, :[]
+
+      def key?(key, opts = { :client => GoodData.connection, :project => GoodData.project })
+        ProjectMetadata[key, opts]
+        true
+      rescue RestClient::ResourceNotFound
+        false
+      end
+
+      def []=(key, opts = { :client => GoodData.connection, :project => GoodData.project }, val = nil)
+        client, project = GoodData.get_client_and_project(opts)
+
+        data = {
+          :metadataItem => {
+            :key => key,
+            :value => val
+          }
+        }
+        uri = "/gdc/projects/#{project.pid}/dataload/metadata/"
+        update_uri = uri + key
+
+        if key?(key, opts)
+          client.put(update_uri, data)
+        else
+          client.post(uri, data)
+        end
+      end
+    end
+  end
+end