foreman_templates 6.0.0 → 6.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 756a604ec2486b327c246ae19a1c2b153c4217b7
-  data.tar.gz: 563b26321d12a2fad2b8d3f8f9202bcba689cf70
+  metadata.gz: d0c518e8c2822c6112f13654488a324eb9c2918a
+  data.tar.gz: b32fab2dde41371d5367c8d95db70fd2eb85d0ab
 SHA512:
-  metadata.gz: 5cc02e60ced60af1126efed7dda4c89c167a8a59f8a009c3eb9c2fcd855d4f4c61cc4083d15c41993e4583f5fe367b5114c3fa38d9f44a388d8422385ca9f37b
-  data.tar.gz: 8638ebcf4335c45bf137ece2bd534a94e3f7dbc30040e4d5ee7c9c020c1258a159d10a769acc89a19c99174fe13d479d450181bf0d16cdad168fc07f6b280fd4
+  metadata.gz: b647c76ec12f0262f6dc95703487fee408ab094bb918b77f0df6141080cb90b3373cb7d4c27620488b75aa66c0d39450c033ed9ae9145e2cf023656189c3ee18
+  data.tar.gz: '0981be0b744c1a1573fdf0c8c7cec0f82357679cb3873d7ff7264a5af6cc1348e8b8d38da4b9ea5e073b82366e2a79ca284f8b0a4e4037a9951b816c5d692902'

data/app/controllers/api/v2/template_controller.rb

@@ -9,7 +9,6 @@ module Api
         param :filter, String, :required => false, :desc => N_("Export templates with names matching this regex (case-insensitive; snippets are not filtered).")
         param :negate, :bool, :required => false, :desc => N_("Negate the prefix (for purging).")
         param :dirname, String, :required => false, :desc => N_("The directory within Git repo containing the templates")
-        param :verbose, :bool, :required => false, :desc => N_("Set verbosity of import")
       end
 
       api :POST, "/templates/import/", N_("Initiate Import")
@@ -17,21 +16,24 @@ module Api
       param :associate, Setting::TemplateSync.associate_types.keys, :required => false, :desc => N_("Associate to OS's, Locations & Organizations. Options are: always, new or never.")
       param :force, :bool, :required => false, :desc => N_("Update templates that are locked")
       param :lock, :bool, :required => false, :desc => N_("Lock imported templates")
+      param :verbose, :bool, :required => false, :desc => N_("Show template diff in response")
       param_group :foreman_template_sync_params
+      param_group :taxonomies, ::Api::V2::BaseController
       def import
-        results = ForemanTemplates::TemplateImporter.new(template_import_params).import!
-        render :json => { :message => results }
+        verbose = params['verbose']
+        @result = ForemanTemplates::TemplateImporter.new(template_import_params).import!
+        render :json => { :message => { :templates => @result[:results].map { |res| res.to_h(verbose) },
+                                        :repo => @result[:repo],
+                                        :branch => @result[:branch] } }
       end
 
       api :POST, "/templates/export", N_("Initiate Export")
       param :metadata_export_mode, Setting::TemplateSync.metadata_export_mode_types.keys, :required => false, :desc => N_("Specify how to handle metadata")
       param_group :foreman_template_sync_params
+      param_group :taxonomies, ::Api::V2::BaseController
       def export
-        ForemanTemplates::TemplateExporter.new(template_export_params).export!
-        render :json => { :message => _('Success') }
-      rescue StandardError => e
-        logger.debug e
-        render :json => { :message => (_('Something went wrong during export: %s') % e.message) }, :status => :internal_server_error
+        @result = ForemanTemplates::TemplateExporter.new(template_export_params).export!
+        render :json => { :message => @result.to_h }, :status => @result.exported ? 200 : 500
       end
     end
   end

data/app/controllers/concerns/foreman/controller/parameters/template_params.rb

@@ -3,6 +3,7 @@ module Foreman
     module Parameters
       module TemplateParams
         extend ActiveSupport::Concern
+        include Foreman::Controller::Parameters::Taxonomix
 
         class_methods do
           def filter_params_list
@@ -25,11 +26,27 @@ module Foreman
         end
 
         def template_import_params
-          self.class.template_params_filter(self.class.extra_import_params).filter_params(params, parameter_filter_context, :none)
+          add_taxonomy_params(self.class.template_params_filter(self.class.extra_import_params)
+            .filter_params(params, parameter_filter_context, :none).with_indifferent_access)
         end
 
         def template_export_params
-          self.class.template_params_filter(self.class.extra_export_params).filter_params(params, parameter_filter_context, :none)
+          add_taxonomy_params(self.class.template_params_filter(self.class.extra_export_params)
+            .filter_params(params, parameter_filter_context, :none).with_indifferent_access)
+        end
+
+        def organization_params
+          self.class.organization_params_filter(Hash).filter_params(params, parameter_filter_context, :none)
+        end
+
+        def location_params
+          self.class.location_params_filter(Hash).filter_params(params, parameter_filter_context, :none)
+        end
+
+        private
+
+        def add_taxonomy_params(params)
+          params.merge(:organization_params => organization_params.to_h).merge(:location_params => location_params.to_h)
         end
       end
     end

data/app/services/foreman_templates/action.rb

@@ -19,20 +19,11 @@ module ForemanTemplates
     end
 
     def initialize(args = {})
+      @taxonomies = { :organizations => args[:organization_params] || {},
+                      :locations => args[:location_params] || {} }
       assign_attributes args
     end
 
-    def get_default_branch(repo)
-      branch_names = repo.branches.map(&:name).uniq
-
-      # Always use develop on Foreman-nightly, if present, or else relevant stable branch
-      target = SETTINGS[:version].tag == 'develop' ? 'develop' : "#{SETTINGS[:version].short}-stable"
-      return target if branch_names.include?(target)
-
-      # stay on default branch as fallback
-      nil
-    end
-
     def git_repo?
       @repo.start_with?('http://', 'https://', 'git://', 'ssh://', 'git+ssh://', 'ssh+git://')
     end

data/app/services/foreman_templates/export_result.rb

@@ -0,0 +1,41 @@
+module ForemanTemplates
+  class ExportResult
+    attr_accessor :exported, :error, :warning
+
+    def initialize(repo, branch, git_user)
+      @repo = repo
+      @branch = branch
+      @git_user = git_user
+      @error = nil
+      @warning = nil
+      @templates = []
+      @exported = false
+    end
+
+    def add_exported_templates(templates)
+      @templates.concat templates
+    end
+
+    def to_h
+      { :error => @error,
+        :warning => @warning,
+        :repo => @repo,
+        :branch => @branch,
+        :git_user => @git_user,
+        :templates => dumped_files_result }
+    end
+
+    private
+
+    def dumped_files_result
+      @templates.map { |template| to_template_h template }
+    end
+
+    def to_template_h(template)
+      { :id => template.id,
+        :name => template.name,
+        :exported => @exported,
+        :type => template.class.name.underscore }
+    end
+  end
+end

data/app/services/foreman_templates/parse_result.rb

@@ -0,0 +1,92 @@
+module ForemanTemplates
+  class ParseResult
+    attr_accessor :name, :template
+    attr_reader :imported, :diff, :exception, :additional_errors
+
+    def initialize(template_file)
+      @template_file = template_file.split('/').last
+    end
+
+    def to_h(verbose = false)
+      res = {
+        :name => @name,
+        :id => @template.present? ? @template.id : nil,
+        :changed => changed?,
+        :imported => @imported,
+        :additional_errors => @additional_errors,
+        :exception => @exception ? @exception.message : nil,
+        :validation_errors => errors.to_h,
+        :file => @template_file,
+        :type => @template.present? ? @template.class.name.underscore : nil
+      }
+
+      # false comes as a string when using rake tasks
+      res[:diff] = @diff if verbose && verbose != 'false'
+      res
+    end
+
+    def errors
+      @template ? @template.errors : nil
+    end
+
+    def corrupted_metadata
+      generic_error "Failed to parse metadata"
+    end
+
+    def matching_filter
+      generic_error "Skipping, 'name' filtered out based on 'filter' and 'negate' settings"
+    end
+
+    def no_metadata_name
+      generic_error "No 'name' found in metadata"
+    end
+
+    def missing_model
+      generic_error "No 'model' found in metadata"
+    end
+
+    def generic_error(additional_msg)
+      @imported = false
+      @additional_errors = additional_msg
+      Logging.logger('app').debug "Error in '#{@template_file}': #{additional_msg}"
+      self
+    end
+
+    def check_for_errors
+      @imported = @template.errors.blank?
+      self
+    end
+
+    def add_exception(exception)
+      @imported = false
+      @exception = exception
+      self
+    end
+
+    def name_error(exception, template_type)
+      @imported = false
+      @exception = exception
+      @additional_errors = "Template type #{template_type} was not found, are you missing a plugin?"
+      self
+    end
+
+    def determine_result_diff
+      return if @template.nil? || !@template.template_changed?
+      template_was = @template.template_was
+      template_is = @template.template
+      @diff = calculate_diff(template_was, template_is)
+    end
+
+    def changed?
+      @diff.present?
+    end
+
+    def calculate_diff(old, new)
+      if old.present? && new.present? && old != new
+        Diffy::Diff.new(old, new, :include_diff_info => true).to_s(:color)
+      else
+        nil
+      end
+    end
+  end
+end

data/app/services/foreman_templates/template_exporter.rb

@@ -7,61 +7,69 @@ module ForemanTemplates
     end
 
     def export!
+      @export_result = ExportResult.new(@repo, @branch, foreman_git_user)
       if git_repo?
         export_to_git
       else
         export_to_files
       end
 
-      return true
+      return @export_result
     end
 
     def export_to_files
       @dir = get_absolute_repo_path
       verify_path!(@dir)
       dump_files!
+      @export_result.exported = true
     end
 
     def export_to_git
       @dir = Dir.mktmpdir
-
+      return @export_result if branch_missing?
       git_repo = Git.clone(@repo, @dir)
-      logger.debug "cloned #{@repo} to #{@dir}"
-      branch = @branch ? @branch : get_default_branch(git_repo)
-      # either checkout to existing or create a new one and checkout afterwards
-      if branch
-        if git_repo.is_branch?(branch)
-          git_repo.checkout(branch)
-        else
-          git_repo.branch(branch).checkout
-          if git_repo.is_remote_branch?(branch) # if we work with remote branch we need to sync it first
-            git_repo.reset_hard("origin/#{branch}")
-          end
-        end
-      end
+      logger.debug "cloned '#{@repo}' to '#{@dir}'"
 
+      setup_git_branch git_repo
       dump_files!
       git_repo.add
 
       status = git_repo.status
       if status.added.any? || status.changed.any? || status.deleted.any? || status.untracked.any?
-        logger.debug 'committing changes in cloned repo'
-        git_repo.commit "Templates export made by Foreman user #{User.current.try(:login) || User::ANONYMOUS_ADMIN}"
-
-        logger.debug "pushing to branch #{branch} at origin #{@repo}"
+        git_repo.commit "Templates export made by Foreman user #{foreman_git_user}"
         git_repo.push 'origin', branch
+        @export_result.exported = true
       else
-        logger.debug 'no change detected, skipping the commit and push'
+        @export_result.warning = 'No change detected, skipping the commit and push'
       end
+    rescue StandardError => e
+      @export_result.error = e.message
     ensure
       FileUtils.remove_entry_secure(@dir) if File.exist?(@dir)
+      @export_result
+    end
+
+    def setup_git_branch(git_repo)
+      logger.debug "checking out branch '#{@branch}'"
+      if git_repo.is_branch?(@branch)
+        git_repo.checkout(@branch)
+      else
+        git_repo.branch(@branch).checkout
+        if git_repo.is_remote_branch?(@branch) # if we work with remote branch we need to sync it first
+          git_repo.reset_hard("origin/#{@branch}")
+        end
+      end
+    end
+
+    def foreman_git_user
+      User.current.try(:login) || User::ANONYMOUS_ADMIN
     end
 
     def dump_files!
-      templates_to_dump.map do |template|
+      templates = templates_to_dump
+      templates.map do |template|
         current_dir = get_dump_dir(template)
         FileUtils.mkdir_p current_dir
-
         filename = File.join(current_dir, get_template_filename(template))
         File.open(filename, 'w+') do |file|
           logger.debug "Writing to file #{filename}"
@@ -69,6 +77,7 @@ module ForemanTemplates
           logger.debug "finished writing #{bytes}"
         end
       end
+      @export_result.add_exported_templates templates
     end
 
     def get_template_filename(template)
@@ -77,13 +86,13 @@ module ForemanTemplates
 
     def get_dump_dir(template)
       kind = template.respond_to?(:template_kind) ? template.template_kind.try(:name) || 'snippet' : nil
-      template_class_dir = template.model_name.human.pluralize.downcase.tr(' ', '_')
-      template_class_dir += '_templates' if template_class_dir == 'partition_tables'
+      template_class_dir = template.model_name.plural
+      template_class_dir = 'partition_tables_templates' if template_class_dir == 'ptables'
       File.join(@dir, dirname.to_s, template_class_dir, kind.to_s)
     end
 
     def templates_to_dump
-      base = Template.all
+      base = find_templates
       if filter.present?
         method = negate ? :reject : :select
         base.public_send(method) { |template| template.name.match(/#{filter}/i) }
@@ -92,6 +101,14 @@ module ForemanTemplates
       end
     end
 
+    def branch_missing?
+      if @branch.blank?
+        @export_result.error = "Please specify a branch when exporting into a git repo"
+        return true
+      end
+      false
+    end
+
     # * refresh - template.to_erb stripping existing metadata,
     # * remove  - just template.template with stripping existing metadata,
     # * keep    - taking the whole template.template
@@ -107,5 +124,36 @@ module ForemanTemplates
           raise "Unknown metadata export mode #{@metadata_export_mode}"
       end
     end
+
+    private
+
+    def find_templates
+      @taxonomies.values.all?(&:empty?) ? Template.all : find_taxed_templates
+    end
+
+    def find_taxed_templates
+      location_ids = taxes_ids('location')
+      organization_ids = taxes_ids('organization')
+      if location_ids.empty?
+        templates_query(organization_ids)
+      elsif organization_ids.empty?
+        templates_query(location_ids)
+      else
+        templates_query(organization_ids) & templates_query(location_ids)
+      end
+    end
+
+    def templates_query(tax_ids)
+      Template.where(:id => TaxableTaxonomy.where(:taxonomy_id => tax_ids,
+                                                  :taxable_type => Template.subclasses.map(&:name)).pluck(:taxable_id))
+    end
+
+    def taxes_ids(tax_type)
+      tax_type.capitalize.constantize
+              .where(:name => @taxonomies[tax_type.pluralize.to_sym]["#{tax_type}_names".to_sym])
+              .pluck(:id)
+              .concat(@taxonomies[tax_type.pluralize.to_sym]["#{tax_type}_ids".to_sym] || [])
+              .uniq
+    end
   end
 end

data/app/services/foreman_templates/template_importer.rb

@@ -11,6 +11,7 @@ module ForemanTemplates
       @verbose = parse_bool(@verbose)
       @force = parse_bool(@force)
       @lock = parse_bool(@lock)
+      @result_lines = []
     end
 
     def import!
@@ -24,7 +25,7 @@ module ForemanTemplates
     def import_from_files
       @dir = get_absolute_repo_path
       verify_path!(@dir)
-      return parse_files!
+      parse_files!
     end
 
     def import_from_git
@@ -32,95 +33,117 @@ module ForemanTemplates
       @dir = Dir.mktmpdir
 
       begin
+        logger.debug "cloned '#{@repo}' to '#{@dir}'"
         gitrepo = Git.clone(@repo, @dir)
-        branch = @branch ? @branch : get_default_branch(gitrepo)
-        gitrepo.checkout(branch) if branch
+        if @branch
+          logger.debug "checking out branch '#{@branch}'"
+          gitrepo.checkout(@branch)
+        end
 
-        return parse_files!
+        parse_files!
       ensure
         FileUtils.remove_entry_secure(@dir) if File.exist?(@dir)
       end
     end
 
     def parse_files!
-      result_lines = []
-
-      # Build a list of ERB files to parse
-      Dir["#{@dir}#{@dirname}/**/*.erb"].each do |template|
-        text = File.read(template)
-        result_lines << 'Parsing: ' + template.gsub(/#{@dir}#{@dirname}/, '') if @verbose
+      Dir["#{@dir}#{@dirname}/**/*.erb"].each do |template_file|
+        logger.debug 'Parsing: ' + template_file.gsub(/#{@dir}#{@dirname}/, '')
+        parse_result = ParseResult.new(template_file)
 
+        text = File.read(template_file)
         metadata = Template.parse_metadata(text)
 
-        # Get the name and filter
-        name = metadata['name']
-        name = auto_prefix(name)
-        if @filter
-          matching = name.match(/#{@filter}/i)
-          matching = !matching if @negate
-          unless matching
-            result_lines << "Skipping template '#{name}' since it's name is matching filter condition" if @verbose
-            next
-          end
-        end
+        next if metadata_corrupted?(metadata, parse_result)
+
+        next unless (name = auto_prefix_name(metadata, parse_result))
+
+        next if filtered_out name, parse_result
 
-        options = { :force => @force, :associate => @associate, :lock => @lock }
-        template_type = 'Unknown template type'
         begin
-          if metadata.key?('model')
-            template_type = metadata['model'].constantize
-          else
-            result_lines << "Metadata for template '#{name}' do not specify 'model', import failed"
-            next
-          end
-
-          template = template_type.import_without_save(name, text, options)
-
-          data = {}
-          if template.template_changed?
-            data[:old] = template.template_was
-            data[:new] = template.template
-            if data[:diff].nil? && data[:old].present? && data[:new].present?
-              data[:diff] = calculate_diff(data[:old], data[:new])
-            end
-          end
-
-          if options[:force]
-            template.ignore_locking { template.save! }
-          else
-            template.save!
-          end
-
-          template_type = template.class.model_name.human
-          data[:status] = template.errors.blank? ? "#{template_type} '#{name}' import successful" : "#{template_type} '#{name}' import failed"
-          data[:errors] = template.errors.full_messages
-
-          if @verbose
-            result_lines << data[:diff] unless data[:diff].nil?
-          end
-          result_lines << data[:status]
-          result_lines << data[:errors] unless data[:errors].empty?
+          next unless (template_type = template_model(metadata, parse_result))
+          template = template_type.import_without_save(name, text, import_options)
+          parse_result.template = template
+          parse_result.determine_result_diff
+
+          save_template template, @force
+          @result_lines << parse_result.check_for_errors
+        rescue NameError => e
+          @result_lines << parse_result.name_error(e, metadata['model'])
         rescue => e
-          Foreman::Logging.exception 'error during template import', e, :level => :debug
-          result_lines << "#{template_type} '#{name}' import failed - #{e.message}"
-          next
+          @result_lines << parse_result.add_exception(e)
         end
       end
-      result_lines
+      { :results => @result_lines, :repo => @repo, :branch => @branch }
     end
 
-    def auto_prefix(name)
-      name.start_with?(@prefix) ? name : [@prefix, name].compact.join
+    def import_options
+      { :force => @force,
+        :associate => @associate,
+        :lock => @lock,
+        :organization_params => @organizations,
+        :location_params => @locations }
     end
 
-    def calculate_diff(old, new)
-      if old != new
-        Diffy::Diff.new(old, new, :include_diff_info => true).to_s(:color)
+    def template_model(metadata, parse_result)
+      if metadata.key?('model')
+        metadata['model'].constantize
       else
+        @result_lines << parse_result.missing_model
         nil
       end
     end
 
+    def save_template(template, force)
+      result = nil
+      if force
+        result = template.ignore_locking { template.save }
+      else
+        result = template.save
+      end
+
+      if result
+        logger.debug 'saved'
+      else
+        logger.error "couldn't save the template because of: #{template.errors.full_messages.join(', ')}"
+      end
+      result
+    end
+
+    def filtered_out(name, parse_result)
+      if @filter && !name_matching_filter?(name)
+        @result_lines << parse_result.matching_filter
+        true
+      end
+    end
+
+    def metadata_corrupted?(metadata, parse_result)
+      if metadata.empty?
+        @result_lines << parse_result.corrupted_metadata
+        return true
+      end
+      false
+    end
+
+    def name_matching_filter?(name)
+      matching = name.match(/#{@filter}/i)
+      return !matching if @negate
+      matching
+    end
+
+    def auto_prefix_name(metadata, parse_result)
+      unless (name = metadata['name'])
+        @result_lines << parse_result.no_metadata_name
+        return nil
+      end
+      parse_result.name = auto_prefix(name)
+      auto_prefix(name)
+    end
+
+    def auto_prefix(name)
+      name.start_with?(@prefix) ? name : [@prefix, name].compact.join
+    end
+
     def purge!
       clause = "name #{@negate ? 'NOT ' : ''}LIKE ?"
       ProvisioningTemplate.where(clause, "#{@prefix}%").each do |template|

data/lib/foreman_templates/version.rb

@@ -1,3 +1,3 @@
 module ForemanTemplates
-  VERSION = '6.0.0'.freeze
+  VERSION = '6.0.1'.freeze
 end

data/lib/tasks/foreman_templates_tasks.rake

@@ -1,4 +1,6 @@
+require 'pp'
 # Tasks
+# rubocop:disable Metrics/BlockLength
 namespace :templates do
   desc 'Import templates according to settings'
   task :import => :environment do
@@ -15,11 +17,11 @@ namespace :templates do
     # * filter    => Import names matching this regex (case-insensitive; snippets are not filtered)
     # * associate => Associate to OS's, Locations & Organizations. Options are: always, new or never  [new]
     # * lock      => Lock imported templates [false]
-
     User.current = User.anonymous_admin
+    verbose = ENV['verbose']
 
     results = ForemanTemplates::TemplateImporter.new({
-      verbose:   ENV['verbose'],
+      verbose:   verbose,
       repo:      ENV['repo'],
       branch:    ENV['branch'],
       prefix:    ENV['prefix'],
@@ -29,35 +31,31 @@ namespace :templates do
       lock:      ENV['lock'],
     }).import!
 
-    puts results.join("\n")
+    pp(results.map { |result| result.to_h(verbose) })
   end
 
   task :sync => :import
-
   desc 'Export templates according to settings'
   task :export => :environment do
     ActiveSupport::Deprecation.warn "You are using a deprecated behavior, 'rake templates:export' will be removed in a future version. Please use appropriate API endpoint for this functionality"
     User.current = User.anonymous_admin
 
-    ForemanTemplates::TemplateExporter.new({
-      verbose:   ENV['verbose'],
+    result = ForemanTemplates::TemplateExporter.new({
       repo: ENV['repo'],
       branch: ENV['branch'],
       prefix: ENV['prefix'],
       dirname: ENV['dirname'],
       filter: ENV['filter'],
-      # associate: ENV['associate'],
       metadata_export_mode: ENV['metadata_export_mode'],
     }).export!
 
-    puts 'Export finished'
+    pp result
   end
 
   desc 'Purge unwanted templates from foreman'
   task :purge => :environment do
     ActiveSupport::Deprecation.warn "You are using a deprecated behavior, 'rake templates:purge' will be removed in a future version. Please use appropriate API endpoint for this functionality"
     User.current = User.anonymous_admin
-
     ForemanTemplates::TemplateImporter.new({
       # * negate  => negate query [false]
       # * prefix  => The string all templates to purge should ( or not ) begin with [Community ]
@@ -75,6 +73,7 @@ namespace :templates do
     puts 'Clean up finished, you can now remove the plugin from your system'
   end
 end
+# rubocop:enable Metrics/BlockLength
 
 # Tests
 namespace :test do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: foreman_templates
 version: !ruby/object:Gem::Version
-  version: 6.0.0
+  version: 6.0.1
 platform: ruby
 authors:
 - Greg Sutcliffe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-10 00:00:00.000000000 Z
+date: 2018-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: diffy
@@ -68,6 +68,8 @@ files:
 - app/models/setting/template_sync.rb
 - app/services/foreman_templates/action.rb
 - app/services/foreman_templates/cleaner.rb
+- app/services/foreman_templates/export_result.rb
+- app/services/foreman_templates/parse_result.rb
 - app/services/foreman_templates/template_exporter.rb
 - app/services/foreman_templates/template_importer.rb
 - config/routes.rb