machinery-tool 1.4.0 → 1.5.0

data/lib/element_filter.rb

@@ -0,0 +1,45 @@
+# Copyright (c) 2013-2015 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 3 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.   See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, contact SUSE LLC.
+#
+# To contact SUSE about this file by physical or electronic mail,
+# you may find current contact information at www.suse.com
+
+class ElementFilter
+  attr_accessor :path, :matchers
+
+  def initialize(path, matchers = nil)
+    @path = path
+    @matchers = []
+
+    raise("Wrong type") if ![NilClass, String, Array].include?(matchers.class)
+
+    add_matchers(matchers) if matchers
+  end
+
+  def add_matchers(matchers)
+    @matchers += Array(matchers)
+  end
+
+  def matches?(value)
+    @matchers.each do |matcher|
+      if matcher.end_with?("*")
+        return true if value.start_with?(matcher[0..-2])
+      else
+        return true if value == matcher
+      end
+    end
+
+    false
+  end
+end

data/lib/exceptions.rb

@@ -38,10 +38,11 @@ module Machinery
     class SystemDescriptionNotFound < SystemDescriptionError; end
 
     class SystemDescriptionIncompatible < SystemDescriptionError
-      attr_reader :name
+      attr_reader :name, :format_version
 
-      def initialize(name)
+      def initialize(name, format_version)
         @name = name
+        @format_version = format_version
       end
 
       def to_s

data/lib/filter.rb

@@ -0,0 +1,121 @@
+# Copyright (c) 2013-2015 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 3 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.   See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, contact SUSE LLC.
+#
+# To contact SUSE about this file by physical or electronic mail,
+# you may find current contact information at www.suse.com
+
+# The Filter class is used to hold the information about the filter conditions
+# that should be applied during certain Machinery commands.
+#
+# Filters are usually created by passing a filter definition string to the
+# constructor, e.g.
+#
+#   filter = Filter.new("/unmanaged_files/files/name=/opt")
+#
+# Existing filters can be extended by amending the definition:
+#
+#   filter.add_element_filter_from_definition("/unmanaged_files/files/name=/srv")
+#
+# or by adding ElementFilters directly:
+#
+#   element_filter = ElementFilter.new("/unmanaged_files/files/name", ["/opt", "/srv"])
+#   filter.add_element_filter(element_filter)
+#
+#
+# The actual filtering can be done by passing values to Filter#matches?
+#
+#   filter = Filter.new("/unmanaged_files/files/name=/opt*")
+#   filter.matches?("/unmanaged_files/files/name", "/opt/foo")
+#   => true
+#   filter.matches?("/unmanaged_files/files/name", "/srv/bar")
+#   => false
+#
+# More details about how the filter work can be found at
+# https://github.com/SUSE/machinery/blob/master/docs/Filtering-Design.md
+class Filter
+  attr_accessor :element_filters
+
+  def self.parse_filter_definitions(filter_definitions)
+    element_filters = {}
+    Array(filter_definitions).each do |definition|
+      path, matcher_definition = definition.split("=", 2)
+
+      element_filters[path] ||= ElementFilter.new(path)
+      if matcher_definition.index(",")
+        matchers = matcher_definition.split(/(?<!\\),/)
+        matchers.map! { |matcher| matcher.gsub("\\,", ",") } # Unescape escaped commas
+
+        element_filters[path].add_matchers([matchers])
+      else
+        element_filters[path].add_matchers(matcher_definition)
+      end
+    end
+
+    element_filters
+  end
+
+  def self.from_default_definition(command)
+    filter = Filter.new
+
+    default_filters_file = File.join(Machinery::ROOT, "helpers/default_filters.json")
+    if File.exists?(default_filters_file)
+      default_filters = JSON.parse(File.read(default_filters_file))
+      if default_filters[command]
+        default_filters[command].each do |definition|
+          filter.add_element_filter_from_definition(definition)
+        end
+      end
+    end
+
+    filter
+  end
+
+  def initialize(definitions = [])
+    @element_filters = Filter.parse_filter_definitions(definitions)
+  end
+
+  def add_element_filter_from_definition(filter_definition)
+    new_element_filters = Filter.parse_filter_definitions(filter_definition)
+
+    new_element_filters.each do |path, element_filter|
+      @element_filters[path] ||= ElementFilter.new(path)
+      @element_filters[path].add_matchers(element_filter.matchers)
+    end
+  end
+
+  def add_element_filter(element_filter)
+    path = element_filter.path
+    @element_filters[path] ||= ElementFilter.new(path)
+    @element_filters[path].add_matchers(element_filter.matchers)
+  end
+
+  def to_array
+    @element_filters.flat_map do |path, element_filter|
+      element_filter.matchers.map do |matcher|
+        "#{path}=#{Array(matcher).join(",")}"
+      end
+    end
+  end
+
+  def matches?(path, value)
+    filter = element_filter_for(path)
+    return false if !filter
+
+    filter.matches?(value)
+  end
+
+  def element_filter_for(path)
+    element_filters[path]
+  end
+end

data/lib/inspect_task.rb

@@ -16,11 +16,12 @@
 # you may find current contact information at www.suse.com
 
 class InspectTask
-  def inspect_system(store, host, name, current_user, scopes, options = {})
+  def inspect_system(store, host, name, current_user, scopes, filter, options = {})
     system = System.for(host)
     check_root(system, current_user)
 
-    description, failed_inspections = build_description(store, name, system, scopes, options)
+    description, failed_inspections = build_description(store, name, system,
+      scopes, filter, options)
 
     if !description.attributes.empty?
       print_description(description, scopes) if options[:show]
@@ -57,7 +58,17 @@ class InspectTask
     end
   end
 
-  def build_description(store, name, system, scopes, options)
+  def adapt_filter_in_metadata(filter_in_metadata, scope, filter)
+    filter_in_metadata.element_filters.
+      reject! { |path, _filter| path.start_with?("/#{scope}") }
+    filter.element_filters.
+      select { |path, _filter| path.start_with?("/#{scope}") }.
+      each do |_path, element_filter|
+      filter_in_metadata.add_element_filter(element_filter)
+    end
+  end
+
+  def build_description(store, name, system, scopes, filter, options)
     begin
       description = SystemDescription.load(name, store)
     rescue Machinery::Errors::SystemDescriptionNotFound
@@ -72,17 +83,27 @@ class InspectTask
 
     failed_inspections = {}
 
+    if description.filters["inspect"]
+      filter_in_metadata = description.filters["inspect"]
+    else
+      filter_in_metadata = Filter.new
+    end
+
     scopes.map { |s| Inspector.for(s) }.each do |inspector|
       Machinery::Ui.puts "Inspecting #{Machinery::Ui.internal_scope_list_to_string(inspector.scope)}..."
       begin
-        summary = inspector.inspect(system, description, options)
+        summary = inspector.inspect(system, description, filter, options)
       rescue Machinery::Errors::MachineryError => e
         Machinery::Ui.puts " -> Inspection failed!"
         failed_inspections[inspector.scope] = e
         next
       end
       description[inspector.scope].set_metadata(timestring, host)
+
+      adapt_filter_in_metadata(filter_in_metadata, inspector.scope, filter)
+
       if !description.attributes.empty?
+        description.set_filter("inspect", filter_in_metadata)
         description.save
       end
       Machinery::Ui.puts " -> " + summary

data/lib/kiwi_config.rb

@@ -221,6 +221,7 @@ EOF
 
   def apply_repositories(xml)
     if @system_description.repositories
+      usable_repositories = false
       @system_description.repositories.each do |repo|
         # workaround kiwi issue by replacing spaces
         # the final image is not affected because the repositories are added by the config.sh
@@ -231,6 +232,7 @@ EOF
         end
         # only use accessible repositories as source for kiwi build
         if repo.enabled && !repo.type.nil? && !repo.external_medium?
+          usable_repositories = true
           xml.repository(parameters) do
             xml.source(path: repo.url)
           end
@@ -244,6 +246,15 @@ EOF
           @sh << "zypper -n mr --priority=#{repo.priority} '#{repo.name}'\n"
         end
       end
+      if !usable_repositories
+        raise(
+          Machinery::Errors::MissingRequirement.new(
+            "The system description doesn't contain any enabled or network reachable repository." \
+              " Please make sure that there is at least one accessible repository with all the" \
+              " required packages."
+          )
+        )
+      end
     end
   end
 

data/lib/list_task.rb

@@ -20,45 +20,72 @@ class ListTask
     descriptions = store.list.sort
 
     descriptions.each do |name|
-      name = File.basename(name)
       begin
         description = SystemDescription.load(name, store, skip_validation: true)
+      rescue Machinery::Errors::SystemDescriptionIncompatible => e
+        if !e.format_version
+          show_error("#{name}: incompatible format version. Can not be upgraded.\n", options)
+        elsif e.format_version < SystemDescription::CURRENT_FORMAT_VERSION
+          show_error("#{name}: format version #{e.format_version}, " \
+            "needs to be upgraded.\n", options)
+        else
+          show_error("#{name}: format version #{e.format_version}. " \
+            "Please upgrade Machinery to the latest version.\n", options)
+        end
+        next
+      rescue Machinery::Errors::SystemDescriptionValidationFailed
+        show_error("#{name}: This description is broken. Use " \
+          "`#{$0} validate #{name}` to see the error message.\n", options)
+        next
       rescue Machinery::Errors::SystemDescriptionError
-        Machinery::Ui.puts " #{name}:\n"
-        Machinery::Ui.puts "   This description has an incompatible data format or is broken.\n" \
-            "   Use `#{$0} validate #{name}` to see the error message.\n\n"
+        show_error("#{name}: This description is broken.\n", options)
         next
       end
-      scopes = []
 
-      description.scopes.each do |scope|
-        entry = Machinery::Ui.internal_scope_list_to_string(scope)
-        if SystemDescription::EXTRACTABLE_SCOPES.include?(scope)
-          if description.scope_extracted?(scope)
-            entry += " (extracted)"
-          else
-            entry += " (not extracted)"
+      if options[:short]
+        Machinery::Ui.puts name
+      else
+        scopes = []
+
+        description.scopes.each do |scope|
+          entry = Machinery::Ui.internal_scope_list_to_string(scope)
+          if SystemDescription::EXTRACTABLE_SCOPES.include?(scope)
+            if description.scope_extracted?(scope)
+              entry += " (extracted)"
+            else
+              entry += " (not extracted)"
+            end
           end
-        end
 
-        if options["verbose"]
-          meta = description[scope].meta
-          if meta
-            time = Time.parse(meta.modified).getlocal
-            date = time.strftime "%Y-%m-%d %H:%M:%S"
-            hostname = meta.hostname
-          else
-            date = "unknown"
-            hostname = "Unknown hostname"
+          if options[:verbose]
+            meta = description[scope].meta
+            if meta
+              time = Time.parse(meta.modified).getlocal
+              date = time.strftime "%Y-%m-%d %H:%M:%S"
+              hostname = meta.hostname
+            else
+              date = "unknown"
+              hostname = "Unknown hostname"
+            end
+            entry += "\n      Host: [#{hostname}]"
+            entry += "\n      Date: (#{date})"
           end
-          entry += "\n      Host: [#{hostname}]"
-          entry += "\n      Date: (#{date})"
+
+          scopes << entry
         end
 
-        scopes << entry
+        Machinery::Ui.puts " #{name}:\n   * " + scopes .join("\n   * ") + "\n\n"
       end
+    end
+  end
+
+  private
 
-      Machinery::Ui.puts " #{name}:\n   * " + scopes .join("\n   * ") + "\n\n"
+  def show_error(error_message, options)
+    if options[:short]
+      Machinery::Ui.puts(error_message)
+    else
+      Machinery::Ui.puts(" " + error_message + "\n")
     end
   end
 end

data/lib/local_system.rb

@@ -24,7 +24,7 @@ class LocalSystem < System
         description = SystemDescription.new("localhost",
           SystemDescriptionMemoryStore.new)
         inspector = OsInspector.new
-        inspector.inspect(System.for("localhost"), description)
+        inspector.inspect(System.for("localhost"), description, nil)
         @@os = description.os
       end
       @@os

data/lib/machinery.rb

@@ -87,6 +87,8 @@ require_relative "scope_file_store"
 require_relative "json_validator"
 require_relative "json_validation_error_cleaner"
 require_relative "file_validator"
+require_relative "element_filter"
+require_relative "filter"
 
 Dir[File.join(Machinery::ROOT, "plugins", "**", "*.rb")].each { |f| require(f) }
 

data/lib/manifest.rb

@@ -55,7 +55,7 @@ class Manifest
 
     errors = JsonValidator.new(@hash).validate
     if !errors.empty?
-      raise Machinery::Errors::SystemDescriptionError.new(errors.join("\n"))
+      raise Machinery::Errors::SystemDescriptionValidationFailed.new(errors)
     end
   end
 
@@ -74,7 +74,6 @@ class Manifest
 
     # remove needless json error information
     lines[0].gsub!(/^\d+: (.*)$/, "\\1")
-    json_error = lines[0..block_end].join("\n")
 
     if error_pos == 1
       json_error = "An opening bracket, a comma or quotation is missing " \
@@ -84,6 +83,8 @@ class Manifest
       error_pos = nil
     end
 
+    json_error ||= lines[0..block_end].join("\n")
+
     error = "The JSON data of the system description '#{name}' " \
             "couldn't be parsed. The following error occured"
     error += " around line #{error_pos}" if error_pos

data/lib/migration.rb

@@ -75,13 +75,13 @@ class Migration
           Machinery::Ui.warn("Warning: System Description validation errors:")
           Machinery::Ui.warn(errors)
         else
-          raise Machinery::Errors::SystemDescriptionError.new(errors.join("\n"))
+          raise Machinery::Errors::SystemDescriptionValidationFailed.new(errors)
         end
       end
 
       current_version = hash["meta"]["format_version"]
       if !current_version
-        raise Machinery::Errors::SystemDescriptionError.new(
+        raise Machinery::Errors::SystemDescriptionIncompatible.new(
           "The system description '#{description_name}' was generated by an old " \
           "version of machinery that is not supported by the upgrade mechanism."
         )

data/lib/system_description.rb

@@ -36,6 +36,7 @@ class SystemDescription < Machinery::Object
   attr_accessor :name
   attr_accessor :store
   attr_accessor :format_version
+  attr_accessor :filters
 
   class << self
     # Load the system description with the given name
@@ -89,14 +90,27 @@ class SystemDescription < Machinery::Object
 
     def from_hash(name, store, hash)
       begin
+        json_format_version = hash["meta"]["format_version"] if hash["meta"]
         description = SystemDescription.new(name, store, create_scopes(hash))
-      rescue NameError
-        raise Machinery::Errors::SystemDescriptionIncompatible.new(name)
+      rescue NameError, TypeError
+        if json_format_version && json_format_version != SystemDescription::CURRENT_FORMAT_VERSION
+          raise Machinery::Errors::SystemDescriptionIncompatible.new(name, json_format_version)
+        else
+          raise Machinery::Errors::SystemDescriptionError.new
+        end
       end
 
-      json_format_version = hash["meta"]["format_version"] if hash["meta"]
       description.format_version = json_format_version
 
+      if hash["meta"] && hash["meta"]["filters"]
+        hash["meta"]["filters"].each do |command, filter_definitions|
+          description.filters[command] = Filter.new
+          filter_definitions.each do |definition|
+            description.filters[command].add_element_filter_from_definition(definition)
+          end
+        end
+      end
+
       description
     end
 
@@ -125,6 +139,7 @@ class SystemDescription < Machinery::Object
     @name = name
     @store = store
     @format_version = CURRENT_FORMAT_VERSION
+    @filters = {}
 
     super(hash)
   end
@@ -136,7 +151,7 @@ class SystemDescription < Machinery::Object
 
   def validate_format_compatibility
     if !compatible?
-      raise Machinery::Errors::SystemDescriptionIncompatible.new(self.name)
+      raise Machinery::Errors::SystemDescriptionIncompatible.new(name, format_version)
     end
   end
 
@@ -161,6 +176,10 @@ class SystemDescription < Machinery::Object
     attributes.keys.each do |key|
       meta[key] = self[key].meta.as_json if self[key].meta
     end
+    @filters.each do |command, filter|
+      meta["filters"] ||= {}
+      meta["filters"][command] = filter.to_array
+    end
 
     hash = as_json
     hash["meta"] = meta unless meta.empty?
@@ -180,6 +199,16 @@ class SystemDescription < Machinery::Object
     File.chmod(0600, path) if created
   end
 
+  def set_filter(command, filter)
+    if !["inspect"].include?(command)
+      raise Machinery::Errors::MachineryError.new(
+        "Storing the filter for command '#{command}' is not supported."
+      )
+    end
+
+    @filters[command] = filter
+  end
+
   def scopes
     attributes.keys.map(&:to_s).sort
   end