machinery-tool 1.16.4 → 1.17.0

data/lib/cli.rb

@@ -150,7 +150,8 @@ class Cli
         "Error: System description name '#{name}' is invalid. By default Machinery" \
           " uses the image name as description name if the parameter `--name` is not" \
           " provided.\nIf the image name contains a slash the `--name=NAME` parameter" \
-          " is mandatory. Valid characters are 'a-zA-Z0-9_:.-'."
+          " is mandatory. Valid characters are 'a-zA-Z0-9_:.-'.\n\nFor example run:\n" \
+          "#{Hint.program_name} #{ARGV.join(" ")} --name='#{image.tr("/", "_")}'"
       )
     end
   end
@@ -584,10 +585,17 @@ class Cli
     c.flag ["remote-user", :r], type: String, required: false, default_value: @config.remote_user,
       desc: "Defines the user which is used to access the inspected system via SSH."\
         "This user needs sudo access on the remote machine or be root.", arg_name: "USER"
+    c.flag ["ssh-port", :p], type: Integer, required: false,
+      desc: "The SSH port of the remote server.",
+      arg_name: "SSH-PORT"
+    c.flag ["ssh-identity-file", :i], type: String, required: false,
+      desc: "The private SSH key location what can be used to authenticate with the remote system.",
+      arg_name: "SSH-KEY"
 
     c.action do |_global_options, options, args|
       host = shift_arg(args, "HOSTNAME")
-      system = System.for(host, options["remote-user"])
+      system = System.for(host, remote_user: options["remote-user"], ssh_port: options["ssh-port"],
+        ssh_identity_file: options["ssh-identity-file"])
       inspector_task = InspectTask.new
 
       name, scope_list, inspect_options, filter = parse_inspect_command_options(host, options)

data/lib/config.rb

@@ -30,20 +30,20 @@ module Machinery
         default: true,
         description: "Show hints about usage of Machinery in the context of the commands ran by" \
           " the user"
-      )
+           )
       entry("remote-user",
         default: "root",
         description: "Defines the user which is used to access the inspected system via SSH"
-      )
+           )
       entry("experimental-features",
         default: false,
         description: "Enable experimental features. See " \
           "https://github.com/SUSE/machinery/wiki/Experimental-Features for more details"
-      )
+           )
       entry("http_server_port",
         default: 7585,
         description: "TCP port used by the HTTP server for the HTML view"
-      )
+           )
     end
 
     def deprecated_entries

data/lib/dpkg_database.rb

@@ -0,0 +1,68 @@
+# Copyright (c) 2013-2015 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 3 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.   See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, contact SUSE LLC.
+#
+# To contact SUSE about this file by physical or electronic mail,
+# you may find current contact information at www.suse.com
+
+class DpkgDatabase < ManagedFilesDatabase
+  def managed_files_list(&block)
+    message = "The list of changed config and managed files is not complete on dpkg systems."\
+     " The reason for this is missing verifcation data:" \
+     " https://github.com/SUSE/machinery/wiki/Ubuntu-Inspection"
+
+    Machinery.logger.warn(message)
+    Machinery::Ui.warn("Warning: #{message}")
+
+    @system.run_command_with_progress("dpkg", "--verify", privileged: true, &block)
+  end
+
+  def package_for_file_path(file)
+    package_name = @system.run_command("dpkg", "-S", file, stdout: :capture).split(":").first
+    package_details = @system.run_command("dpkg", "-s", package_name, stdout: :capture)
+    package_version = package_details.match(/^Version: (.*)$/)[1]
+
+    [package_name, package_version]
+  end
+
+  def handle_verify_fail(_path)
+    # dpkg can only check for md5sum. Thus, dpkg will report for every file
+    # that not all tests (e.g. owner, mode) could be performed.
+    #
+    # We won't show a warning for each and every file but one warning
+    # per inspection.
+    #
+    # This method is a no-op for DpkgDatabase.
+  end
+
+  def parse_changes_line(line)
+    file, changes, type = super(line)
+
+    # dpkg doesn't report deleted files as deleted but reports md5sum changes instead
+    if changes.include?("md5")
+      begin
+        @system.run_command("ls", file)
+      rescue Cheetah::ExecutionFailed
+        changes = ["deleted"]
+      end
+    end
+
+    [file, changes, type]
+  end
+
+  def check_requirements
+    @system.check_requirement("dpkg", "--version")
+    @system.check_requirement("stat", "--version")
+    @system.check_requirement("find", "--version")
+  end
+end

data/lib/element_filter.rb

@@ -50,6 +50,8 @@ class ElementFilter
           value_array = value.elements
 
           (value_array - Array(matcher)).empty? && (Array(matcher) - value_array).empty?
+        when ::Array
+          (value - Array(matcher)).empty? && (Array(matcher) - value).empty?
         when String
           if matcher.is_a?(Array)
             exception = Machinery::Errors::ElementFilterTypeMismatch.new

data/lib/file_diff.rb

@@ -19,8 +19,8 @@ class FileDiff
   def self.diff(description1, description2, scope, path)
     return nil if !description1.scope_extracted?(scope) || !description2.scope_extracted?(scope)
 
-    file1 = description1[scope].files.find { |f| f.name == path }
-    file2 = description2[scope].files.find { |f| f.name == path }
+    file1 = description1[scope].find { |f| f.name == path }
+    file2 = description2[scope].find { |f| f.name == path }
     return nil if !file1 || !file2
 
     if file1.binary? || file2.binary?

data/lib/file_scope.rb

@@ -15,58 +15,19 @@
 # To contact SUSE about this file by physical or electronic mail,
 # you may find current contact information at www.suse.com
 
-class FileScope < Machinery::Object
+class FileScope < Machinery::Array
   def compare_with(other)
-    validate_attributes(other)
+    only_self, only_other, changed, common = super(other)
 
-    only_self = self.class.new
-    only_other = self.class.new
-    shared = self.class.new
-
-    compare_extracted(other, only_self, only_other, shared)
-    changed = compare_files(other, only_self, only_other, shared)
-
-    only_self = nil if only_self.empty?
-    only_other = nil if only_other.empty?
-    shared = nil if shared.empty?
-    [only_self, only_other, changed, shared]
-  end
-
-  def length
-    files.try(:length) || 0
-  end
-
-  private
-
-  def validate_attributes(other)
-    expected_attributes = ["extracted", "files"]
-    actual_attributes = (attributes.keys + other.attributes.keys).uniq.sort
-
-    if actual_attributes != expected_attributes
-      unsupported = actual_attributes - expected_attributes
-      raise Machinery::Errors::MachineryError.new(
-        "The following attributes are not covered by FileScope#compare_with: " +
-          unsupported.join(", ")
-      )
-    end
-  end
-
-  def compare_extracted(other, only_self, only_other, shared)
-    if extracted == other.extracted
-      shared.extracted = extracted
-    else
-      only_self.extracted = extracted
-      only_other.extracted = other.extracted
+    if only_self && only_other
+      changed = Machinery::Scope.extract_changed_elements(only_self, only_other, :name)
     end
-  end
-
-  def compare_files(other, only_self, only_other, shared)
-    own_files, other_files, changed, shared_files = files.compare_with(other.files)
-
-    only_self.files = own_files if own_files
-    only_other.files = other_files if other_files
-    shared.files = shared_files if shared_files
 
-    changed
+    [
+      only_self,
+      only_other,
+      changed,
+      common
+    ].map { |e| (e && !e.empty?) ? e : nil }
   end
 end

data/lib/file_validator.rb

@@ -56,16 +56,22 @@ class FileValidator
   def scope_extracted?(scope)
     if @format_version == 1
       @json_hash[scope] && ScopeFileStore.new(@base_path, scope.to_s).path
-    else
+    elsif @format_version < 6
       @json_hash[scope] && @json_hash[scope]["extracted"]
+    else
+      @json_hash[scope] && @json_hash[scope]["_attributes"]["extracted"]
     end
   end
 
   def expected_files(scope)
+    changes_proc = -> (file) { file["changes"] }
     if @format_version == 1
       files = @json_hash[scope]
-    else
+    elsif @format_version < 6
       files = @json_hash[scope]["files"]
+    else
+      changes_proc = -> (file) { file["changes"]}
+      files = @json_hash[scope]["_elements"]
     end
 
     if scope == "unmanaged_files"
@@ -79,8 +85,8 @@ class FileValidator
       expected_files += tree_tarballs
     else
       expected_files = files.reject do |file|
-          file["changes"].include?("deleted") || (file["type"] && file["type"] != "file")
-        end.map { |file| file["name"] }
+        changes_proc.call(file).include?("deleted") || (file["type"] && file["type"] != "file")
+      end.map { |file| file["name"] }
     end
 
     store_base_path = ScopeFileStore.new(@base_path, scope.to_s).path

data/lib/filter.rb

@@ -21,24 +21,24 @@
 # Filters are usually created by passing a filter definition string to the
 # constructor, e.g.
 #
-#   filter = Filter.new("/unmanaged_files/files/name=/opt")
+#   filter = Filter.new("/unmanaged_files/name=/opt")
 #
 # Existing filters can be extended by amending the definition:
 #
-#   filter.add_element_filter_from_definition("/unmanaged_files/files/name=/srv")
+#   filter.add_element_filter_from_definition("/unmanaged_files/name=/srv")
 #
 # or by adding ElementFilters directly:
 #
-#   element_filter = ElementFilter.new("/unmanaged_files/files/name", ["/opt", "/srv"])
+#   element_filter = ElementFilter.new("/unmanaged_files/name", ["/opt", "/srv"])
 #   filter.add_element_filter(element_filter)
 #
 #
 # The actual filtering can be done by passing values to Filter#matches?
 #
-#   filter = Filter.new("/unmanaged_files/files/name=/opt*")
-#   filter.matches?("/unmanaged_files/files/name", "/opt/foo")
+#   filter = Filter.new("/unmanaged_files/name=/opt*")
+#   filter.matches?("/unmanaged_files/name", "/opt/foo")
 #   => true
-#   filter.matches?("/unmanaged_files/files/name", "/srv/bar")
+#   filter.matches?("/unmanaged_files/name", "/srv/bar")
 #   => false
 #
 # More details about how the filter work can be found at

data/lib/filter_option_parser.rb

@@ -63,7 +63,7 @@ class FilterOptionParser
       files.reject!(&:empty?) # Ignore empty filters
       files.map! { |file| file.chomp("/") } # List directories without the trailing /, in order to
                                             # not confuse the unmanaged files inspector
-      files.map { |file| "/unmanaged_files/files/name=#{file}" }
+      files.map { |file| "/unmanaged_files/name=#{file}" }
     end
 
     def expand_filter_file(path)

data/lib/kiwi_config.rb

@@ -29,6 +29,7 @@ class KiwiConfig < Exporter
       "packages",
       "os"
     )
+    check_exported_os
     check_existance_of_extracted_files
     check_repositories
     generate_config
@@ -101,7 +102,7 @@ class KiwiConfig < Exporter
       output_root_path = File.join(output_location, "root")
       FileUtils.mkdir_p(output_root_path)
 
-      @system_description[scope].files.each do |file|
+      @system_description[scope].each do |file|
         if file.deleted?
           @sh << "rm -rf '#{quote(file.name)}'\n"
         elsif file.directory?
@@ -172,6 +173,15 @@ EOF
     end
   end
 
+  def check_exported_os
+    unless @system_description.os.is_a?(OsSuse)
+      raise Machinery::Errors::ExportFailed.new(
+        "Export is not possible because the operating system " \
+        "'#{@system_description.os.display_name}' is not supported."
+      )
+    end
+  end
+
   def generate_config
     @sh = <<EOF
 test -f /.kconfig && . /.kconfig
@@ -182,13 +192,6 @@ suseImportBuildKey
 suseConfig
 EOF
 
-    unless @system_description.os.is_a?(OsSuse)
-      raise Machinery::Errors::ExportFailed.new(
-        "Export is not possible because the operating system " \
-        "'#{@system_description.os.display_name}' is not supported."
-      )
-    end
-
     builder = Nokogiri::XML::Builder.new do |xml|
       xml.image(schemaversion: "5.8", name: @system_description.name) do
         xml.description(type: "system") do
@@ -292,7 +295,7 @@ EOF
 
       case init_system
         when "sysvinit"
-          @system_description["services"].services.each do |service|
+          @system_description["services"].each do |service|
             if service.state == "on"
               @sh << "chkconfig #{service.name} on\n"
             else
@@ -303,7 +306,7 @@ EOF
         when "systemd"
           # possible systemd service states:
           # http://www.freedesktop.org/software/systemd/man/systemctl.html#Unit%20File%20Commands
-          @system_description["services"].services.each do |service|
+          @system_description["services"].each do |service|
             case service.state
               when "enabled"
                 @sh << "systemctl enable #{service.name}\n"

data/lib/machinery.rb

@@ -109,7 +109,9 @@ require_relative "workload_mapper_dsl"
 require_relative "containerized_app"
 require_relative "move_task"
 require_relative "docker_system"
+require_relative "managed_files_database"
 require_relative "rpm_database"
+require_relative "dpkg_database"
 
 Dir[File.join(Machinery::ROOT, "plugins", "**", "*.rb")].each { |f| require(f) }
 

data/lib/machinery_helper.rb

@@ -52,7 +52,7 @@ class MachineryHelper
 
   def run_helper(scope)
     json = @system.run_command(remote_helper_path, stdout: :capture, stderr: STDERR)
-    scope.set_attributes(JSON.parse(json))
+    scope.insert(0, *JSON.parse(json)["files"])
   end
 
   def remove_helper

data/lib/managed_files_database.rb

@@ -0,0 +1,200 @@
+# Copyright (c) 2013-2015 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 3 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.   See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, contact SUSE LLC.
+#
+# To contact SUSE about this file by physical or electronic mail,
+# you may find current contact information at www.suse.com
+
+class ManagedFilesDatabase
+  class ChangedFile < Machinery::Object
+    attr_accessor :type
+
+    def initialize(type, attrs)
+      super(attrs)
+      @type = type
+    end
+
+    def config_file?
+      @type == "c"
+    end
+  end
+
+  def initialize(system)
+    @system = system
+  end
+
+  def expected_tag?(character, position)
+    if @rpm_changes[position] == character
+      true
+    else
+      @unknown_tag ||= ![".", "?"].include?(@rpm_changes[position])
+      false
+    end
+  end
+
+  def changed_files(&block)
+    return @changed_files if @changed_files
+    check_requirements
+
+    result = managed_files_list(&block).each_line.map do |line|
+      line.chomp!
+      next unless line =~ /^[^ ]+[ ]+. \/.*$/
+
+      file, changes, type = parse_changes_line(line)
+
+      package_name, package_version = package_for_file_path(file)
+
+      ChangedFile.new(
+        type,
+        name:            file,
+        package_name:    package_name,
+        package_version: package_version,
+        status:          "changed",
+        changes:         changes
+      )
+    end.compact.uniq
+
+    paths = result.reject { |f| f.changes == ["deleted"] }.map(&:name)
+    path_data = get_path_data(paths)
+    result.each do |pkg|
+      next unless path_data[pkg.name]
+
+      path_data[pkg.name].each do |key, value|
+        pkg[key] = value
+      end
+    end
+
+    @changed_files = result
+  end
+
+  def parse_changes_line(line)
+    # rpm provides lines per config file where first 9 characters indicate which
+    # properties of the file are modified
+    @rpm_changes, *fields = line.split(" ")
+    # nine rpm changes are known
+    @unknown_tag = @rpm_changes.size > 9
+
+    # For config or documentation files there's an additional field which
+    # contains "c" or "d"
+    type = fields[0].start_with?("/") ? "" : fields.shift
+    path = fields.join(" ")
+
+    changes = []
+    if @rpm_changes == "missing"
+      changes << "deleted"
+    elsif @rpm_changes == "........." && path.end_with?(" (replaced)")
+      changes << "replaced"
+      path.slice!(/ \(replaced\)$/)
+    else
+      changes << "size" if expected_tag?("S", 0)
+      changes << "mode" if expected_tag?("M", 1)
+      changes << "md5" if expected_tag?("5", 2)
+      changes << "device_number" if expected_tag?("D", 3)
+      changes << "link_path" if expected_tag?("L", 4)
+      changes << "user" if expected_tag?("U", 5)
+      changes << "group" if expected_tag?("G", 6)
+      changes << "time" if expected_tag?("T", 7)
+      changes << "capabilities" if @rpm_changes.size > 8 && expected_tag?("P", 8)
+    end
+
+    if @unknown_tag
+      changes << "other_rpm_changes"
+    end
+
+    handle_verify_fail(path) if @rpm_changes.include?("?")
+
+    [path, changes, type]
+  end
+
+  def parse_stat_line(line)
+    mode, user, group, uid, gid, type, *path_line = line.split(":")
+    path = path_line.join(":").chomp
+
+    user = uid if user == "UNKNOWN"
+    group = gid if group == "UNKNOWN"
+
+    type = case type
+           when "directory"
+             "dir"
+           when "symbolic link"
+             "link"
+           when /file$/
+             "file"
+           else
+             raise(
+               "The inspection failed because of the unknown type `#{type}` of file `#{path}`."
+             )
+    end
+
+    [path, {
+      mode:  mode,
+      user:  user,
+      group: group,
+      type:  type
+    }]
+  end
+
+  def get_link_target(link)
+    @system.run_command(
+      "find", link, "-prune", "-printf", "%l",
+      stdout:     :capture,
+      privileged: true
+    ).strip
+  end
+
+  # get path data for list of files
+  # cur_files is guaranteed to not exceed max command line length
+  def get_file_properties(cur_files)
+    ret = {}
+    out = @system.run_command(
+      "stat", "--printf", "%a:%U:%G:%u:%g:%F:%n\\n",
+      *cur_files,
+      stdout: :capture,
+      privileged: true
+    )
+    out.each_line do |l|
+      path, values       = parse_stat_line(l)
+      ret[path]          = values
+      ret[path][:target] = get_link_target(path) if values[:type] == "link"
+    end
+    ret
+  end
+
+  def get_path_data(paths)
+    ret = {}
+    path_index = 0
+    # arbitrary number for maximum command line length that should always work
+    max_len = 50000
+    cur_files = []
+    cur_len = 0
+    while path_index < paths.size
+      if cur_files.empty? || paths[path_index].size + cur_len + 1 < max_len
+        cur_files << paths[path_index]
+        cur_len += paths[path_index].size + 1
+        path_index += 1
+      else
+        ret.merge!(get_file_properties(cur_files))
+        cur_files.clear
+        cur_len = 0
+      end
+    end
+    ret.merge!(get_file_properties(cur_files)) unless cur_files.empty?
+    ret
+  end
+
+  def handle_verify_fail(path)
+    message = "Could not perform all tests on rpm changes for file '#{path}'."
+    Machinery.logger.warn(message)
+    Machinery::Ui.warn("Warning: #{message}")
+  end
+end