machinery-tool 1.6.0 → 1.7.0

data/lib/cli.rb

@@ -56,6 +56,16 @@ class Cli
 
   GLI::Commands::Help.skips_post = false
 
+  def self.buildable_distributions
+    distribution_string = ""
+    Os.supported_host_systems.each do |distribution|
+      distribution_string += "* #{distribution.canonical_name}\n\n"
+      distribution_string += distribution.buildable_systems.map(&:canonical_name).join(", ")
+      distribution_string += "\n\n"
+    end
+    distribution_string
+  end
+
   def self.handle_error(e)
     case e
     when GLI::UnknownCommandArgument, GLI::UnknownGlobalArgument,
@@ -116,6 +126,12 @@ class Cli
     Cli.handle_error(e)
   end
 
+  def self.show_filter_note(scopes, filter)
+    if scopes.any? { |scope| !filter.element_filters_for_scope(scope).empty? }
+      Machinery::Ui.puts "\nFilters are applied during inspection.\n\n"
+    end
+  end
+
   def self.shift_arg(args, name)
     if !res = args.shift
       raise GLI::BadCommandLine.new("You need to provide the required argument #{name}.")
@@ -187,7 +203,7 @@ class Cli
       )
     end
 
-    scopes.uniq
+    Inspector.sort_scopes(scopes.uniq)
   end
 
   AVAILABLE_SCOPE_LIST = Machinery::Ui.internal_scope_list_to_string(
@@ -232,6 +248,10 @@ class Cli
   long_desc <<-LONGDESC
     Build image from a given system description and store it to the given
     location.
+
+    The following combinations of build hosts and targets are supported:
+
+    #{buildable_distributions}
   LONGDESC
   arg "NAME"
   command :build do |c|
@@ -274,6 +294,10 @@ class Cli
       desc: "Exclude specified scopes", arg_name: "SCOPE_LIST"
     c.switch "show-all", required: false, negatable: false,
       desc: "Show also common properties"
+    if @config.experimental_features
+      c.switch "html", required: false, negatable: false,
+        desc: "Open comparison in HTML format in your web browser."
+    end
     c.switch "pager", required: false, default_value: true,
       desc: "Pipe output into a pager"
 
@@ -289,6 +313,7 @@ class Cli
 
       task = CompareTask.new
       opts = {
+        show_html: options["html"],
         show_all: options["show-all"]
       }
       task.compare(description1, description2, scope_list, opts)
@@ -427,6 +452,9 @@ class Cli
     c.flag "skip-files", required: false, negatable: false,
       desc: "Do not consider given files or directories during inspection. " \
         "Either provide one file or directory name or a list of names separated by commas."
+    c.flag ["remote-user", :r], type: String, required: false, default_value: @config.remote_user,
+      desc: "Defines the user which is used to access the inspected system via SSH."\
+        "This user needs sudo access on the remote machine or be root.", arg_name: "USER"
     c.switch ["extract-files", :x], required: false, negatable: false,
       desc: "Extract changed configuration files and unmanaged files from inspected system"
     c.switch "extract-changed-config-files", required: false, negatable: false,
@@ -456,6 +484,9 @@ class Cli
       if options["show"]
         inspect_options[:show] = true
       end
+      if options["verbose"]
+        inspect_options[:verbose] = true
+      end
       if options["extract-files"] || options["extract-changed-config-files"]
         inspect_options[:extract_changed_config_files] = true
       end
@@ -465,12 +496,15 @@ class Cli
       if options["extract-files"] || options["extract-unmanaged-files"]
         inspect_options[:extract_unmanaged_files] = true
       end
+      inspect_options[:remote_user] = options["remote-user"]
 
       filter = FilterOptionParser.parse("inspect", options, global_options)
 
       if options["verbose"] && !filter.empty?
         Machinery::Ui.puts "\nThe following filters are applied during inspection:"
         Machinery::Ui.puts filter.to_array.join("\n") + "\n\n"
+      else
+        show_filter_note(scope_list, filter)
       end
 
       inspector_task.inspect_system(
@@ -573,7 +607,7 @@ class Cli
 
       inspected_filters = description.filter_definitions("inspect")
 
-      if options["verbose"]
+      if options[:verbose]
         if !inspected_filters.empty?
           Machinery::Ui.puts "\nThe following filters were applied during inspection:"
           Machinery::Ui.puts inspected_filters.join("\n") + "\n\n"

data/lib/compare_task.rb

@@ -17,9 +17,36 @@
 
 class CompareTask
   def compare(description1, description2, scopes, options = {})
-    output = render_comparison(description1, description2, scopes, options)
+    if options[:show_html]
+      render_html_comparison(description1, description2, scopes, options)
+    else
+      output = render_comparison(description1, description2, scopes, options)
 
-    Machinery::Ui.puts output
+      Machinery::Ui.puts output
+    end
+  end
+
+  def render_html_comparison(description1, description2, scopes, options)
+    diff = {
+      meta: {
+        description_a: description1.name,
+        description_b: description2.name,
+      }
+    }
+
+    scopes.each do |scope|
+      if description1[scope] && description2[scope]
+        comparison = description1[scope].compare_with(description2[scope])
+        diff[scope] = comparison.map { |scope| scope.as_json if scope }
+      end
+    end
+
+    target = "/tmp/machinery-html-comparison"
+    FileUtils.rm_r(target) if Dir.exists?(target)
+    FileUtils.mkdir_p(target)
+
+    Html.generate_comparison(diff, target)
+    LoggedCheetah.run("xdg-open", File.join(target, "index.html"))
   end
 
   def render_comparison(description1, description2, scopes, options = {})

data/lib/config.rb

@@ -29,10 +29,14 @@ module Machinery
         description: "Show hints about usage of Machinery in the context of the commands ran by" \
           " the user"
       )
+      entry("remote-user",
+        default: "root",
+        description: "Defines the user which is used to access the inspected system via SSH"
+      )
       entry("experimental-features",
         default: false,
         description: "Enable experimental features. See " \
-          "https://github.com/SUSE/machinery/wiki/Experimental-Features for more details."
+          "https://github.com/SUSE/machinery/wiki/Experimental-Features for more details"
       )
     end
   end

data/lib/config_task.rb

@@ -45,7 +45,10 @@ class ConfigTask
       elsif value_string == "false" || value_string == "off"
         return false
       else
-        raise Machinery::Errors::MachineryError.new("The value '#{value_string}' is not valid for key '#{key}'.")
+        raise Machinery::Errors::MachineryError.new(
+          "The value '#{value_string}' is not valid for key '#{key}'." \
+            " Please enter a valid variable of type boolean."
+        )
       end
     elsif current_value.kind_of?(Integer)
       return value_string.to_i

data/lib/export_task.rb

@@ -36,7 +36,12 @@ class ExportTask
       end
     end
 
-    FileUtils.mkdir_p(output_dir, mode: 0700) if !Dir.exists?(output_dir)
+    begin
+      FileUtils.mkdir_p(output_dir, mode: 0700) if !Dir.exists?(output_dir)
+    rescue Errno::EACCES
+      raise(Machinery::Errors::ExportFailed, \
+            "Permission denied. Directory '#{output_dir}' is not writable")
+    end
 
     if @exporter.system_description["unmanaged_files"]
       filters = File.read(

data/lib/filter.rb

@@ -54,6 +54,7 @@ class Filter
     Array(filter_definitions).each do |definition|
       path, operator, matcher_definition = definition.scan(/([a-zA-Z_\/]+)(.*=)(.*)/)[0]
 
+      raise Machinery::Errors::InvalidFilter.new("Invalid filter: '#{definition}'") if !operator
       element_filters[path] ||= ElementFilter.new(path)
       if matcher_definition.index(",")
         matchers = matcher_definition.split(/(?<!\\),/)

data/lib/filter_option_parser.rb

@@ -38,13 +38,10 @@ class FilterOptionParser
     def exclude_definitions(exclude)
       return [] if !exclude
 
-      filters = exclude.scan(/(@[^,]+)|\"([^,]+?=[^\"]+)\"|([^,]+=[^=]+)$|([^,]+=[^,]+)/).
-        map(&:compact).flat_map do |filter_definition|
-        if filter_definition[0].start_with?("@")
-          expand_filter_file(filter_definition[0])
-        else
-          filter_definition
-        end
+      filters = if exclude.start_with?("@")
+        expand_filter_file(exclude)
+      else
+        [exclude]
       end
 
       filters.reject!(&:empty?) # Ignore empty filters

data/lib/helper.rb

@@ -19,6 +19,18 @@ module Machinery
   def self.is_int?(string)
     (string =~ /^\d+$/) != nil
   end
+
+  # Implementation of String#scrub for Ruby < 2.1. Assumes the string is in
+  # UTF-8.
+  def self.scrub(s)
+    # We have a string in UTF-8 with possible invalid byte sequences. It turns
+    # out that String#encode can remove these sequences when given appropriate
+    # options, but just converting into UTF-8 would be a no-op. So let's convert
+    # into UTF-16 (which has the same character set as UTF-8) and back.
+    #
+    # See also: http://stackoverflow.com/a/21315619
+    s.dup.force_encoding("UTF-8").encode("UTF-16", invalid: :replace).encode("UTF-8")
+  end
 end
 
 def with_c_locale(&block)

data/lib/html.rb

@@ -16,6 +16,10 @@
 # you may find current contact information at www.suse.com
 
 class Html
+  def self.escape_javascript(object)
+    object.gsub(/([\\'"])/, "\\\\\\1")
+  end
+
   def self.generate(description)
     template = Haml::Engine.new(
       File.read(File.join(Machinery::ROOT, "html", "index.html.haml"))
@@ -36,7 +40,7 @@ class Html
 
     # Generate JSON and escape the 's and "s in order to not break the JSON
     # string in javascript.
-    json = description.to_hash.to_json.gsub("'", "\\\\'").gsub("\"", "\\\\\"")
+    json = escape_javascript(description.to_hash.to_json)
     File.write(File.join(target, "assets/description.js"),<<-EOT
       function getDescription() {
         return JSON.parse('#{json}'
@@ -47,14 +51,33 @@ class Html
     target
   end
 
+  def self.generate_comparison(diff, target)
+    FileUtils.mkdir_p(File.join(target, "assets"))
+    template = Haml::Engine.new(
+      File.read(File.join(Machinery::ROOT, "html", "comparison.html.haml"))
+    )
+
+    FileUtils.cp_r(File.join(Machinery::ROOT, "html", "assets"), target)
+    File.write(File.join(target, "index.html"), template.render(binding))
+    json = diff.to_json.gsub("'", "\\\\'").gsub("\"", "\\\\\"")
+    File.write(File.join(target, "assets/diff.js"),<<-EOT
+      function getDiff() {
+        return JSON.parse('#{json}'
+        )
+      }
+      EOT
+    )
+  end
+
   # Template helpers
 
   def self.scope_help(scope)
-    text = File.read(File.join(Machinery::ROOT, "plugins", "docs", "#{scope}.md"))
+    text = File.read(File.join(Machinery::ROOT, "plugins", "#{scope}/#{scope}.md"))
     Kramdown::Document.new(text).to_html
   end
 
   def self.diff_to_object(diff)
+    diff = Machinery.scrub(diff)
     lines = diff.lines[2..-1]
     diff_object = {
       file: diff[/--- a(.*)/, 1],

data/lib/inspect_task.rb

@@ -17,7 +17,7 @@
 
 class InspectTask
   def inspect_system(store, host, name, current_user, scopes, filter, options = {})
-    system = System.for(host)
+    system = System.for(host, options[:remote_user])
     check_root(system, current_user)
 
     description, failed_inspections = build_description(store, name, system,

data/lib/inspector.rb

@@ -38,6 +38,14 @@ class Inspector
   @inspectors = []
 
   class << self
+    def priority
+      @priority || 1000
+    end
+
+    def has_priority(value)
+      @priority = value
+    end
+
     def inherited(klass)
       @inspectors << klass
     end
@@ -53,7 +61,19 @@ class Inspector
     end
 
     def all_scopes
-      all.map(&:scope)
+      sort_scopes(all.map(&:scope))
+    end
+
+    def sort_scopes(scope_list)
+      scope_priority = {}
+
+      scope_list.each do |scope|
+        scope_priority[self.for(scope).priority] = scope
+      end
+
+      scope_priority.sort.map do |key, value|
+        scope_priority[key] = value
+      end
     end
 
     def scope

data/lib/json_validator.rb

@@ -61,22 +61,23 @@ class JsonValidator
     JSON.parse(File.read(File.join(
       Machinery::ROOT,
       "schema",
-      "v#{format_version}",
-      "system-description-global.schema.json"
+      "system-description-global.schema-v#{format_version}.json"
     )))
   end
 
   def scope_schemas(format_version = SystemDescription::CURRENT_FORMAT_VERSION)
-    schema_dir = File.join(
+    schema_path = File.join(
       Machinery::ROOT,
       "plugins",
+      "**",
       "schema",
-      "v#{format_version}",
+      "*.schema-v#{format_version}.json"
     )
 
     Hash[
-      Dir["#{schema_dir}/*.schema.json"].map do |file|
-        scope = file.match(/system-description-(.*)\.schema\.json$/)[1].tr("-", "_")
+      Dir[schema_path].map do |file|
+        scope = file.match(/system-description-(.*)\.schema-v#{format_version}\.json$/)[1].
+          tr("-", "_")
         schema = JSON.parse(File.read(file))
 
         [scope, schema]

data/lib/machinery.rb

@@ -69,7 +69,7 @@ require_relative "helper"
 require_relative "deploy_task"
 require_relative "analyze_config_file_diffs_task"
 require_relative "copy_task"
-require_relative "scope_mixin"
+require_relative "scope"
 require_relative "ui"
 require_relative "validate_task"
 require_relative "migration"
@@ -90,6 +90,7 @@ require_relative "file_validator"
 require_relative "element_filter"
 require_relative "filter"
 require_relative "filter_option_parser"
+require_relative "file_scope"
 
 Dir[File.join(Machinery::ROOT, "plugins", "**", "*.rb")].each { |f| require(f) }
 

data/lib/manifest.rb

@@ -46,7 +46,7 @@ class Manifest
     errors = JsonValidator.new(@hash).validate
     if !errors.empty?
       Machinery::Ui.warn("Warning: System Description validation errors:")
-      Machinery::Ui.warn(errors)
+      Machinery::Ui.warn(errors.join("\n"))
     end
   end
 

data/lib/migration.rb

@@ -73,7 +73,7 @@ class Migration
       if !errors.empty?
         if options[:force]
           Machinery::Ui.warn("Warning: System Description validation errors:")
-          Machinery::Ui.warn(errors)
+          Machinery::Ui.warn(errors.join(", "))
         else
           raise Machinery::Errors::SystemDescriptionValidationFailed.new(errors)
         end

data/lib/remote_system.rb

@@ -17,9 +17,11 @@
 
 class RemoteSystem < System
   attr_accessor :host
+  attr_accessor :remote_user
 
-  def initialize(host)
+  def initialize(host, remote_user = "root")
     @host = host
+    @remote_user = remote_user
 
     connect
   end
@@ -70,19 +72,21 @@ class RemoteSystem < System
       cheetah_class = LoggedCheetah
     end
 
-    cheetah_class.run("ssh", "root@#{host}", "LC_ALL=C", *piped_args, options)
+    sudo = "sudo" if options[:privileged] && remote_user != "root"
+    cmds = ["ssh", "#{remote_user}@#{host}", sudo, "LC_ALL=C", *piped_args, options].compact
+    cheetah_class.run(*cmds)
   end
 
   # Tries to connect to the remote system as root (without a password or passphrase)
   # and raises an Machinery::Errors::SshConnectionFailed exception when it's not successful.
   def connect
-    LoggedCheetah.run "ssh", "-q", "-o", "BatchMode=yes", "root@#{host}"
+    LoggedCheetah.run "ssh", "-q", "-o", "BatchMode=yes", "#{remote_user}@#{host}"
   rescue Cheetah::ExecutionFailed
     raise Machinery::Errors::SshConnectionFailed.new(
       "Could not establish SSH connection to host '#{host}'. Please make sure that " \
-      "you can connect non-interactively as root, e.g. using ssh-agent.\n\n" \
+      "you can connect non-interactively as #{remote_user}, e.g. using ssh-agent.\n\n" \
       "To copy your default ssh key to the machine run:\n" \
-      "ssh-copy-id root@#{host}"
+      "ssh-copy-id #{remote_user}@#{host}"
     )
   end
 
@@ -91,9 +95,26 @@ class RemoteSystem < System
   # Machinery::Errors::RsyncFailed exception when it's not successful. Destination is
   # the directory where to put the files.
   def retrieve_files(filelist, destination)
-    source="root@#{host}:/"
+    source = "#{remote_user}@#{host}:/"
+    if remote_user != "root"
+      rsync_path = "sudo rsync"
+    else
+      rsync_path = "rsync"
+    end
+
+    cmd = [
+      "rsync",
+      "-e", "ssh",
+      "--chmod=go-rwx",
+      "--files-from=-",
+      "--rsync-path=#{rsync_path}",
+      source,
+      destination,
+      stdout: :capture,
+      stdin: filelist.join("\n")
+    ]
     begin
-      LoggedCheetah.run("rsync", "-e", "ssh", "--chmod=go-rwx", "--files-from=-", source, destination, :stdout => :capture, :stdin => filelist.join("\n") )
+      LoggedCheetah.run(*cmd)
     rescue Cheetah::ExecutionFailed  => e
       raise Machinery::Errors::RsyncFailed.new(
       "Could not rsync files from host '#{host}'.\n" \
@@ -104,7 +125,7 @@ class RemoteSystem < System
 
   # Reads a file from the System. Returns nil if it does not exist.
   def read_file(file)
-    run_command("cat", file, stdout: :capture)
+    run_command("cat", file, stdout: :capture, privileged: true)
   rescue Cheetah::ExecutionFailed => e
     if e.status.exitstatus == 1
       # File not found, return nil