machinery-tool 1.11.2 → 1.12.0

data/lib/system.rb

@@ -27,6 +27,8 @@ class System
   abstract_method :kiwi_describe
   abstract_method :retrieve_files
   abstract_method :read_file
+  abstract_method :inject_file
+  abstract_method :remove_file
 
   def self.for(host, remote_user = "root")
     if host && host != "localhost"
@@ -90,4 +92,8 @@ class System
   rescue Cheetah::ExecutionFailed
     false
   end
+
+  def arch
+    run_command("uname", "-m", stdout: :capture).chomp
+  end
 end

data/lib/system_description.rb

@@ -269,4 +269,30 @@ class SystemDescription < Machinery::Object
   def description_path
     @store.description_path(name)
   end
+
+  def runs_service?(name)
+    self["services"].services.any? { |service| service.name == "#{name}.service" }
+  end
+
+  def has_file?(name)
+    self["config_files"].files.any? { |file| file.name == name } ||
+      self["unmanaged_files"].files.any? { |file| file.name == name }
+  end
+
+  def read_config(path, key)
+    if scope_extracted?("config_files")
+      file = self["config_files"].files.find { |f| f.name == path }
+      return parse_variable_assignment(file.content, key) if file
+    end
+    if scope_extracted?("unmanaged_files")
+      file = self["unmanaged_files"].files.find { |f| f.name == path }
+      return parse_variable_assignment(file.content, key) if file
+    end
+  end
+
+  private
+
+  def parse_variable_assignment(string, variable)
+    /^(?#) *#{variable} *(=|:| ) *(.*)/.match(string).to_a.fetch(2, "").gsub(/\"/, "").strip
+  end
 end

data/lib/version.rb

@@ -17,6 +17,6 @@
 
 module Machinery
 
-  VERSION = "1.11.2"
+  VERSION = "1.12.0"
 
 end

data/lib/workload_mapper.rb

@@ -0,0 +1,141 @@
+# Copyright (c) 2013-2015 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 3 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.   See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, contact SUSE LLC.
+#
+# To contact SUSE about this file by physical or electronic mail,
+# you may find current contact information at www.suse.com
+class WorkloadMapper
+  def save(workloads, path)
+    workloads.each do |workload, config|
+      FileUtils.mkdir_p(File.join(path, workload))
+      FileUtils.cp_r(
+        File.join(workload_mapper_path, workload, "container", "."),
+        File.join(path, workload)
+      )
+    end
+    compose_services = compose_services(workloads)
+    linked_services = link_compose_services(compose_services)
+    File.write(File.join(path, "docker-compose.yml"), linked_services.to_yaml)
+  end
+
+  def link_compose_services(services)
+    services.each do |service, config|
+      config.fetch("links", {}).each do |linked_service|
+        services[service]["environment"] ||= {}
+        if services[linked_service]
+          vars = services[linked_service].fetch("environment", {})
+          services[service]["environment"].merge!(vars)
+        else
+          raise Machinery::Errors::ComposeServiceLink.new(
+            "Could not detect '#{linked_service}', which is referenced by '#{service}'."\
+          )
+        end
+      end
+    end
+  end
+
+  def compose_service(workload, config)
+    name = config["service"]
+    service = {
+      name => compact(load_compose_template(workload))
+    }
+    fill_in_template(service[name], config["parameters"])
+    service
+  end
+
+  def load_compose_template(workload)
+    template = YAML::load(
+      File.read(File.join(workload_mapper_path, workload, "compose-template.yml"))
+    )
+    template[workload]
+  end
+
+  def identify_workloads(system_description)
+    system_description.assert_scopes("services", "unmanaged_files", "config_files")
+
+    ["unmanaged_files", "config_files"].each do |scope|
+      if !system_description.scope_extracted?(scope)
+        raise Machinery::Errors::SystemDescriptionError.new(
+          "Required scope: '#{scope}' was not extracted. Can't continue."
+        )
+      end
+    end
+
+    workloads = {}
+
+    Dir["#{File.expand_path(workload_mapper_path)}/*"].each do |workload_dir|
+      mapper = WorkloadMapperDSL.new(system_description)
+      workload = mapper.check_clue(File.read(File.join(workload_dir, "clue.rb")))
+      workloads.merge!(workload.to_h)
+    end
+    workloads
+  end
+
+  def fill_in_template(service, parameters)
+    service.each do |key, value|
+      if value.is_a?(Hash)
+        fill_in_template(value, parameters)
+      elsif value.is_a?(Symbol)
+        service[key] = parameters[value.to_s]
+      end
+    end
+  end
+
+  def extract(system_description, workloads, path)
+    Dir.mktmpdir do |dir|
+      if !workloads.select { |_, w| w["data"] }.empty?
+        system_description.unmanaged_files.export_files_as_tarballs(dir)
+        workloads.each do |workload, config|
+          config.fetch("data", {}).each do |origin, destination|
+            file = system_description.unmanaged_files.files.find { |f| f.name == origin }
+            file ||= system_description.config_files.files.find { |f| f.name == origin }
+            if file && file.directory?
+              tgz_file = File.join(dir, "trees", "#{origin.chop}.tgz")
+              output_path = File.join(path, workload, destination)
+              FileUtils.mkdir_p(output_path)
+              Cheetah.run("tar", "zxf", tgz_file, "-C", output_path, "--strip=1")
+              copy_workload_config_files(workload, output_path)
+            end
+            if file && file.file?
+              output_path = File.join(path, workload, destination, origin)
+              FileUtils.mkdir_p(File.dirname(output_path))
+              File.write(output_path, file.content)
+            end
+          end
+        end
+      end
+    end
+  end
+
+  private
+
+  def copy_workload_config_files(workload, path)
+    FileUtils.cp_r(File.join(workload_mapper_path, workload, "config", "."), path)
+  end
+
+  def workload_mapper_path
+    File.join(Machinery::ROOT, "workload_mapper")
+  end
+
+  def compact(service)
+    service.each { |_, attr| attr.is_a?(Hash) && attr.reject! { |_, val| val.nil? } }
+  end
+
+  def compose_services(workloads)
+    compose_services = {}
+    workloads.each do |workload, config|
+      compose_services.merge!(compose_service(workload, config))
+    end
+    compose_services
+  end
+end

data/lib/workload_mapper_dsl.rb

@@ -0,0 +1,54 @@
+# Copyright (c) 2013-2015 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 3 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.   See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, contact SUSE LLC.
+#
+# To contact SUSE about this file by physical or electronic mail,
+# you may find current contact information at www.suse.com
+class WorkloadMapperDSL
+  attr_reader :system, :name, :service, :parameters, :data
+
+  def initialize(system)
+    @system = system
+    @parameters = {}
+    @data = {}
+  end
+
+  def identify(name, service = nil)
+    @name = name
+    @service = !!service ? service : name
+  end
+
+  def parameter(name, value)
+    @parameters[name] = value
+  end
+
+  def extract(origin, destination)
+    @data[origin] = destination
+  end
+
+  def to_h
+    return {} unless service
+    {
+      name => {
+        "service" => service,
+        "parameters" => parameters,
+        "data" => data
+      }
+    }
+  end
+
+  def check_clue(clue)
+    instance_eval(clue)
+    self
+  end
+end

data/man/generated/machinery.1.html

@@ -867,6 +867,13 @@ via <code>ssh-copy-id</code> to the inspected host, e.g.: <code>ssh-copy-id root
 whitelist given in the sudoers file:</p>
 
 <p>machinery ALL=(ALL) NOPASSWD: /usr/bin/find,/usr/bin/cat,/bin/cat,/usr/bin/rsync,/bin/rpm -V *,/bin/tar --create *</p></li>
+<li><p>To add a remote <code>machinery</code> user run as root:</p>
+
+<h1><code>useradd -m machinery -c "remote user for machinery"</code></h1>
+
+<p>To configure a password for the new user run:</p>
+
+<h1><code>passwd machinery</code></h1></li>
 </ul>
 
 
@@ -1198,7 +1205,7 @@ manually editing it.</p>
 
   <ol class='man-decor man-foot man foot'>
     <li class='tl'></li>
-    <li class='tc'>July 2015</li>
+    <li class='tc'>September 2015</li>
     <li class='tr'>machinery(1)</li>
   </ol>
 

data/plugins/os/os_inspector.rb

@@ -18,10 +18,6 @@
 #  Inspect name, version, and other attributes of the operating system
 class OsInspector < Inspector
   has_priority 10
-  # determines the architecture
-  def get_arch
-    @system.run_command("uname", "-m", :stdout => :capture).chomp
-  end
 
   def strip_arch_from_name(name)
     # architecture information in the name might be misleading
@@ -49,7 +45,7 @@ class OsInspector < Inspector
 
     os = get_os
     if os
-      os.architecture = get_arch
+      os.architecture = @system.arch
       os.version += get_additional_version if os.version
     else
       raise Machinery::Errors::UnknownOs

data/plugins/repositories/repositories_inspector.rb

@@ -84,6 +84,10 @@ class RepositoriesInspector < Inspector
       end
     rescue JSON::ParserError
       raise Machinery::Errors::InspectionFailed.new("Extraction of YUM repositories failed.")
+    rescue Cheetah::ExecutionFailed => e
+      raise Machinery::Errors::InspectionFailed.new(
+        "Extraction of YUM repositories failed:\n#{e.stderr}"
+      )
     end
 
     RepositoriesScope.new(repositories)

data/plugins/unmanaged_files/unmanaged_files_inspector.rb

@@ -53,6 +53,17 @@ class UnmanagedFilesInspector < Inspector
         files[pair.first] = pair[1]
       end
     end
+    # make sure that all parent directories of managed rpm directories are considered
+    # managed
+    dirh.dup.keys.each do |d|
+      dir = d.rpartition("/").first
+
+      while !dirh.has_key?(dir) && dir.size > 1
+        dirh[dir] = false
+        dir = dir[0..dir.rindex("/") - 1]
+      end
+    end
+
     files.each do |f,e|
       dir, sep, file = f.rpartition("/")
 
@@ -97,6 +108,12 @@ class UnmanagedFilesInspector < Inspector
           os = osl.find do |o|
             o.name == "/#{file[:path]}#{file[:type] == :dir ? "/" : ""}"
           end
+          if !os
+            raise Machinery::Errors::UnexpectedInputData.new(
+              "The inspection failed because of the unexpected input data:\n#{file.inspect}\n\n" \
+                "Please file a bug report at: https://github.com/SUSE/machinery/issues/new"
+            )
+          end
 
           os.user = file[:user]
           os.group = file[:group]
@@ -144,7 +161,6 @@ class UnmanagedFilesInspector < Inspector
       Machinery::Ui.warn(message)
     end
 
-
     # find creates three field per path
     out.split("\0", -1).each_slice(3) do |type, raw_path, raw_link|
       next unless raw_path && !raw_path.empty?
@@ -213,6 +229,57 @@ class UnmanagedFilesInspector < Inspector
 
     scope.scope_file_store = file_store_tmp
 
+    file_filter = filter.element_filter_for("/unmanaged_files/files/name").dup if filter
+    file_filter ||= ElementFilter.new("/unmanaged_files/files/name")
+    file_filter.add_matchers("=", @description.store.base_path)
+
+    # Add a recursive pendant to each ignored element
+    file_filter.matchers.each do |operator, matchers|
+      file_filter.add_matchers(operator, matchers.map { |entry| File.join(entry, "/*") })
+    end
+
+    helper = MachineryHelper.new(@system)
+    if helper_usable?(helper, options)
+      begin
+        helper.inject_helper
+        helper.run_helper(scope)
+      ensure
+        helper.remove_helper
+      end
+      scope.extracted = false
+
+      scope.files.delete_if { |f| file_filter.matches?(f.name) }
+
+      @description["unmanaged_files"] = scope
+    else
+      run_inspection(file_filter, options, do_extract, file_store_tmp, file_store_final, scope)
+    end
+  end
+
+  def helper_usable?(helper, options)
+    if !helper.can_help?
+      Machinery::Ui.puts(
+        "Note: Using traditional inspection because there is no helper binary for" \
+        " architecture '#{@system.arch}' available."
+      )
+    elsif options[:extract_unmanaged_files]
+      Machinery::Ui.puts(
+        "Note: Using traditional inspection because file extraction is not" \
+        " supported by the helper binary."
+      )
+    elsif options[:remote_user] && options[:remote_user] != "root"
+      Machinery::Ui.puts(
+        "Note: Using traditional inspection because only 'root' is supported as remote user."
+      )
+    else
+      Machinery::Ui.puts "Note: Using helper binary for inspection of unmanaged files."
+      return true
+    end
+
+    false
+  end
+
+  def run_inspection(file_filter, options, do_extract, file_store_tmp, file_store_final, scope)
     mount_points = MountPoints.new(@system)
 
     rpm_files, rpm_dirs = extract_rpm_database
@@ -228,14 +295,6 @@ class UnmanagedFilesInspector < Inspector
     remote_dirs = mount_points.remote
     special_dirs = mount_points.special
 
-    file_filter = filter.element_filter_for("/unmanaged_files/files/name").dup if filter
-    file_filter ||= ElementFilter.new("/unmanaged_files/files/name")
-    file_filter.add_matchers("=", @description.store.base_path)
-
-    # Add a recursive pendant to each ignored element
-    file_filter.matchers.each do |operator, matchers|
-      file_filter.add_matchers(operator, matchers.map { |entry| File.join(entry, "/*") })
-    end
 
     remote_dirs.delete_if { |e| file_filter.matches?(e) }
 
@@ -335,6 +394,18 @@ class UnmanagedFilesInspector < Inspector
       Machinery::Ui.progress(progress)
     end
     Machinery.logger.debug "inspect unmanaged files find calls:#{find_count} files:#{unmanaged_files.size} trees:#{unmanaged_trees.size}"
+
+    processed_files = run_extraction(unmanaged_files, unmanaged_trees, unmanaged_links,
+      excluded_files, remote_dirs, do_extract, file_store_tmp, file_store_final, scope)
+
+    scope.extracted = !!do_extract
+    scope.files = UnmanagedFileList.new(processed_files.sort_by(&:name))
+
+    @description["unmanaged_files"] = scope
+  end
+
+  def run_extraction(unmanaged_files, unmanaged_trees, unmanaged_links, excluded_files, remote_dirs,
+      do_extract, file_store_tmp, file_store_final, scope)
     begin
       if do_extract
         file_store_tmp.remove
@@ -374,10 +445,7 @@ class UnmanagedFilesInspector < Inspector
       osl << UnmanagedFile.new( name: remote_dir + "/", type: "remote_dir")
     end
 
-    scope.extracted = !!do_extract
-    scope.files = UnmanagedFileList.new(osl.sort_by(&:name))
-
-    @description["unmanaged_files"] = scope
+    osl
   end
 
   def summary