machinery-tool 1.13.0 → 1.14.0

data/lib/machinery.rb

@@ -107,6 +107,8 @@ require_relative "workload_mapper"
 require_relative "containerize_task"
 require_relative "workload_mapper_dsl"
 require_relative "containerized_app"
+require_relative "move_task"
+require_relative "docker_system"
 
 Dir[File.join(Machinery::ROOT, "plugins", "**", "*.rb")].each { |f| require(f) }
 

data/lib/machinery_helper.rb

@@ -22,45 +22,48 @@
 # The inspection checks, if a binary helper is available on the machine where
 # the inspection is started. It looks at the location
 #
-#    /usr/share/machinery/helpers/<arch>/machinery-helper
-#
-# where <arch> is the hardware architecture of the target system. Valid values
-# are x86_64, i586, s390x, and ppcle.
+# <machinery-installation-path>/machinery-helper/machinery-helper
 
 class MachineryHelper
   attr_accessor :local_helpers_path
+  attr_accessor :remote_helper_path
 
   def initialize(s)
     @system = s
     @arch = @system.arch
 
-    @local_helpers_path = "/usr/share/machinery/helpers"
+    @local_helpers_path = File.join(Machinery::ROOT, "machinery-helper")
+    @remote_helper_path = File.join(Machinery::HELPER_REMOTE_PATH, "machinery-helper")
   end
 
   def local_helper_path
-    File.join(@local_helpers_path, @arch, "machinery-helper")
+    File.join(local_helpers_path, "machinery-helper")
   end
 
   # Returns true, if there is a helper binary matching the architecture of the
   # inspected system. Return false, if not.
   def can_help?
-    File.exist?(local_helper_path)
+    File.exist?(local_helper_path) && LocalSystem.matches_architecture?(@arch)
   end
 
   def inject_helper
-    @system.inject_file(local_helper_path, Machinery::REMOTE_HELPERS_PATH)
+    @system.inject_file(local_helper_path, Machinery::HELPER_REMOTE_PATH)
   end
 
   def run_helper(scope)
-    json = @system.run_command(
-      File.join(
-        Machinery::REMOTE_HELPERS_PATH, "machinery-helper"
-      ), stdout: :capture, stderr: STDERR
-    )
+    json = @system.run_command(remote_helper_path, stdout: :capture, stderr: STDERR)
     scope.set_attributes(JSON.parse(json))
   end
 
   def remove_helper
-    @system.remove_file(File.join(Machinery::REMOTE_HELPERS_PATH, "machinery-helper"))
+    @system.remove_file(remote_helper_path)
+  end
+
+  def has_compatible_version?
+    output = @system.run_command(remote_helper_path, "--version", stdout: :capture).chomp
+
+    version = output[/^Version: ([a-f0-9]{40})$/, 1]
+
+    version == File.read(File.join(Machinery::ROOT, ".git_revision"))
   end
 end

data/lib/move_task.rb

@@ -0,0 +1,22 @@
+# Copyright (c) 2013-2015 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 3 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.   See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, contact SUSE LLC.
+#
+# To contact SUSE about this file by physical or electronic mail,
+# you may find current contact information at www.suse.com
+
+class MoveTask
+  def move(store, from, to)
+    store.move(from, to)
+  end
+end

data/lib/remote_system.rb

@@ -19,6 +19,10 @@ class RemoteSystem < System
   attr_accessor :host
   attr_accessor :remote_user
 
+  def type
+    "remote"
+  end
+
   def initialize(host, remote_user = "root")
     @host = host
     @remote_user = remote_user

data/lib/scope_file_access_archive.rb

@@ -32,6 +32,16 @@ module ScopeFileAccessArchive
     end
   end
 
+  def has_file?(name)
+    return true if files.any? { |file| file.name == name }
+    if files.any? { |file| file.name == File.join(File.dirname(name), "") }
+      tgz_file = File.join(scope_file_store.path, "trees", "#{File.dirname(name)}.tgz")
+      return Cheetah.run("tar", "ztf", tgz_file, stdout: :capture).split(/\n/).
+        any? { |f| "/#{f}" == name }
+    end
+    false
+  end
+
   def tarball_path(system_file)
     if system_file.directory?
       File.join(

data/lib/scope_file_access_flat.rb

@@ -33,4 +33,8 @@ module ScopeFileAccessFlat
     output = Cheetah.run("file", path, stdout: :capture)
     !output.include?("ASCII")
   end
+
+  def has_file?(name)
+    return true if files.any? { |file| file.name == name }
+  end
 end

data/lib/show_task.rb

@@ -30,7 +30,7 @@ class ShowTask
 
   def show_html(description, options)
     begin
-      LocalSystem.validate_existence_of_package("xdg-utils")
+      LocalSystem.validate_existence_of_command("xdg-open", "xdg-utils")
 
       url = "http://#{options[:ip]}:#{options[:port]}/#{CGI.escape(description.name)}"
 

data/lib/system.rb

@@ -29,6 +29,7 @@ class System
   abstract_method :read_file
   abstract_method :inject_file
   abstract_method :remove_file
+  abstract_method :type
 
   attr_writer :locale
 
@@ -55,10 +56,22 @@ class System
     )
   end
 
+  def check_retrieve_files_dependencies
+    check_requirement("rsync", "--version")
+  end
+
+  def check_create_archive_dependencies
+    check_requirement("tar", "--version")
+    check_requirement("gzip", "--version")
+  end
+
   # Retrieves files specified in filelist from the remote system and create an archive.
   # To be able to deal with arbitrary filenames we use zero-terminated
   # filelist and the --null option of tar
   def create_archive(file_list, archive, exclude = [])
+    Machinery.logger.info(
+      "The following files are packaged in #{archive}: " + Array(file_list).join(", ")
+    )
     created = !File.exists?(archive)
     out = File.open(archive, "w")
     begin
@@ -67,7 +80,8 @@ class System
         *exclude.flat_map { |f| ["--exclude", f]},
         stdout: out,
         stdin: Array(file_list).join("\0"),
-        privileged: true
+        privileged: true,
+        disable_logging: true
       )
     rescue Cheetah::ExecutionFailed => e
       if e.status.exitstatus == 1
@@ -83,7 +97,7 @@ class System
   end
 
   def run_script(*args)
-    script = File.read(File.join(Machinery::ROOT, "helpers", args.shift))
+    script = File.read(File.join(Machinery::ROOT, "inspect_helpers", args.shift))
 
     run_command("bash", "-c", script, *args)
   end

data/lib/system_description.rb

@@ -276,19 +276,29 @@ class SystemDescription < Machinery::Object
   end
 
   def has_file?(name)
-    self["config_files"].files.any? { |file| file.name == name } ||
-      self["unmanaged_files"].files.any? { |file| file.name == name }
+    EXTRACTABLE_SCOPES.each do |scope|
+      if scope_extracted?(scope)
+        return true if self[scope] && self[scope].has_file?(name)
+      end
+    end
+    false
   end
 
   def read_config(path, key)
-    if scope_extracted?("config_files")
-      file = self["config_files"].files.find { |f| f.name == path }
-      return parse_variable_assignment(file.content, key) if file
-    end
-    if scope_extracted?("unmanaged_files")
-      file = self["unmanaged_files"].files.find { |f| f.name == path }
-      return parse_variable_assignment(file.content, key) if file
+    EXTRACTABLE_SCOPES.each do |scope|
+      if scope_extracted?(scope)
+        file = self[scope].files.find { |f| f.name == path }
+        return parse_variable_assignment(file.content, key) if file
+      end
     end
+    # if scope_extracted?("config_files")
+    #   file = self["config_files"].files.find { |f| f.name == path }
+    #   return parse_variable_assignment(file.content, key) if file
+    # end
+    # if scope_extracted?("unmanaged_files")
+    #   file = self["unmanaged_files"].files.find { |f| f.name == path }
+    #   return parse_variable_assignment(file.content, key) if file
+    # end
   end
 
   private

data/lib/system_description_store.rb

@@ -74,6 +74,16 @@ class SystemDescriptionStore
     FileUtils.cp_r(description_path(from), description_path(to))
   end
 
+  def move(from, to)
+    SystemDescription.validate_name(from)
+    SystemDescription.validate_name(to)
+
+    validate_existence_of_description(from)
+    validate_nonexistence_of_description(to)
+
+    FileUtils.mv(description_path(from), description_path(to))
+  end
+
   def backup(description_name)
     SystemDescription.validate_name(description_name)
     validate_existence_of_description(description_name)

data/lib/version.rb

@@ -17,6 +17,6 @@
 
 module Machinery
 
-  VERSION = "1.13.0"
+  VERSION = "1.14.0"
 
 end

data/lib/workload_mapper.rb

@@ -104,7 +104,8 @@ class WorkloadMapper
               tgz_file = File.join(dir, "trees", "#{origin.chop}.tgz")
               output_path = File.join(path, workload, destination)
               FileUtils.mkdir_p(output_path)
-              Cheetah.run("tar", "zxf", tgz_file, "-C", output_path, "--strip=1")
+              strip_number = origin.split("/").size - 1
+              Cheetah.run("tar", "zxf", tgz_file, "-C", output_path, "--strip=#{strip_number}")
               copy_workload_config_files(workload, output_path)
             end
             if file && file.file?

data/machinery-helper/README.md

@@ -0,0 +1,13 @@
+# machinery-helper
+
+A helper binary used by Machinery - http://machinery-project.org
+
+It inspects a system for unmanaged-files (files not tracked by rpm)
+and outputs the result in the
+[Machinery json format](https://github.com/SUSE/machinery/blob/master/docs/System-Description-Format.md).
+
+## Build
+
+Make sure that the official Go Development environment is installed.
+
+To build the helper binary just run `go build`.

data/machinery-helper/Rakefile

@@ -0,0 +1,124 @@
+# Copyright (c) 2013-2015 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 3 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.   See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, contact SUSE LLC.
+#
+# To contact SUSE about this file by physical or electronic mail,
+# you may find current contact information at www.suse.com
+
+task :default => "try_build"
+
+HELPER_DIR = File.expand_path(File.dirname(__FILE__))
+GIT_REVISION_FILE = File.join(HELPER_DIR, "..", ".git_revision")
+
+def write_go_version_file
+  file = <<-EOF
+// This is a generated file and shouldn't be changed
+
+package main
+
+const VERSION = "#{git_revision}"
+  EOF
+  File.write(File.join(HELPER_DIR, "version.go"), file)
+end
+
+def build_machinery_helper
+  FileUtils.rm_f(File.join(HELPER_DIR, "machinery-helper"))
+  Dir.chdir(HELPER_DIR) do
+    puts("Building machinery-helper binary.")
+    if !system("go build")
+      STDERR.puts("Warning: Building of the machinery-helper failed!")
+      false
+    else
+      true
+    end
+  end
+end
+
+def go_available?
+  if !system("which go > /dev/null 2>&1")
+    STDERR.puts(
+      "Warning: The Go compiler is not available on this system. Skipping building the" \
+        " machinery-helper.\nThe machinery-helper increases the inspection speed significantly."
+    )
+    false
+  else
+    true
+  end
+end
+
+def arch_supported?
+  arch = `uname -p`.chomp
+  if !["x86_64"].include?(arch)
+    STDERR.puts(
+      "Warning: The hardware architecture #{arch} is not yet supported by the machinery-helper."
+    )
+    false
+  else
+    true
+  end
+end
+
+def runs_in_git?
+  Dir.exist?(File.join(HELPER_DIR, "..", ".git")) && system("which git > /dev/null 2>&1")
+end
+
+def git_revision
+  `git rev-parse HEAD`.chomp
+end
+
+def write_git_revision_file
+  File.write(GIT_REVISION_FILE, git_revision)
+end
+
+def changed_revision?
+  if File.exist?(GIT_REVISION_FILE)
+    old_revision = File.read(GIT_REVISION_FILE)
+  else
+    old_revision = "unknown"
+  end
+  git_revision != old_revision
+end
+
+def run_build
+  # An unsupported architecture is no error
+  return true if !arch_supported?
+  return false if !go_available?
+
+  # handle changed branches (where go files are older than the helper)
+  if runs_in_git? && changed_revision?
+    write_go_version_file
+    if build_machinery_helper
+      write_git_revision_file
+      return true
+    else
+      return false
+    end
+  end
+
+  if !FileUtils.uptodate?(
+    File.join(HELPER_DIR, "machinery-helper"), Dir.glob(File.join(HELPER_DIR, "*.go"))
+  )
+    return build_machinery_helper
+  end
+  true
+end
+
+desc "Build the helper if needed"
+task :build do
+  raise "Error: Build of Machinery helper failed" if !run_build
+end
+
+desc "Don't fail on build errors"
+task :try_build do
+  run_build
+end

data/machinery-helper/machinery_helper.go

@@ -0,0 +1,228 @@
+// Copyright (c) 2015 SUSE LLC
+//
+// This program is free software; you can redistribute it and/or
+// modify it under the terms of version 3 of the GNU General Public License as
+// published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.   See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, contact SUSE LLC.
+//
+// To contact SUSE about this file by physical or electronic mail,
+// you may find current contact information at www.suse.com
+
+package main
+
+import (
+  "os/exec"
+  "bytes"
+  "log"
+  "strings"
+  "io/ioutil"
+  "encoding/json"
+  "os"
+  "sort"
+  "fmt"
+  "path/filepath"
+  "unicode/utf8"
+  "flag"
+)
+
+func getRpms() []string {
+  cmd := exec.Command("rpm", "-qlav")
+  var out bytes.Buffer
+  cmd.Stdout = &out
+  err := cmd.Run()
+  if err != nil {
+    log.Fatal(err)
+  }
+
+  f := func(c rune) bool {
+    return c == '\n'
+  }
+  packages := strings.FieldsFunc(out.String(), f)
+  return packages
+}
+
+func parseRpmLine(line string) (fileType string, fileName string, linkTarget string) {
+  fileType = line[0:1]
+
+  index := strings.Index(line, "/")
+  if index < 0 {
+    panic(line)
+  }
+  file := line[index:]
+  fields := strings.Split(file, " -> ")
+  if len(fields) == 2 {
+    fileName = fields[0]
+    linkTarget = fields[1]
+  } else {
+    fileName = file
+  }
+  return
+}
+
+func addImplicitlyManagedDirs(dirs map[string]bool, files map[string]string) {
+  for file, target := range files {
+    for i:= 1; i < len(file); i++ {
+      if file[i] == '/' {
+        topdir := file[:i]
+        if _, ok := dirs[topdir]; !ok {
+          dirs[topdir] = false
+        }
+      }
+    }
+
+    if target != "" {
+      if _, ok := dirs[target]; ok {
+        dirs[file] = false
+      }
+    }
+  }
+  return
+}
+
+func getManagedFiles() (map[string]string, map[string]bool) {
+  files := make(map[string]string)
+  dirs := make(map[string]bool)
+
+  for _, pkg := range getRpms() {
+    if pkg != "(contains no files)" {
+      fileType, fileName, linkTarget := parseRpmLine(pkg)
+      switch fileType {
+        case "-":
+          files[fileName] = ""
+        case "d":
+          dirs[fileName] = true
+        case "l":
+          files[fileName] = linkTarget
+      }
+    }
+  }
+
+  addImplicitlyManagedDirs(dirs, files)
+
+  return files, dirs
+}
+
+func assembleJSON(unmanagedFilesMap interface{}) string {
+  jsonMap := map[string]interface{}{ "extracted": false, "files": unmanagedFilesMap }
+  json, _ := json.MarshalIndent(jsonMap, " ", "  ")
+  return string(json)
+}
+
+var readDir = func(dir string) ([]os.FileInfo, error) {
+  return ioutil.ReadDir(dir)
+}
+
+func hasManagedDirs(dir string, rpmDirs map[string]bool) bool {
+  for rpmDir := range rpmDirs {
+    if strings.HasPrefix(rpmDir, dir + "/") {
+       return true
+    }
+  }
+  return false
+}
+
+func findUnmanagedFiles(dir string, rpmFiles map[string]string, rpmDirs map[string]bool,
+    unmanagedFiles map[string]string, ignoreList map[string]bool) {
+  files, _ := readDir(dir)
+  for _, f := range files {
+    fileName := dir + f.Name()
+    if !utf8.ValidString(fileName) {
+      fmt.Fprintln(os.Stderr, fileName, "contains invalid UTF-8 characters. Skipping.")
+    } else {
+      if _, ok := ignoreList[fileName]; !ok {
+        if f.IsDir() {
+          if _, ok := rpmDirs[fileName]; ok {
+            findUnmanagedFiles(fileName + "/", rpmFiles, rpmDirs, unmanagedFiles, ignoreList)
+          } else {
+            if !hasManagedDirs(fileName, rpmDirs) {
+              unmanagedFiles[fileName + "/"] = "dir"
+            }
+          }
+        } else {
+          if _, ok := rpmFiles[fileName]; !ok {
+            if f.Mode() &
+              (os.ModeSocket | os.ModeNamedPipe | os.ModeDevice | os.ModeCharDevice) != 0 {
+              // Ignore sockets, named pipes and devices
+            } else if f.Mode() & os.ModeSymlink == os.ModeSymlink {
+              unmanagedFiles[fileName] = "link"
+            } else {
+              unmanagedFiles[fileName] = "file"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+func printVersion() {
+  fmt.Println("Version:", VERSION)
+  os.Exit(0)
+}
+
+func main() {
+  // check for tar extraction
+  if len(os.Args) >= 2 {
+    switch os.Args[1] {
+    case "tar":
+      Tar(os.Args[2:])
+      os.Exit(0)
+    }
+  }
+
+  // parse CLI arguments
+  var versionFlag = flag.Bool("version", false, "shows the version number")
+  flag.Parse()
+
+  // show version
+  if (*versionFlag == true) {
+    printVersion()
+  }
+
+  // fetch unmanaged files
+  unmanagedFiles := make(map[string]string)
+  thisBinary, _ := filepath.Abs(os.Args[0])
+
+  ignoreList := map[string]bool{
+    thisBinary: true,
+  }
+  for _, mount := range RemoteMounts() {
+    ignoreList[mount] = true
+  }
+  for _, mount := range SpecialMounts() {
+    ignoreList[mount] = true
+  }
+
+  for _, mount := range RemoteMounts() {
+    unmanagedFiles[mount + "/"] = "remote_dir"
+  }
+
+  rpmFiles, rpmDirs := getManagedFiles()
+  findUnmanagedFiles("/", rpmFiles, rpmDirs, unmanagedFiles, ignoreList)
+
+  files := make([]string, len(unmanagedFiles))
+  i := 0
+  for k := range unmanagedFiles {
+    files[i] = k
+    i++
+  }
+  sort.Strings(files)
+
+  unmanagedFilesMap := make([]map[string]string, len(unmanagedFiles))
+  for j := range files {
+    entry := make(map[string]string)
+    entry["name"] = files[j]
+    entry["type"] = unmanagedFiles[files[j]]
+    unmanagedFilesMap[j] = entry
+  }
+
+  json := assembleJSON(unmanagedFilesMap)
+  fmt.Println(json)
+}