machinery-tool 1.17.0 → 1.18.0

data/lib/system_description.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2013-2015 SUSE LLC
+# Copyright (c) 2013-2016 SUSE LLC
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of version 3 of the GNU General Public License as
@@ -26,7 +26,7 @@
 # The sub directories storing the data for specific scopes are handled by the
 # ScopeFileStore class.
 class SystemDescription < Machinery::Object
-  CURRENT_FORMAT_VERSION = 6
+  CURRENT_FORMAT_VERSION = 7
   EXTRACTABLE_SCOPES = [
     "changed_managed_files",
     "config_files",

data/lib/system_description_memory_store.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2013-2015 SUSE LLC
+# Copyright (c) 2013-2016 SUSE LLC
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of version 3 of the GNU General Public License as

data/lib/system_description_store.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2013-2015 SUSE LLC
+# Copyright (c) 2013-2016 SUSE LLC
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of version 3 of the GNU General Public License as

data/lib/system_file.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2013-2015 SUSE LLC
+# Copyright (c) 2013-2016 SUSE LLC
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of version 3 of the GNU General Public License as

data/lib/tarball.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2013-2015 SUSE LLC
+# Copyright (c) 2013-2016 SUSE LLC
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of version 3 of the GNU General Public License as

data/lib/tee_io.rb

@@ -0,0 +1,39 @@
+# Copyright (c) 2013-2016 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 3 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, contact SUSE LLC.
+#
+# To contact SUSE about this file by physical or electronic mail,
+# you may find current contact information at www.suse.com
+
+# = TeeIO
+#
+# Class to allow the storage and forwarding of input at the same time
+# For example if stderr output should be handled by our code but at the
+# same time be put out directly
+#
+# To prevent for example double error messages data can be filtered
+# before being passed on to the io_object
+# Only the data passed along is filtered, not the data stored in this
+# IO object.
+class TeeIO < StringIO
+  def initialize(io_object, io_filter = [])
+    super()
+    @io_object = io_object
+    @io_filter = Array(io_filter)
+  end
+
+  def write(data)
+    super
+    @io_object.puts(data) unless @io_filter.include?(data)
+  end
+end

data/lib/try.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2013-2015 SUSE LLC
+# Copyright (c) 2013-2016 SUSE LLC
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of version 3 of the GNU General Public License as

data/lib/ui.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2013-2015 SUSE LLC
+# Copyright (c) 2013-2016 SUSE LLC
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of version 3 of the GNU General Public License as

data/lib/upgrade_format_task.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2013-2015 SUSE LLC
+# Copyright (c) 2013-2016 SUSE LLC
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of version 3 of the GNU General Public License as

data/lib/validate_task.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2013-2015 SUSE LLC
+# Copyright (c) 2013-2016 SUSE LLC
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of version 3 of the GNU General Public License as

data/lib/version.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2013-2015 SUSE LLC
+# Copyright (c) 2013-2016 SUSE LLC
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of version 3 of the GNU General Public License as
@@ -17,6 +17,6 @@
 
 module Machinery
 
-  VERSION = "1.17.0"
+  VERSION = "1.18.0"
 
 end

data/lib/workload_mapper.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2013-2015 SUSE LLC
+# Copyright (c) 2013-2016 SUSE LLC
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of version 3 of the GNU General Public License as

data/lib/workload_mapper_dsl.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2013-2015 SUSE LLC
+# Copyright (c) 2013-2016 SUSE LLC
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of version 3 of the GNU General Public License as

data/lib/zypper.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2013-2015 SUSE LLC
+# Copyright (c) 2013-2016 SUSE LLC
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of version 3 of the GNU General Public License as
@@ -76,8 +76,8 @@ class Zypper
   def download_package(package)
     raw_xml = call_zypper "-x", "download", package, stdout: :capture
 
-    xml = Nokogiri::XML(raw_xml)
-    xml.xpath("//localfile/@path").to_s
+    xml = REXML::Document.new(raw_xml)
+    xml.elements["//localfile"].attributes["path"]
   end
 
   def version

data/machinery-helper/Rakefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2013-2015 SUSE LLC
+# Copyright (c) 2013-2016 SUSE LLC
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of version 3 of the GNU General Public License as
@@ -17,7 +17,7 @@
 
 require_relative "../tools/helper_builder"
 
-task :default => "try_build"
+task :default => "build"
 
 HELPER_DIR = File.expand_path(File.dirname(__FILE__))
 
@@ -26,9 +26,3 @@ task :build do
   helper_builder = HelperBuilder.new(HELPER_DIR)
   raise "Error: Build of Machinery helper failed" if !helper_builder.run_build
 end
-
-desc "Don't fail on build errors"
-task :try_build do
-  helper_builder = HelperBuilder.new(HELPER_DIR)
-  helper_builder.run_build
-end

data/machinery-helper/file_utils.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"bytes"
+	"log"
+	"os/exec"
+	"strings"
+)
+
+var statFile = func(path string) string {
+	cmd := exec.Command("stat", "-c", "%U:%G", path)
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	err := cmd.Run()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return strings.Trim(out.String(), "\n")
+}
+
+func getFileOwnerGroup(path string) (user, group string) {
+	split := strings.Split(statFile(path), ":")
+	user = split[0]
+	group = split[1]
+
+	return
+}

data/machinery-helper/file_utils_test.go

@@ -0,0 +1,21 @@
+package main
+
+import (
+	"testing"
+)
+
+func TestGetFileOwnerGroup(t *testing.T) {
+	statFile = func(path string) string {
+		return "foo:bar"
+	}
+	path := "/etc/passwd"
+
+	owner, group := getFileOwnerGroup(path)
+
+	if owner != "foo" {
+		t.Errorf("GetFileOwner('%v') = '%v', want '%v'", path, owner, "foo")
+	}
+	if group != "bar" {
+		t.Errorf("GetFileOwner('%v') = '%v', want '%v'", path, owner, "bar")
+	}
+}

data/machinery-helper/machinery_helper.go

@@ -29,10 +29,24 @@ import (
 	"path/filepath"
 	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 	"unicode/utf8"
 )
 
+// An UnmanagedFile represents an unmanaged file in the system description.
+type UnmanagedFile struct {
+	Name       string `json:"name"`
+	User       string `json:"user,omitempty"`
+	Group      string `json:"group,omitempty"`
+	Type       string `json:"type"`
+	Mode       string `json:"mode,omitempty"`
+	Files      *int   `json:"files,omitempty"`
+	FilesValue int    `json:"-"`
+	Size       *int64 `json:"size,omitempty"`
+	SizeValue  int64  `json:"-"`
+}
+
 func getDpkgContent() []string {
 	cmd := exec.Command("bash", "-c", "dpkg --get-selections | grep -v deinstall | awk '{print $1}'")
 	var out bytes.Buffer
@@ -193,6 +207,19 @@ var readDir = func(dir string) ([]os.FileInfo, error) {
 	return ioutil.ReadDir(dir)
 }
 
+var dirSize = func(path string) int64 {
+	dir, err := os.Open(path)
+	if err != nil {
+		log.Fatal(err)
+	}
+	stat, err := dir.Stat()
+	if err != nil {
+		log.Fatal(err)
+	}
+	dir.Close()
+	return stat.Size()
+}
+
 func hasManagedDirs(dir string, rpmDirs map[string]bool) bool {
 	for rpmDir := range rpmDirs {
 		if strings.HasPrefix(rpmDir, dir+"/") {
@@ -236,11 +263,91 @@ func findUnmanagedFiles(dir string, rpmFiles map[string]string, rpmDirs map[stri
 	}
 }
 
+func amendMode(entry *UnmanagedFile, perm os.FileMode) {
+	if entry.Type == "link" {
+		return
+	}
+
+	result := int64(perm.Perm())
+
+	if perm&os.ModeSticky > 0 {
+		result |= 01000
+	}
+	if perm&os.ModeSetuid > 0 {
+		result |= 04000
+	}
+	if perm&os.ModeSetgid > 0 {
+		result |= 02000
+	}
+	entry.Mode = strconv.FormatInt(result, 8)
+
+	// Pad mode string to a length of three
+	for len(entry.Mode) < 3 {
+		entry.Mode = "0" + entry.Mode
+	}
+}
+
+func dirInfo(path string) (size int64, fileCount int) {
+	files, _ := readDir(path)
+
+	size = int64(0)
+	fileCount = len(files)
+	for _, f := range files {
+		if f.IsDir() {
+			if _, ok := IgnoreList[path + f.Name()]; !ok {
+				subSize, subCount := dirInfo(path + f.Name() + "/")
+				size += subSize
+				fileCount += subCount
+			}
+		} else if f.Mode()&os.ModeSymlink != os.ModeSymlink {
+			size += f.Size()
+		}
+	}
+
+	return
+}
+
+func amendSize(entry *UnmanagedFile, size int64) {
+	if entry.Type == "file" {
+		entry.SizeValue = size
+		entry.Size = &entry.SizeValue
+	} else if entry.Type == "dir" {
+		size, files := dirInfo(entry.Name)
+		entry.SizeValue = size
+		entry.Size = &entry.SizeValue
+		entry.FilesValue = files
+		entry.Files = &entry.FilesValue
+	}
+}
+
+func amendPathAttributes(entry *UnmanagedFile, fileType string) {
+	if fileType != "link" {
+		file, err := os.Open(entry.Name)
+		if err != nil {
+			log.Fatal(err)
+		}
+		fi, err := file.Stat()
+		if err != nil {
+			log.Fatal(err)
+		}
+		file.Close()
+
+		amendMode(entry, fi.Mode())
+		amendSize(entry, fi.Size())
+	}
+
+	entry.User, entry.Group = getFileOwnerGroup(entry.Name)
+}
+
 func printVersion() {
 	fmt.Println("Version:", VERSION)
 	os.Exit(0)
 }
 
+// IgnoreList includes mounts and any other file type that will be ignored when
+// evaluating the unmanaged files in a system.
+var IgnoreList = map[string]bool{}
+
 func main() {
 	// check for tar extraction
 	if len(os.Args) >= 2 {
@@ -253,6 +360,7 @@ func main() {
 
 	// parse CLI arguments
 	var versionFlag = flag.Bool("version", false, "shows the version number")
+	var extractMetadataFlag = flag.Bool("extract-metadata", false, "extracts metadata without extracting files")
 	flag.Parse()
 
 	// show version
@@ -264,14 +372,14 @@ func main() {
 	unmanagedFiles := make(map[string]string)
 	thisBinary, _ := filepath.Abs(os.Args[0])
 
-	ignoreList := map[string]bool{
+	IgnoreList = map[string]bool{
 		thisBinary: true,
 	}
 	for _, mount := range RemoteMounts() {
-		ignoreList[mount] = true
+		IgnoreList[mount] = true
 	}
 	for _, mount := range SpecialMounts() {
-		ignoreList[mount] = true
+		IgnoreList[mount] = true
 	}
 
 	for _, mount := range RemoteMounts() {
@@ -279,7 +387,7 @@ func main() {
 	}
 
 	managedFiles, managedDirs := getManagedFiles()
-	findUnmanagedFiles("/", managedFiles, managedDirs, unmanagedFiles, ignoreList)
+	findUnmanagedFiles("/", managedFiles, managedDirs, unmanagedFiles, IgnoreList)
 
 	files := make([]string, len(unmanagedFiles))
 	i := 0
@@ -289,11 +397,16 @@ func main() {
 	}
 	sort.Strings(files)
 
-	unmanagedFilesMap := make([]map[string]string, len(unmanagedFiles))
+	unmanagedFilesMap := make([]UnmanagedFile, len(unmanagedFiles))
 	for j := range files {
-		entry := make(map[string]string)
-		entry["name"] = files[j]
-		entry["type"] = unmanagedFiles[files[j]]
+		entry := UnmanagedFile{}
+		entry.Name = files[j]
+		entry.Type = unmanagedFiles[files[j]]
+
+		if *extractMetadataFlag {
+			amendPathAttributes(&entry, unmanagedFiles[files[j]])
+		}
+
 		unmanagedFilesMap[j] = entry
 	}
 

data/machinery-helper/machinery_helper_test.go

@@ -190,3 +190,105 @@ func TestSubdirIsNotAccidentallyConsideredManaged(t *testing.T) {
 		t.Errorf("hasManagedDirs() = '%v', want '%v'", hasDirs, want)
 	}
 }
+
+func TestAmendMode(t *testing.T) {
+	entry := UnmanagedFile{Type: "file"}
+
+	amendMode(&entry, os.FileMode(0777))
+	want := "777"
+	if entry.Mode != want {
+		t.Errorf("amendMode() = '%v', want '%v", entry.Mode, want)
+	}
+
+	amendMode(&entry, os.FileMode(0222))
+	want = "222"
+	if entry.Mode != want {
+		t.Errorf("amendMode() = '%v', want '%v", entry.Mode, want)
+	}
+
+	perm := os.FileMode(0222 | os.ModeSticky)
+	amendMode(&entry, perm)
+	want = "1222"
+	if entry.Mode != want {
+		t.Errorf("amendMode() = '%v', want '%v", entry.Mode, want)
+	}
+
+	perm = os.FileMode(0222 | os.ModeSetuid)
+	amendMode(&entry, perm)
+	want = "4222"
+	if entry.Mode != want {
+		t.Errorf("amendMode() = '%v', want '%v", entry.Mode, want)
+	}
+
+	perm = os.FileMode(0555 | os.ModeSticky | os.ModeSetgid)
+	amendMode(&entry, perm)
+	want = "3555"
+	if entry.Mode != want {
+		t.Errorf("amendMode() = '%v', want '%v", entry.Mode, want)
+	}
+
+	perm = os.FileMode(0000)
+	amendMode(&entry, perm)
+	want = "000"
+	if entry.Mode != want {
+		t.Errorf("amendMode() = '%v', want '%v", entry.Mode, want)
+	}
+
+	perm = os.FileMode(0044)
+	amendMode(&entry, perm)
+	want = "044"
+	if entry.Mode != want {
+		t.Errorf("amendMode() = '%v', want '%v", entry.Mode, want)
+	}
+
+	entry = UnmanagedFile{Type: "link"}
+	amendMode(&entry, perm)
+	if entry.Mode != "" {
+		t.Errorf("links should not have a mode")
+	}
+}
+
+func TestAmendSize(t *testing.T) {
+	entry := UnmanagedFile{Type: "link"}
+
+	amendSize(&entry, 0)
+	if entry.Size != nil {
+		t.Errorf("Link should not get a size")
+	}
+
+	entry = UnmanagedFile{Type: "file"}
+
+	amendSize(&entry, 0)
+	if *entry.Size != 0 {
+		t.Errorf("File should get a size")
+	}
+
+	entry = UnmanagedFile{Name: "/opt/", Type: "dir"}
+	readDir = func(dir string) ([]os.FileInfo, error) {
+		dirs := make([]os.FileInfo, 0, 1)
+		switch dir {
+		// 0 is used as a type to define a Normal file
+		case "/opt/":
+			dirs = append(dirs, fakefileinfo.New("foo", int64(12), 0, time.Now(), false, nil))
+			dirs = append(dirs, fakefileinfo.New("bar", int64(12), 0, time.Now(), false, nil))
+			dirs = append(dirs, fakefileinfo.New("baz", int64(4096), os.ModeDir, time.Now(), true, nil))
+			dirs = append(dirs, fakefileinfo.New("foo-link", int64(8), os.ModeSymlink, time.Now(), false, nil))
+		case "/opt/baz/":
+			dirs = append(dirs, fakefileinfo.New("foo", int64(12), 0, time.Now(), false, nil))
+			dirs = append(dirs, fakefileinfo.New("bar", int64(12), 0, time.Now(), false, nil))
+		}
+		return dirs, nil
+	}
+	dirSize = func(path string) int64 {
+		return 4096
+	}
+	amendSize(&entry, 0)
+	want := int64(48) // 4 files (12 bytes each)
+	if *entry.Size != want {
+		t.Errorf("entry.Size = '%v', want '%v'", *entry.Size, want)
+	}
+	wantFiles := 6
+	if *entry.Files != wantFiles {
+		t.Errorf("entry.Files = '%v', want '%v'", *entry.Files, wantFiles)
+	}
+}