machinery-tool 1.8.2 → 1.9.0

data/lib/file_validator.rb

@@ -78,9 +78,9 @@ class FileValidator
       expected_files << "files.tgz" if has_files_tarball
       expected_files += tree_tarballs
     else
-      expected_files = files.
-        reject { |file| file["changes"].include?("deleted") }.
-        map { |file| file["name"] }
+      expected_files = files.reject do |file|
+          file["changes"].include?("deleted") || (file["type"] && file["type"] != "file")
+        end.map { |file| file["name"] }
     end
 
     store_base_path = ScopeFileStore.new(@base_path, scope.to_s).path

data/lib/helper.rb

@@ -32,12 +32,18 @@ module Machinery
     s.dup.force_encoding("UTF-8").encode("UTF-16", invalid: :replace).encode("UTF-8")
   end
 
-  def self.pluralize(text, number)
-    if number > 1 || number == 0
-      text + "s"
+  def self.pluralize(count, singular, plural = nil)
+    val = if count > 1 || count == 0
+      if !plural
+        singular + "s"
+      else
+        plural
+      end
     else
-      text
+      singular
     end
+
+    val.gsub("%d", count.to_s)
   end
 end
 

data/lib/inspector.rb

@@ -89,4 +89,30 @@ class Inspector
   def scope
     self.class.scope
   end
+
+  # Runs the given script on the inspected machine asynchronously and calls the callback method
+  # periodically with new output when it occurs.
+  #
+  # Example:
+  #
+  #     count = 0
+  #     raw_list = run_script_with_progress("changed_managed_files.sh") do |chunk|
+  #       count += chunk.lines.count
+  #       Machinery::Ui.progress("Found #{count} changed files...")
+  #     end
+  def run_script_with_progress(*script, &callback)
+    output = ""
+    write_io = StringIO.new(output, "a")
+    read_io = StringIO.new(output, "r")
+
+    inspect_thread = Thread.new { @system.run_script(*script, stdout: write_io) }
+
+    while inspect_thread.alive?
+      sleep 0.1
+      chunk = read_io.read
+      callback.call(chunk)
+    end
+
+    output
+  end
 end

data/lib/kiwi_config.rb

@@ -29,7 +29,7 @@ class KiwiConfig < Exporter
       "packages",
       "os"
     )
-    check_existance_of_extraced_files
+    check_existance_of_extracted_files
 
     generate_config
   end
@@ -95,25 +95,46 @@ class KiwiConfig < Exporter
   end
 
   def inject_extracted_files(output_location)
-    ["config_files", "changed_managed_files"].each do |dir|
-      path = @system_description.scope_file_store(dir).path
-      if path
-        output_root_path = File.join(output_location, "root")
-        FileUtils.mkdir_p(output_root_path)
-        FileUtils.cp_r(Dir.glob("#{path}/*"), output_root_path)
+    ["changed_managed_files", "config_files"].each do |scope|
+      next if !@system_description.scope_extracted?(scope)
+
+      output_root_path = File.join(output_location, "root")
+      FileUtils.mkdir_p(output_root_path)
+
+      @system_description[scope].files.each do |file|
+        if file.deleted?
+          @sh << "rm -rf '#{quote(file.name)}'\n"
+        elsif file.directory?
+          @sh << <<EOF
+chmod #{file.mode} '#{quote(file.name)}'
+chown #{file.user}:#{file.group} '#{quote(file.name)}'
+EOF
+        elsif file.file?
+          @system_description[scope].write_file(file, output_root_path)
+          @sh << <<EOF
+chmod #{file.mode} '#{quote(file.name)}'
+chown #{file.user}:#{file.group} '#{quote(file.name)}'
+EOF
+        elsif file.link?
+          @sh << <<EOF
+rm -rf '#{quote(file.name)}'
+ln -s '#{quote(file.target)}' '#{quote(file.name)}'
+chown --no-dereference #{file.user}:#{file.group} '#{quote(file.name)}'
+EOF
+        end
       end
     end
 
-    unmanaged_files_path = @system_description.
-      scope_file_store("unmanaged_files").path
-    if unmanaged_files_path
-      filter = "unmanaged_files_#{@name}_excludes"
-      destination = File.join(output_location, "root", "tmp")
+    if @system_description.scope_extracted?("unmanaged_files")
+      destination = File.join(output_location, "root", "tmp", "unmanaged_files")
       FileUtils.mkdir_p(destination, mode: 01777)
-      FileUtils.cp_r(unmanaged_files_path, destination)
+      filter = "unmanaged_files_#{@name}_excludes"
+
+      @system_description.unmanaged_files.export_files_as_tarballs(destination)
+
       FileUtils.cp(
         File.join(Machinery::ROOT, "export_helpers/#{filter}"),
-        destination
+        File.join(output_location, "root", "tmp")
       )
 
       @sh << "# Apply the extracted unmanaged files\n"
@@ -123,7 +144,7 @@ class KiwiConfig < Exporter
     end
   end
 
-  def check_existance_of_extraced_files
+  def check_existance_of_extracted_files
     missing_scopes = []
     ["config_files", "changed_managed_files", "unmanaged_files"].each do |scope|
 
@@ -190,7 +211,6 @@ EOF
 
         apply_repositories(xml)
         apply_packages(xml)
-        apply_extracted_files_attributes
         apply_services
       end
     end
@@ -261,25 +281,6 @@ EOF
     end
   end
 
-  def apply_extracted_files_attributes
-    ["config_files", "changed_managed_files"].each do |scope|
-      if @system_description[scope]
-        deleted, files = @system_description[scope].files.partition do |f|
-          f.changes == Machinery::Array.new(["deleted"])
-        end
-
-        files.each do |file|
-          @sh << "chmod #{file.mode} '#{file.name}'\n"
-          @sh << "chown #{file.user}:#{file.group} '#{file.name}'\n"
-        end
-
-        deleted.each do |file|
-          @sh << "rm -rf '#{file.name}'\n"
-        end
-      end
-    end
-  end
-
   def apply_services
     if @system_description["services"]
       init_system = @system_description["services"].init_system

data/lib/local_system.rb

@@ -44,21 +44,23 @@ class LocalSystem < System
     end
 
     def validate_machinery_compatibility
-      if !os.can_run_machinery?
-        supported_oses = Os.supported_host_systems.map { |o| o.canonical_name }.
-          sort.join(", ")
-        message = "Running Machinery is not supported on this system.\n" \
-          "Supported operating systems are: #{supported_oses}"
+      return if !Machinery::Config.new.perform_support_check || os.can_run_machinery?
 
-        raise(Machinery::Errors::IncompatibleHost.new(message))
-      end
+      supported_oses = Os.supported_host_systems.map { |o| o.canonical_name }.sort.join(", ")
+      message = <<EOF
+You are running Machinery on a platform we do not explicitly support and test.
+It still could work very well. If you run into issues or would like to provide us feedback, you are welcome to file an issue at https://github.com/SUSE/machinery/issues/new or write an email to machinery@lists.suse.com.
+Oficially supported operating systems are: #{supported_oses}"
+EOF
+
+      Machinery::Ui.warn message
     end
 
     def validate_build_compatibility(system_description)
       if !os.can_build?(system_description.os)
         message = "Building '#{system_description.os.display_name}' is " \
           "not supported on this distribution.\n" \
-          "Check the 'BUILD SUPPORT MATRIX' section in our man page for " \
+          "Check the 'BUILD SUPPORT MATRIX' by running `#{$0} build --help` for " \
           "further information which build targets are supported."
 
         raise(Machinery::Errors::BuildFailed.new(message))
@@ -94,7 +96,15 @@ class LocalSystem < System
   # the directory where to put the files.
   def retrieve_files(filelist, destination)
     begin
-      LoggedCheetah.run("rsync",  "--chmod=go-rwx", "--files-from=-", "/", destination, :stdout => :capture, :stdin => filelist.join("\n") )
+      LoggedCheetah.run(
+        "rsync",
+        "--chmod=go-rwx",
+        "--files-from=-",
+        "/",
+        destination,
+        stdout: :capture,
+        stdin: filelist.join("\n")
+        )
     rescue Cheetah::ExecutionFailed => e
       raise Machinery::Errors::RsyncFailed.new(
       "Could not rsync files from localhost. \n" \

data/lib/machinery.rb

@@ -91,6 +91,8 @@ require_relative "element_filter"
 require_relative "filter"
 require_relative "filter_option_parser"
 require_relative "file_scope"
+require_relative "system_file"
+require_relative "scope_file_access"
 
 Dir[File.join(Machinery::ROOT, "plugins", "**", "*.rb")].each { |f| require(f) }
 

data/lib/manifest.rb

@@ -24,7 +24,7 @@ class Manifest
   def self.load(name, path)
     unless File.exists?(path)
       raise Machinery::Errors::SystemDescriptionNotFound.new(
-        "A system description with the name #{name} was not found."
+        "Couldn't find a system description with the name '#{name}'."
       )
     end
 

data/lib/object.rb

@@ -55,11 +55,19 @@ module Machinery
     end
 
     attr_reader :attributes
+    attr_accessor :scope
 
     def initialize(attrs = {})
       set_attributes(attrs)
     end
 
+    def scope=(scope)
+      @scope = scope
+      @attributes.values.each do |child|
+        child.scope = @scope if child.respond_to?(:scope=)
+      end
+    end
+
     def set_attributes(attrs)
       attrs = self.class.convert_raw_hash(attrs) if attrs.is_a?(Hash)
       @attributes = attrs.inject({}) do |attributes, (key, value)|

data/lib/scope.rb

@@ -26,9 +26,14 @@ module Machinery
       @scopes
     end
 
-    def self.class_for(scope_name)
+    def self.for(scope_name, json, scope_file_store)
       all_scopes.each do |scope|
         if scope.new.scope_name == scope_name
+          scope = scope.from_json(json)
+          scope.scope_file_store = scope_file_store
+
+          scope.scope = scope
+
           return scope
         end
       end
@@ -36,6 +41,7 @@ module Machinery
     end
 
     attr_accessor :meta
+    attr_accessor :scope_file_store
 
     def set_metadata(timestring, host)
       self.meta = Machinery::Object.new(

data/lib/scope_file_access.rb

@@ -0,0 +1,67 @@
+module ScopeFileAccessFlat
+  def retrieve_files_from_system(system, paths)
+    system.retrieve_files(paths, scope_file_store.path)
+  end
+
+  def write_file(system_file, target)
+    raise Machinery::Errors::FileUtilsError, "Not a file" if !system_file.file?
+
+    target_path = File.join(target, system_file.name)
+    FileUtils.mkdir_p(File.dirname(target_path))
+    FileUtils.cp(file_path(system_file), target_path)
+  end
+
+  def file_path(system_file)
+    raise Machinery::Errors::FileUtilsError, "Not a file" if !system_file.file?
+
+    File.join(scope_file_store.path, system_file.name)
+  end
+end
+
+module ScopeFileAccessArchive
+  def retrieve_files_from_system_as_archive(system, files, excluded_files)
+    archive_path = File.join(scope_file_store.path, "files.tgz")
+    system.create_archive(files, archive_path, excluded_files)
+  end
+
+  def retrieve_trees_from_system_as_archive(system, trees, excluded_files, &callback)
+    trees.each_with_index do |tree, index|
+      tree_name = File.basename(tree)
+      parent_dir = File.dirname(tree)
+      sub_dir = File.join("trees", parent_dir)
+
+      scope_file_store.create_sub_directory(sub_dir)
+      archive_path = File.join(scope_file_store.path, sub_dir, "#{tree_name}.tgz")
+      system.create_archive(tree, archive_path, excluded_files)
+
+      callback.call(index + 1) if callback
+    end
+  end
+
+  def export_files_as_tarballs(destination)
+    FileUtils.cp(File.join(scope_file_store.path, "files.tgz"), destination)
+
+    target = File.join(destination, "trees")
+    files.select(&:directory?).each do |system_file|
+      raise Machinery::Errors::FileUtilsError if !system_file.directory?
+
+      tarball_target = File.join(target, File.dirname(system_file.name))
+
+      FileUtils.mkdir_p(tarball_target)
+      FileUtils.cp(tarball_path(system_file), tarball_target)
+    end
+  end
+
+  def tarball_path(system_file)
+    if system_file.directory?
+      File.join(
+        system_file.scope.scope_file_store.path,
+        "trees",
+        File.dirname(system_file.name),
+        File.basename(system_file.name) + ".tgz"
+      )
+    else
+      File.join(system_file.scope.scope_file_store.path, "files.tgz")
+    end
+  end
+end

data/lib/system.rb

@@ -54,7 +54,7 @@ class System
   # Retrieves files specified in filelist from the remote system and create an archive.
   # To be able to deal with arbitrary filenames we use zero-terminated
   # filelist and the --null option of tar
-  def create_archive(filelist, archive, exclude = [])
+  def create_archive(file_list, archive, exclude = [])
     created = !File.exists?(archive)
     out = File.open(archive, "w")
     begin
@@ -62,7 +62,7 @@ class System
         "tar", "--create", "--gzip", "--null", "--files-from=-",
         *exclude.flat_map { |f| ["--exclude", f]},
         stdout: out,
-        stdin: filelist,
+        stdin: Array(file_list).join("\0"),
         privileged: true
       )
     rescue Cheetah::ExecutionFailed => e

data/lib/system_description.rb

@@ -26,7 +26,7 @@
 # The sub directories storing the data for specific scopes are handled by the
 # ScopeFileStore class.
 class SystemDescription < Machinery::Object
-  CURRENT_FORMAT_VERSION = 3
+  CURRENT_FORMAT_VERSION = 4
   EXTRACTABLE_SCOPES = [
     "changed_managed_files",
     "config_files",
@@ -91,7 +91,7 @@ class SystemDescription < Machinery::Object
     def from_hash(name, store, hash)
       begin
         json_format_version = hash["meta"]["format_version"] if hash["meta"]
-        description = SystemDescription.new(name, store, create_scopes(hash))
+        description = SystemDescription.new(name, store, hash)
       rescue NameError, TypeError
         if json_format_version && json_format_version != SystemDescription::CURRENT_FORMAT_VERSION
           raise Machinery::Errors::SystemDescriptionIncompatible.new(name, json_format_version)
@@ -108,35 +108,40 @@ class SystemDescription < Machinery::Object
 
       description
     end
+  end
 
-    private
+  def initialize(name, store, hash = {})
+    @name = name
+    @store = store
+    @format_version = CURRENT_FORMAT_VERSION
+    @filter_definitions = {}
 
-    def create_scopes(hash)
-      scopes = hash.map do |scope_name, scope_json|
-        next if scope_name == "meta"
+    super(create_scopes(hash))
+  end
+
+  def create_scopes(hash)
+    scopes = hash.map do |scope_name, json|
+      next if scope_name == "meta"
 
-        scope_class = Machinery::Scope.class_for(scope_name)
-        scope_object = scope_class.from_json(scope_json)
+      if store.persistent?
+        scope_file_store = scope_file_store(scope_name)
+      end
+
+      if json.is_a?(Hash) || json.is_a?(Array)
+        scope_object = Machinery::Scope.for(scope_name, json, scope_file_store)
 
         # Set metadata
         if hash["meta"] && hash["meta"][scope_name]
           scope_object.meta = Machinery::Object.from_json(hash["meta"][scope_name])
         end
+      else
+        scope_object = json
+      end
 
-        [scope_name, scope_object]
-      end.compact
-
-      Hash[scopes]
-    end
-  end
-
-  def initialize(name, store, hash = {})
-    @name = name
-    @store = store
-    @format_version = CURRENT_FORMAT_VERSION
-    @filter_definitions = {}
+      [scope_name, scope_object]
+    end.compact
 
-    super(hash)
+    Hash[scopes]
   end
 
   def compatible?