machinery-tool 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c161742e300133d4d5a6514ddfe3e4284fc29b52
-  data.tar.gz: 28c974344534d7ad4df48e43dce31b60125bb1b1
+  metadata.gz: f3e26096068d827b9178938a39eead5000562a09
+  data.tar.gz: 78c08edb97fc7993b173cf477821f6eee75bb27c
 SHA512:
-  metadata.gz: ddd663cea68dfa8a41b8cd8921c91436a5e70a049439ee5b628b4d628f427ef4375923f9b33e967e4d6604ba487950cad0ebed0e2b62291ba66fed49ebfa1087
-  data.tar.gz: 285ed630148abb78038681ed823dd2079501a18ecd64d0882e642300c8d6f648b62dfb7f672e8efd4dcdfc1fc3131401cf82e88f09c6d3cdb6c886423f69302f
+  metadata.gz: 370cb5832ee25886ade589381967014b8bb8d746d9b3451fcd52e30d08ff32ce4a71670ab2c2debfa451d8c2673f5dc7fd5e7fbf66db76e616af7e1753d2fbfa
+  data.tar.gz: 74487bdaf7324bc54bf8b1a62f0437d69086570a1f61de8f7517c7c6c7d2c6debfdd8f83c60954432be28c40fa2505831bf910a80aecdcb1db879157aac582f5

data/NEWS

@@ -1,6 +1,18 @@
 # Machinery Release Notes
 
 
+## Version 1.2.0 - Mon Dec 22 15:06:20 CET 2014 - aduffeck@suse.de
+
+* Added AutoYaST export for system descriptions
+* Handle zypper locks gracefully (gh#SUSE/machinery#415)
+* Prevent scopes from being inspected multiple times (gh#SUSE/machinery#303)
+* Keep already extracted results after canceling an inspection (gh#SUSE/machinery#364)
+* Don't fail on file validation errors, but report them as warnings (gh#SUSE/machinery#391)
+* List scopes more natural with a space after comma (gh#SUSE/machinery#356)
+* Use UTF-8 for displaying HTML report (gh#SUSE/machinery#429)
+* Do not ignore Btrfs subvolumes during unmanaged-files inspection (gh#SUSE/machinery#427)
+* Store KIWI and AutoYaST exports in subdirectories
+
 ## Version 1.1.1 - Tue Dec 02 13:24:38 CET 2014 - thardeck@suse.de
 
 * Skip Gnome Virtual File System mounts on inspection (issue #393)

data/export_helpers/autoyast_export_readme.md

@@ -0,0 +1,48 @@
+# README for AutoYaST export from Machinery
+
+This directory contains an AutoYaST configuration that was exported by
+Machinery.
+
+The user is expected to be familiar with using AutoYaST.
+
+## Using the AutoYaST export
+
+The export directory contains both the AutoYaST profile and additional data that
+is required during installation. This directory needs to be made available to
+the installer via network, e.g. by running:
+
+  cd /path/to/autoyast_export; python -m SimpleHTTPServer
+
+You can then point the installer to the profile by specifying the AutoYaST
+option on the kernel command line.
+
+For SLES12 and openSUSE 13.2:
+
+  autoyast2=http://192.168.121.1:8000/autoinst.xml
+
+For SLES11:
+
+  autoyast=http://192.168.121.1:8000/autoinst.xml netsetup=dhcp
+
+## Changing permissions of the AutoYaST export
+
+By default the AutoYaST export is only accessible by the user. This is also the
+case for all sub directories.
+
+The installation via for example an HTTP server is only possible if all files
+and sub directories are readable by the HTTP server user.
+To make the export directory readable for all users run:
+
+  chmod -R a+rX /path/to/autoyast_export
+
+## Reference
+
+Additional information regarding AutoYaST is available in the official documentation:
+
+For SLES12:
+
+  https://www.suse.com/documentation/sles-12/singlehtml/book_autoyast/book_autoyast.html
+
+For SLES11
+
+  https://www.suse.com/documentation/sles11/singlehtml/book_autoyast/book_autoyast.html

data/{kiwi_helpers → export_helpers}/unmanaged_files_build_excludes

@@ -3,3 +3,5 @@ etc/passwd*
 etc/shadow*
 etc/group*
 etc/fstab
+lib/mkinitrd
+etc/udev/rules.d/70-persistent-net.rules

data/html/index.html.haml

@@ -3,6 +3,7 @@
   %head
     %title
       = description.name + " - Machinery System Description"
+    %meta{:charset => 'utf-8'}/
     %link{:href => "assets/machinery-base.css", :rel => "stylesheet", :type => "text/css"}/
     %link{:href => "assets/machinery.css", :rel => "stylesheet", :type => "text/css"}/
     %script{:src => "assets/jquery-2.1.1.min.js"}

data/lib/analyze_config_file_diffs_task.rb

@@ -17,15 +17,20 @@
 
 class AnalyzeConfigFileDiffsTask
   def analyze(description)
-    description.assert_scopes("repositories", "config_files")
+    description.assert_scopes(
+      "repositories",
+      "config_files",
+      "os"
+    )
     if !description.scope_extracted?("config_files")
       raise Machinery::Errors::MissingExtractedFiles.new(description, ["config_files"])
     end
 
     with_repositories(description) do |zypper|
-      description.initialize_file_store("config-file-diffs")
-      diffs_path = description.file_store("config-file-diffs")
-      extracted_files_path = description.file_store("config_files")
+      file_store = description.scope_file_store("config-file-diffs")
+      file_store.create
+      diffs_path = file_store.path
+      extracted_files_path = description.scope_file_store("config_files").path
 
       Machinery::Ui.puts "Generating diffs..."
       cnt = 1
@@ -90,7 +95,8 @@ class AnalyzeConfigFileDiffsTask
 
   def with_repositories(description, &block)
     Machinery::Ui.puts "Setting up repository access..."
-    Zypper.isolated do |zypper|
+    arch = description.os.architecture
+    Zypper.isolated(arch: arch) do |zypper|
       begin
         remote_repos = description.repositories.reject do |repo|
           repo.url.start_with?("cd://") || repo.url.start_with?("dvd://")

data/lib/autoyast.rb

@@ -15,13 +15,44 @@
 # To contact SUSE about this file by physical or electronic mail,
 # you may find current contact information at www.suse.com
 
-class Autoyast
+class Autoyast < Exporter
   def initialize(description)
+    @chroot_scripts = []
     @system_description = description
+    @system_description.assert_scopes(
+      "repositories",
+      "packages"
+    )
   end
 
   def write(output_dir)
+    FileUtils.cp(
+      File.join(Machinery::ROOT, "export_helpers/unmanaged_files_build_excludes"),
+      output_dir
+    )
+    # Filter log files to prevent an issue with hanging gzip during installation
+    File.open(File.join(output_dir, "unmanaged_files_build_excludes"), "a") do |file|
+      file.puts "var/log/*"
+    end
+    FileUtils.chmod(0600, File.join(output_dir, "unmanaged_files_build_excludes"))
+    FileUtils.cp(
+      File.join(Machinery::ROOT, "export_helpers/autoyast_export_readme.md"),
+      File.join(output_dir, "README.md")
+    )
+    Dir["#{@system_description.description_path}/*"].each do |content|
+      FileUtils.cp_r(content, output_dir, preserve: true)
+    end
     File.write(File.join(output_dir, "autoinst.xml"), profile)
+    FileUtils.chmod(0600, File.join(output_dir, "autoinst.xml"))
+    Machinery::Ui.puts(
+      "Note: The permssions of the AutoYaST directory are restricted to be" \
+        " only accessible by the current user. Further instructions are" \
+        " provided by the README.md in the exported directory."
+    )
+  end
+
+  def export_name
+    "#{@system_description.name}-autoyast"
   end
 
   def profile
@@ -31,6 +62,8 @@ class Autoyast
         "xmlns" => "http://www.suse.com/1.0/yast2ns",
         "xmlns:config" => "http://www.suse.com/1.0/configns"
       ) do
+        apply_non_interactive_mode(xml)
+        apply_basic_network(xml)
         apply_repositories(xml)
         xml.software do
           apply_packages(xml)
@@ -39,6 +72,20 @@ class Autoyast
         apply_users(xml)
         apply_groups(xml)
         apply_services(xml)
+
+        apply_extracted_files("config_files")
+        apply_extracted_files("changed_managed_files")
+        apply_unmanaged_files
+        xml.scripts do
+          apply_url_extraction(xml)
+          xml.send("chroot-scripts", "config:type" => "list") do
+            xml.script do
+              xml.source do
+                xml.cdata @chroot_scripts.join("\n")
+              end
+            end
+          end
+        end
       end
     end
 
@@ -47,19 +94,55 @@ class Autoyast
 
   private
 
+  def apply_non_interactive_mode(xml)
+    xml.general do
+      xml.mode do
+        xml.confirm "false", "config:type" => "boolean"
+      end
+    end
+  end
+
+  def apply_basic_network(xml)
+    xml.networking do
+      xml.keep_install_network "true", "config:type" => "boolean"
+    end
+  end
+
   def apply_repositories(xml)
     return if !@system_description.repositories
 
     xml.send("add-on") do
       xml.add_on_products("config:type" => "list") do
         @system_description.repositories.each do |repository|
-          xml.listentry do
-            xml.media_url repository.url
-            xml.name repository.alias
+          if repository.enabled && !repository.external_medium?
+            xml.listentry do
+              xml.media_url repository.url
+              xml.name repository.alias
+            end
           end
         end
       end
     end
+
+    @system_description.repositories.each do |repository|
+      # Disabled repositories or external media can't be added by AutoYaST so we add them manually
+      if !repository.enabled || repository.external_medium?
+        zypper_ar = "zypper -n ar --name='#{repository.name}'"
+        zypper_ar << " --type='#{repository.type}'" if repository.type
+        zypper_ar << " --disable" if !repository.enabled
+        zypper_ar << " '#{repository.url}' '#{repository.alias}'"
+        @chroot_scripts << zypper_ar.strip
+      end
+
+      zypper_mr = "zypper -n mr --priority=#{repository.priority} '#{repository.alias}'"
+      zypper_mr <<  " --name='#{repository.name}'"
+      if repository.autorefresh
+        zypper_mr << " --refresh"
+      else
+        zypper_mr << " --no-refresh"
+      end
+      @chroot_scripts << zypper_mr.strip
+    end
   end
 
   def apply_packages(xml)
@@ -125,20 +208,28 @@ class Autoyast
   end
 
   def apply_services(xml)
+    return if !@system_description.services
+
     xml.send("services-manager") do
-      xml.services do
+      xml.services("config:type" => "list") do
         @system_description.services.services.each do |service|
+          name = service.name
+          if @system_description.services.init_system == "systemd"
+            # Yast can only handle services right now
+            next if !(name =~ /\.service$/)
+            name = name.gsub(/\.service$/, "")
+          end
           # systemd service states like "masked" and "static" are
           # not supported by Autoyast
           if service.enabled?
             xml.service do
-              xml.service_name service.name
+              xml.service_name name
               xml.service_status "enable"
             end
           end
           if service.disabled?
             xml.service do
-              xml.service_name service.name
+              xml.service_name name
               xml.service_status "disable"
             end
           end
@@ -146,4 +237,63 @@ class Autoyast
       end
     end
   end
+
+  def apply_url_extraction(xml)
+    xml.send("pre-scripts", "config:type" => "list") do
+      xml.script do
+        xml.source do
+          xml.cdata 'sed -n \'/.*autoyast2\?=\(.*\)\/.*[^\s]*/s//\1/p\'' \
+            ' /proc/cmdline > /tmp/description_url'
+        end
+      end
+    end
+  end
+
+  def apply_extracted_files(scope)
+    return if !@system_description[scope] || !@system_description[scope].extracted
+
+    base = Pathname(@system_description.scope_file_store(scope).path)
+    Dir["#{base}/**/*"].sort.each do |path|
+      next if File.directory?(path)
+
+      relative_path = Pathname(path).relative_path_from(base).to_s
+      url = "`cat /tmp/description_url`/#{URI.escape(File.join(scope, relative_path))}"
+
+      @chroot_scripts << "mkdir -p '#{File.join("/mnt", File.dirname(relative_path))}'"
+      @chroot_scripts << "curl -o '#{File.join("/mnt", relative_path)}' \"#{url}\""
+    end
+
+    @system_description[scope].files.map do |file|
+      if file.user
+        @chroot_scripts << "chown #{file.user}:#{file.group} '#{File.join("/mnt", file.name)}'"
+      end
+      if file.mode
+        @chroot_scripts << "chmod #{file.mode} '#{File.join("/mnt", file.name)}'"
+      end
+    end
+  end
+
+  def apply_unmanaged_files
+    return if !@system_description.unmanaged_files ||
+      !@system_description.unmanaged_files.extracted
+
+    base = Pathname(@system_description.scope_file_store("unmanaged_files").path)
+    @chroot_scripts << <<-EOF
+      curl -o '/mnt/tmp/filter' "`cat /tmp/description_url`/unmanaged_files_build_excludes"
+    EOF
+
+    Dir["#{base}/**/*.tgz"].sort.each do |path|
+      next if !path.end_with?(".tgz")
+
+      relative_path = Pathname(path).relative_path_from(base).to_s
+      tarball_name = File.basename(path)
+      url = "`cat /tmp/description_url`#{URI.escape(File.join("/unmanaged_files", relative_path))}"
+
+      @chroot_scripts << <<-EOF
+        curl -o '/mnt/tmp/#{tarball_name}' "#{url}"
+        tar -C /mnt/ -X '/mnt/tmp/filter' -xf '#{File.join("/mnt/tmp", tarball_name)}'
+        rm '#{File.join("/mnt/tmp", tarball_name)}'
+      EOF
+    end
+  end
 end

data/lib/build_task.rb

@@ -28,16 +28,6 @@ class BuildTask
     config = KiwiConfig.new(system_description, options)
     config.write(tmp_config_dir)
 
-    if system_description["unmanaged_files"]
-      filters = File.read(
-        File.join(Machinery::ROOT, "kiwi_helpers/unmanaged_files_build_excludes")
-      )
-      Machinery::Ui.puts "\nUnmanaged files following these patterns are not added " \
-        "to the built image:"
-      Machinery::Ui.puts filters
-      Machinery::Ui.puts "\n"
-    end
-
     FileUtils.mkdir_p(output_path)
     if tmp_image_dir.start_with?("/tmp/") && tmp_config_dir.start_with?("/tmp/")
       tmp_script = write_kiwi_wrapper(tmp_config_dir, tmp_image_dir,

data/lib/cli.rb

@@ -179,7 +179,7 @@ class Cli
       )
     end
 
-    scopes
+    scopes.uniq
   end
 
   AVAILABLE_SCOPE_LIST = Machinery::Ui.internal_scope_list_to_string(
@@ -203,7 +203,7 @@ class Cli
     c.action do |global_options,options,args|
       name = shift_arg(args, "NAME")
       store = SystemDescriptionStore.new
-      description = store.load(name)
+      description = SystemDescription.load(name, store)
 
       case options[:operation]
         when "config-file-diffs"
@@ -238,7 +238,7 @@ class Cli
     c.action do |global_options,options,args|
       name = shift_arg(args, "NAME")
       store = SystemDescriptionStore.new
-      description = store.load(name)
+      description = SystemDescription.load(name, store)
 
       task = BuildTask.new
       task.build(description, File.expand_path(options["image-dir"]), {:enable_dhcp => options["enable-dhcp"], :enable_ssh => options["enable-ssh"]})
@@ -272,8 +272,8 @@ class Cli
       name1 = shift_arg(args, "NAME1")
       name2 = shift_arg(args, "NAME2")
       store = SystemDescriptionStore.new
-      description1 = store.load(name1)
-      description2 = store.load(name2)
+      description1 = SystemDescription.load(name1, store)
+      description2 = SystemDescription.load(name2, store)
       scope_list = process_scope_option(options[:scope], options["exclude-scope"])
 
       task = CompareTask.new
@@ -327,7 +327,7 @@ class Cli
     c.action do |global_options,options,args|
       name = shift_arg(args, "NAME")
       store = SystemDescriptionStore.new
-      description = store.load(name)
+      description = SystemDescription.load(name, store)
 
       task = DeployTask.new
       opts = {
@@ -345,8 +345,8 @@ class Cli
   long_desc <<-LONGDESC
     Export system description as KIWI image description.
 
-    The description will be placed in the given location. The image format in the
-    description is 'vmx'.
+    The description will be placed in a subdirectory at the given location. The image format in
+    the description is 'vmx'.
   LONGDESC
   arg "NAME"
   command "export-kiwi" do |c|
@@ -358,7 +358,7 @@ class Cli
     c.action do |global_options,options,args|
       name = shift_arg(args, "NAME")
       store = SystemDescriptionStore.new
-      description = store.load(name)
+      description = SystemDescription.load(name, store)
       exporter = KiwiConfig.new(description)
 
       task = ExportTask.new(exporter)
@@ -375,7 +375,8 @@ class Cli
   long_desc <<-LONGDESC
     Export system description as AutoYaST profile
 
-    The profile will be placed in the location given by the 'autoyast-dir' option.
+    The profile will be placed in a subdirectory at the given location by the 'autoyast-dir'
+    option.
   LONGDESC
   arg "NAME"
   command "export-autoyast" do |c|
@@ -387,7 +388,7 @@ class Cli
     c.action do |_global_options, options, args|
       name = shift_arg(args, "NAME")
       store = SystemDescriptionStore.new
-      description = store.load(name)
+      description = SystemDescription.load(name, store)
       exporter = Autoyast.new(description)
 
       task = ExportTask.new(exporter)
@@ -544,7 +545,7 @@ class Cli
       end
 
       store = SystemDescriptionStore.new
-      description = store.load(name)
+      description = SystemDescription.load(name, store)
       scope_list = process_scope_option(options[:scope], options["exclude-scope"])
 
 
@@ -605,7 +606,7 @@ class Cli
       name = shift_arg(args, "NAME")
 
       store = SystemDescriptionStore.new
-      description = store.load(name)
+      description = SystemDescription.load(name, store)
       task = GenerateHtmlTask.new
       task.generate(description)
     end