machinery-tool 1.0.2

data/kiwi_helpers/kiwi_export_readme.md

@@ -0,0 +1,22 @@
+# README for Kiwi export from Machinery
+
+This directory contains a Kiwi configuration that was exported by
+Machinery.
+
+The user is expected to be familiar with using Kiwi, otherwise
+`machinery build` is recommended.
+Details on Kiwi can be found at http://opensuse.github.com/kiwi
+
+
+## Creating the image
+
+The following command builds the image:
+
+`sudo /usr/sbin/kiwi --build EXPORTED_DIRECTORY --destdir OUTPUT_DIRECTORY`
+
+
+For example if the exported kiwi description is stored under "/tmp/export"
+and the image should be saved under "/tmp/image" the command would look like
+this:
+
+`sudo /usr/sbin/kiwi --build  /tmp/export --destdir /tmp/image`

data/kiwi_helpers/merge_users_and_groups.pl.erb

@@ -0,0 +1,231 @@
+#!/usr/bin/env perl
+
+# Copyright (c) 2013-2014 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 3 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.   See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, contact SUSE LLC.
+#
+# To contact SUSE about this file by physical or electronic mail,
+# you may find current contact information at www.suse.com
+
+use strict;
+use warnings;
+use FileHandle;
+
+my $local_passwd_path = shift;
+my $local_shadow_path = shift;
+my $local_group_path = shift;
+
+
+my $fh = FileHandle->new;
+my @existing_passwd_entries = [];
+if($fh->open("< $local_passwd_path")) {
+  @existing_passwd_entries = <$fh>;
+  $fh->close;
+}
+my @existing_shadow_entries = [];
+if($fh->open("< $local_shadow_path")) {
+  @existing_shadow_entries = <$fh>;
+  $fh->close;
+}
+my @existing_group_entries = [];
+if($fh->open("< $local_group_path")) {
+  @existing_group_entries = <$fh>;
+  $fh->close;
+}
+
+# This array contains arrays of the new passwd and shadow entries, e.g.
+# (
+#   ["svn:x:482:476:user for Apache Subversion svnserve:/srv/svn:/sbin/nologin", "svn:!:16058::::::"],
+#   ["nscd:x:484:478:User for nscd:/var/run/nscd:/sbin/nologin", "nscd:!:16058::::::"]
+# )
+my @new_passwd_entries = (
+<%= passwd_entries %>
+);
+
+# This array contains the new group entries, e.g.:
+# (
+#   "ntp:x:102:",
+#   "at:x:103:"
+# )
+my @new_group_entries = (
+<%= group_entries %>
+);
+
+# Takes a passwd entry as the argument and replaces the uid with an id that is
+# not already used in the existing passwd file
+sub fix_ids {
+  my $passwd_entry = shift;
+  my @fields = split(/:/x, $passwd_entry);
+  my $uid = $fields[2];
+  my $gid = $fields[3];
+
+  while(grep { /^[^:]*:[^:]*:$uid:/x } @existing_passwd_entries) {
+    $uid = $uid + 1;
+  }
+
+  $passwd_entry =~ s/^([^:]*:[^:]*):\d+:/$1:$uid:/x;
+
+  return $passwd_entry;
+}
+
+# Takes a group entry as the argument and replaces the uid with an id that is
+# not already used in the existing group file
+sub fix_group_id {
+  my $group = shift;
+  my @fields = split(/:/x, $group);
+  my $original_gid = $fields[2];
+  my $gid = $original_gid;
+
+  while(grep { /^[^:]*:[^:]*:$gid:/x } @existing_group_entries) {
+    $gid = $gid + 1;
+  }
+  $group =~ s/^([^:]*:[^:]*):$original_gid:/$1:$gid:/x;
+
+  # Also update the gid in the list of new passwd entries
+  foreach my $entry_ref (@new_passwd_entries) {
+    $entry_ref->[0] =~ s/^([^:]*:[^:]*:[^:]*):$original_gid:/$1:$gid:/x;
+  }
+
+  return $group;
+}
+
+# Takes two comma-separated lists and merges them. Duplicates are removed.
+sub merge_group_users {
+  my $a = $_[0] // "";
+  my $b = $_[1] // "";
+  chomp($a);
+  chomp($b);
+  my @users = (split(",", $a), split(",", $b));
+
+  #remove duplicate entries
+  my %hash = ();
+  foreach my $item (@users) {
+    $hash{$item} = '';
+  }
+  @users = sort keys %hash;
+
+  return join(",", @users);
+}
+
+# Iterate over the new group entries and fix ids where necessary.
+#
+# If the group already exists on the system the group is not added again, but the
+# new passwd entries that reference this group are adjusted to the other gid.
+#
+# If the group does not exist yet, the group is appended to /etc/group. If
+# necessary, the gid will be adjusted to not conflict with the existing ones.
+my %gid_changes = ();
+foreach my $entry (@new_group_entries) {
+  my @fields = split(/:/x, $entry);
+  my $id = $fields[0];
+  my $gid = $fields[2];
+
+  my @match = grep { /^$id:.*/x } @existing_group_entries;
+  if(scalar @match > 0) {
+    print "Group match: $match[0]\n";
+
+    my @original_fields = split(/:/x, $match[0]);
+    my $original_gid = $original_fields[2];
+
+    # Store all gid transformations in the %gid_changes hash
+    $gid_changes{$gid} = $original_gid;
+
+
+    # Update existing entry with the new attributes
+    $fields[2] = $original_gid;
+    my $users = merge_group_users($original_fields[3], $fields[3]);
+    $entry =~ s/^([^:]*:[^:]*):.*/$1:$original_gid:$users/x;
+    foreach my $group_ref (@existing_group_entries) {
+      $group_ref =~ s/\Q$match[0]\E/$entry/x;
+    }
+  } else {
+    print "No match for group ", $id, " (", $entry,")\n";
+
+    my $fixed_entry = fix_group_id($entry);
+    print "  -> Writing: ", $fixed_entry, "\n";
+
+    push(@existing_group_entries, $fixed_entry);
+  }
+}
+
+# Apply all gid transformations to the list of new passwd entries
+foreach my $entry_ref (@new_passwd_entries) {
+  my @fields = split(/:/x, $entry_ref->[0]);
+  my $gid = $fields[3];
+
+  if($gid_changes{$gid}) {
+    $entry_ref->[0] =~ s/^([^:]*:[^:]*:[^:]*):$gid:/$1:$gid_changes{$gid}:/x;
+  }
+}
+
+# Iterate over the new passwd and shadow entries and fix ids where necessary.
+#
+# If the user already exists on the system the existing uid/gid is reused.
+#
+# If the user does not exist yet it is appended to /etc/passwd and /etc/shadow
+# with a uid that does not conflict.
+foreach my $entry (@new_passwd_entries) {
+  my @fields = split(/:/x, @$entry[0]);
+  my $id = $fields[0];
+
+  my @match = grep { /^$id:.*/x } @existing_passwd_entries;
+  if(scalar @match > 0) {
+    my @original_fields = split(/:/x, $match[0]);
+    my $uid = $original_fields[2];
+
+    $fields[2] = $uid;
+    my $new_entry = join(":", @fields);
+
+    foreach my $passwd_ref (@existing_passwd_entries) {
+      $passwd_ref =~ s/\Q$match[0]\E/$new_entry/x;
+    }
+    if(@$entry[1]) {
+      foreach my $shadow_ref (@existing_shadow_entries) {
+        $shadow_ref =~ s/^$id.*/@$entry[1]/x;
+      }
+    }
+  } else {
+    print "No match for user $id (@$entry[0])\n";
+
+    my $fixed_entry = fix_ids(@$entry[0]);
+    print "  ->Writing: ", $fixed_entry, "\n";
+
+    push(@existing_passwd_entries, $fixed_entry);
+    if(@$entry[1]) {
+      push(@existing_shadow_entries, @$entry[1]);
+    }
+  }
+}
+
+# Write final lists to files
+if($fh->open("> $local_passwd_path")) {
+  foreach(@existing_passwd_entries) {
+    chomp;
+    $fh->print("$_\n");
+  }
+  $fh->close;
+}
+if($fh->open("> $local_shadow_path")) {
+  foreach(@existing_shadow_entries) {
+    chomp;
+    $fh->print("$_\n");
+  }
+  $fh->close;
+}
+if($fh->open("> $local_group_path")) {
+  foreach(@existing_group_entries) {
+    chomp;
+    $fh->print("$_\n");
+  }
+  $fh->close;
+}

data/kiwi_helpers/unmanaged_files_build_excludes

@@ -0,0 +1,5 @@
+var/lib/rpm/*
+etc/passwd*
+etc/shadow*
+etc/group*
+etc/fstab

data/lib/analyze_config_file_diffs_task.rb

@@ -0,0 +1,130 @@
+# Copyright (c) 2013-2014 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 3 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.   See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, contact SUSE LLC.
+#
+# To contact SUSE about this file by physical or electronic mail,
+# you may find current contact information at www.suse.com
+
+class AnalyzeConfigFileDiffsTask
+  def analyze(description)
+    description.assert_scopes("repositories", "config_files")
+    if !description.scope_extracted?("config_files")
+      raise Machinery::Errors::MissingExtractedFiles.new(description, ["config_files"])
+    end
+
+    with_repositories(description) do |zypper|
+      description.initialize_file_store("config-file-diffs")
+      diffs_path = description.file_store("config-file-diffs")
+      extracted_files_path = description.file_store("config_files")
+
+      Machinery::Ui.puts "Generating diffs..."
+      cnt = 1
+      list = files_by_package(description)
+      total = list.map(&:files).flatten.length.to_s
+      list.each do |package|
+        path = zypper.download_package("#{package.name}-#{package.version}")
+
+        if !path || path.empty?
+          Machinery::Ui.warn "Warning: Could not download package #{package.name}-#{package.version}."
+          cnt += package.files.length
+          next
+        end
+
+        package.files.each do |file|
+          diff = Rpm.new(path).diff(file, File.join(extracted_files_path, file))
+
+          if !diff || diff.empty?
+            Machinery::Ui.warn "Warning: Could not generate diff for #{file}."
+          else
+            diff_path = File.join(diffs_path, file + ".diff")
+            FileUtils.mkdir_p(File.dirname(diff_path))
+            File.write(diff_path, diff)
+            Machinery::Ui.puts "[#{cnt.to_s.rjust(total.length)}/#{total}] #{file}"
+          end
+
+          cnt += 1
+        end
+      end
+      Machinery::Ui.puts "done"
+    end
+  end
+
+  private
+
+  # Creates an array of hashes with the RPM names, version and the list of
+  # changed file paths, e.g.
+  #
+  # [
+  #   {
+  #     "name"    => "aaa_base",
+  #     "version" => "3.11.1",
+  #     "files"   => ["/etc/modprobe.d/unsupported-modules", "/etc/inittab"]
+  #   }
+  # ]
+  def files_by_package(description)
+    files = description["config_files"].files.
+      select { |f| f.changes.include?("md5") }
+
+    files.inject({}) do |result, file|
+        key = "#{file.package_name}-#{file.package_version}"
+        result[key] ||= Package.new(
+          "name"    => file["package_name"],
+          "version" => file["package_version"],
+          "files"   => []
+        )
+
+        result[key]["files"] << file.name
+        result
+    end.values
+  end
+
+  def with_repositories(description, &block)
+    Machinery::Ui.puts "Setting up repository access..."
+    Zypper.isolated do |zypper|
+      begin
+        remote_repos = description.repositories.reject do |repo|
+          repo.url.start_with?("cd://") || repo.url.start_with?("dvd://")
+        end
+        remote_repos.each do |repo|
+          uri = URI.parse(repo.url)
+
+          if repo.username && repo.password
+            uri.user = repo.username
+            uri.password = repo.password
+          end
+
+          zypper.add_repo(uri.to_s, repo.alias)
+        end
+
+        begin
+          zypper.refresh
+        rescue Cheetah::ExecutionFailed => e
+          # If zypper is locked (exit code 7) we're doomed. Bail out completeley.
+          raise if e.status.exitstatus == 7
+
+          # Refreshing repositories might fail for various other reasons, but we
+          # still stick to the plan because the relevant config files might be
+          # available from the other repositories.
+          # If they aren't an error message will then be generated for each of
+          # the actually missing files instead.
+          Machinery.logger.error(e.message)
+          Machinery.logger.debug(e.backtrace.join("\n"))
+          Machinery.logger.debug("Standard output:\n #{e.stdout}")
+          Machinery.logger.debug("Error output:\n #{e.stderr}")
+        end
+
+        block.call(zypper)
+      end
+    end
+  end
+end

data/lib/array.rb

@@ -0,0 +1,98 @@
+# Copyright (c) 2013-2014 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 3 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.   See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, contact SUSE LLC.
+#
+# To contact SUSE about this file by physical or electronic mail,
+# you may find current contact information at www.suse.com
+
+module Machinery
+  class Array
+    class << self
+      def has_elements(options)
+        @element_class = options[:class]
+      end
+
+      def from_json(json)
+        elements = json.map do |element|
+          if @element_class
+            @element_class.from_json(element)
+          else
+            case element
+              when ::Array
+                Machinery::Array.from_json(element)
+              when Hash
+                Machinery::Object.from_json(element)
+              else
+                element
+            end
+          end
+        end
+
+        new(elements)
+      end
+    end
+
+    attr_reader :elements
+
+    def initialize(elements = [])
+      @elements = elements
+    end
+
+    def ==(other)
+      self.class == other.class && @elements == other.elements
+    end
+
+    # Various Array operators such as "-" and "&" use #eql? and #hash to compare
+    # array elements, which is why we need to make sure they work properly.
+
+    alias eql? ==
+
+    def hash
+      @elements.hash
+    end
+
+    def -(other)
+      self.class.new(@elements - other.elements)
+    end
+
+    def &(other)
+      self.class.new(@elements & other.elements)
+    end
+
+    def as_json
+      @elements.map do |element|
+        if element.is_a?(Machinery::Array) || element.is_a?(Machinery::Object)
+          element.as_json
+        else
+          element
+        end
+      end
+    end
+
+    def compare_with(other)
+      [
+        self - other,
+        other - self,
+        self & other
+      ].map { |e| !e.empty? ? e : nil }
+    end
+
+    def method_missing(name, *args, &block)
+      @elements.send(name, *args, &block)
+    end
+
+    def respond_to?(name, include_all = false)
+      super || @elements.respond_to?(name, include_all)
+    end
+  end
+end