machinery-tool 1.13.0 → 1.14.0

data/plugins/os/os_inspector.rb

@@ -91,7 +91,7 @@ class OsInspector < Inspector
       result["pretty_name"] = strip_arch_from_name(result["pretty_name"])
 
       # remove always changing Tumbleweed version from pretty name
-      if result["pretty_name"] =~ /^openSUSE.*\(Tumbleweed\)/
+      if result["pretty_name"] =~ /^openSUSE.*Tumbleweed/
         result["pretty_name"] = "openSUSE Tumbleweed"
       end
     end
@@ -99,6 +99,13 @@ class OsInspector < Inspector
     # name instead of an abbreviation
     os = Os.for(result["pretty_name"] || result["name"])
     os.version = result["version"]
+
+    # since Tumbleweed does no longer store the version number in the
+    # "version" property, we've to read from the "version_id" property
+    if os.version == "Tumbleweed"
+      os.version = result["version_id"]
+    end
+
     os
   end
 
@@ -111,19 +118,24 @@ class OsInspector < Inspector
     # name is always the first line in /etc/SuSE-release
     result["name"] = strip_arch_from_name(suse_release.split("\n").first)
 
-    patch = nil
-    suse_release.each_line do |l|
-      if l.start_with?("VERSION")
-        result["version"] = l.split("=").last.strip
-      end
-      if l.start_with?("PATCHLEVEL")
-        patch = l.split("=").last.strip
+    result["patchlevel"] = nil
+    suse_release.split("\n").slice(1, suse_release.length).each do |line|
+      if !line.start_with?("#")
+        key, value = line.split("=")
+        result[key.strip.downcase] = value.strip
       end
     end
-    if result["version"] && !patch.nil?
-      result["version"] = "#{result["version"]} SP#{patch}"
+
+    if result["version"] && !result["patchlevel"].nil?
+      result["version"] = "#{result["version"]} SP#{result["patchlevel"]}"
+    end
+
+    if result["codename"] == "Tumbleweed"
+      os = Os.for ("openSUSE Tumbleweed")
+    else
+      os = Os.for(result["name"])
     end
-    os = Os.for(result["name"])
+
     os.version = result["version"]
     os
   end

data/plugins/repositories/repositories_inspector.rb

@@ -70,7 +70,7 @@ class RepositoriesInspector < Inspector
   end
 
   def inspect_yum_repositories
-    script = File.read(File.join(Machinery::ROOT, "helpers", "yum_repositories.py"))
+    script = File.read(File.join(Machinery::ROOT, "inspect_helpers", "yum_repositories.py"))
     begin
       repositories = JSON.parse(system.run_command(
         "bash", "-c", "python", stdin: script, stdout: :capture

data/plugins/services/services_inspector.rb

@@ -24,7 +24,12 @@ class ServicesInspector < Inspector
   end
 
   def inspect(_filter, _options = {})
-    if @system.has_command?("systemctl")
+    if @description.environment.system_type == "docker"
+      result = ServicesScope.new(
+        init_system: "none",
+        services: []
+      )
+    elsif @system.has_command?("systemctl")
       result = ServicesScope.new(
         init_system: "systemd",
         services: inspect_systemd_services

data/plugins/unmanaged_files/unmanaged_files_inspector.rb

@@ -24,8 +24,7 @@ class UnmanagedFilesInspector < Inspector
     @system.check_requirement("sed", "--version")
     @system.check_requirement("cat", "--version")
     @system.check_requirement("find", "--version")
-    @system.check_requirement("tar", "--version") if check_tar
-    @system.check_requirement("gzip", "--version") if check_tar
+    @system.check_create_archive_dependencies if check_tar
   end
 
   # extract pathes from rpm database into ruby hashes
@@ -237,7 +236,7 @@ class UnmanagedFilesInspector < Inspector
     end
 
     helper = MachineryHelper.new(@system)
-    if helper_usable?(helper, options)
+    if helper_usable?(helper)
       run_helper_inspection(helper, file_filter, do_extract, file_store_tmp, file_store_final,
         scope)
     else
@@ -245,18 +244,17 @@ class UnmanagedFilesInspector < Inspector
     end
   end
 
-  def helper_usable?(helper, options)
+  def helper_usable?(helper)
     if !helper.can_help?
       Machinery::Ui.puts(
         "Note: Using traditional inspection because there is no helper binary for" \
         " architecture '#{@system.arch}' available."
       )
-    elsif options[:remote_user] && options[:remote_user] != "root"
+    elsif @system.respond_to?(:remote_user) && @system.remote_user != "root"
       Machinery::Ui.puts(
         "Note: Using traditional inspection because only 'root' is supported as remote user."
       )
     else
-      Machinery::Ui.puts "Note: Using helper binary for inspection of unmanaged files."
       return true
     end
 
@@ -266,37 +264,44 @@ class UnmanagedFilesInspector < Inspector
   def run_helper_inspection(helper, filter, do_extract, file_store_tmp, file_store_final, scope)
     begin
       helper.inject_helper
+      if !helper.has_compatible_version?
+        raise Machinery::Errors::UnsupportedHelperVersion.new(
+          "Error: machinery-helper is not compatible with this Machinery version." \
+            "\nTry to reinstall the package or gem to fix the issue."
+        )
+      end
+
       helper.run_helper(scope)
-    ensure
-      helper.remove_helper
-    end
 
-    scope.files.delete_if { |f| filter.matches?(f.name) }
+      scope.files.delete_if { |f| filter.matches?(f.name) }
 
-    if do_extract
-      mount_points = MountPoints.new(@system)
-      excluded_trees = mount_points.remote + mount_points.special
+      if do_extract
+        mount_points = MountPoints.new(@system)
+        excluded_trees = mount_points.remote + mount_points.special
 
-      file_store_tmp.remove
-      file_store_tmp.create
+        file_store_tmp.remove
+        file_store_tmp.create
 
-      files = scope.files.select { |f| f.file? || f.link? }.map(&:name)
-      scope.retrieve_files_from_system_as_archive(@system, files, [])
-      show_extraction_progress(files.count)
+        files = scope.files.select { |f| f.file? || f.link? }.map(&:name)
+        scope.retrieve_files_from_system_as_archive(@system, files, [])
+        show_extraction_progress(files.count)
 
-      scope.retrieve_trees_from_system_as_archive(@system,
-        scope.files.select(&:directory?).map(&:name), excluded_trees) do |count|
-        show_extraction_progress(files.count + count)
-      end
+        scope.retrieve_trees_from_system_as_archive(@system,
+          scope.files.select(&:directory?).map(&:name), excluded_trees) do |count|
+          show_extraction_progress(files.count + count)
+        end
 
-      scope.files = extract_tar_metadata(scope.files, file_store_tmp.path)
-      file_store_final.remove
-      file_store_tmp.rename(file_store_final.store_name)
-      scope.scope_file_store = file_store_final
-      scope.extracted = true
-    else
-      file_store_final.remove
-      scope.extracted = false
+        scope.files = extract_tar_metadata(scope.files, file_store_tmp.path)
+        file_store_final.remove
+        file_store_tmp.rename(file_store_final.store_name)
+        scope.scope_file_store = file_store_final
+        scope.extracted = true
+      else
+        file_store_final.remove
+        scope.extracted = false
+      end
+    ensure
+      helper.remove_helper
     end
 
     @description["unmanaged_files"] = scope

data/plugins/unmanaged_files/unmanaged_files_model.rb

@@ -22,14 +22,20 @@ class UnmanagedFileList < Machinery::Array
   has_elements class: UnmanagedFile
 
   def compare_with(other)
-    only_self = elements.reject do |element|
-      other.elements.find { |other_element| files_match(element, other_element) }
+    self_hash = elements.inject({}) { |hash, e| hash[e.name] = e; hash }
+    other_hash = other.elements.inject({}) { |hash, e| hash[e.name] = e; hash }
+
+    both = []
+    only_self = []
+    elements.each do |element|
+      if other_hash.has_key?(element.name) && files_match(element, other_hash[element.name])
+        both << element
+      else
+        only_self << element
+      end
     end
     only_other = other.elements.reject do |element|
-      elements.find { |other_element| files_match(element, other_element) }
-    end
-    both = elements.select do |element|
-      other.elements.find { |other_element| files_match(element, other_element) }
+      self_hash.has_key?(element.name) && files_match(element, self_hash[element.name])
     end
 
     [

data/workload_mapper/docker-registry/clue.rb

@@ -1,8 +1,6 @@
 docker_registry_config = "/etc/registry/config.yml"
 if system.runs_service?("registry") && system.has_file?(docker_registry_config)
   identify "docker-registry", "registry"
-  volume = "registry:/registry"
-  parameter "volumes", [volume]
   used_port = /.*:(\d\d\d\d+)/.match(system.read_config(docker_registry_config, "addr")).
     to_a.fetch(1, nil)
   port = "#{used_port}:5000"

data/workload_mapper/docker-registry/compose-template.yml

@@ -1,5 +1,6 @@
 docker-registry:
   build: ./docker-registry
+  volumes:
+    - ./registry:/registry
   ports: :ports
-  volumes: :volumes
   links: :links

data/workload_mapper/docker-registry/container/Dockerfile

@@ -3,7 +3,7 @@ FROM opensuse:13.2
 RUN zypper -n --gpg-auto-import-keys ar http://download.opensuse.org/repositories/Virtualization:/containers/openSUSE_13.2/ virt
 RUN zypper -n --gpg-auto-import-keys refresh
 RUN zypper -n --gpg-auto-import-keys update
-RUN zypper -n --gpg-auto-import-keys install docker docker-distribution-registry
+RUN zypper -n --gpg-auto-import-keys install docker-distribution-registry
 
 ADD ./data/etc/registry /etc/registry
 

data/workload_mapper/mariadb/clue.rb

@@ -1,5 +1,6 @@
 if system.runs_service?("mysql")
   identify "mariadb", "db"
   parameter "user", "dbuser"
+  parameter "name", "defaultdb"
   parameter "password", SecureRandom.base64
 end

data/workload_mapper/mariadb/compose-template.yml

@@ -1,9 +1,9 @@
 mariadb:
   build: ./mariadb
   volumes:
-    - mariadb/data:/var/lib/mysql
+    - ./mariadb/data:/var/lib/mysql
   environment:
     DB_ROOT_PASS:
     DB_USER: :user
     DB_PASS: :password
-    DB_NAME:
+    DB_NAME: :name

data/workload_mapper/rails/compose-template.yml

@@ -2,4 +2,3 @@ rails:
   build: ./rails
   ports: :ports
   links: :links
-  command: "/usr/sbin/apache2ctl -D FOREGROUND"

data/workload_mapper/rails/container/Dockerfile

@@ -1,6 +1,5 @@
 FROM opensuse:13.2
 
-# see update.sh for why all "apt-get install"s have to stay as one long line
 RUN zypper -n --gpg-auto-import-keys refresh
 RUN zypper -n --gpg-auto-import-keys update
 RUN zypper -n --gpg-auto-import-keys install ca-certificates-mozilla \
@@ -28,14 +27,14 @@ RUN touch /etc/apache2/sysconfig.d/include.conf
 RUN gem install passenger -v 5.0.7
 RUN passenger-install-apache2-module.ruby2.1 -a
 
-#variable?
 RUN mkdir /srv/www/rails
 WORKDIR /srv/www/rails
 ADD ./data /srv/www/rails
+RUN echo "gem: --no-ri --no-rdoc" > /srv/www/rails/.gemrc
 RUN chown -R wwwrun:www  /srv/www/rails
 
 RUN bundle config build.nokogiri --use-system-libraries
-RUN bundle install
+RUN bundle install --without test development
 
 ADD apache2/sysconfig_apache2 /etc/sysconfig/apache2
 ADD apache2/httpd.conf.local /etc/apache2/httpd.conf.local
@@ -43,6 +42,6 @@ ADD apache2/listen.conf /etc/apache2/listen.conf
 ADD apache2/rails_app_vhost.conf /etc/apache2/vhosts.d/
 RUN cat /etc/apache2/httpd.conf.local >> /etc/apache2/httpd.conf
 
-ENV COMPOSE=1
+CMD ["/usr/sbin/apache2ctl", "-D", "FOREGROUND"]
 
 EXPOSE 3000

data/workload_mapper/wordpress/clue.rb

@@ -0,0 +1,13 @@
+if system.runs_service?("apache2")
+  apache_vhosts = system["unmanaged_files"].files.select { |f| /vhosts\.d/ =~ f.name }.map(&:name)
+  apache_vhosts.each do |vhost|
+    root = system.read_config(vhost, "DocumentRoot")
+    if system.has_file?(File.join(root, "wp-config.php"))
+      identify "wordpress", "web"
+      parameter "ports", ["8000:8000"]
+      parameter "links", ["db"]
+      extract File.join(root, ""), "data"
+      break # for now we only handle the first wordpress that was found
+    end
+  end
+end

data/workload_mapper/wordpress/compose-template.yml

@@ -0,0 +1,5 @@
+wordpress:
+  build: ./wordpress
+  ports: :ports
+  links: :links
+  command: "/usr/sbin/apache2ctl -D FOREGROUND"

data/workload_mapper/wordpress/config/wp-config.php

@@ -0,0 +1,38 @@
+<?php
+define('DB_NAME', getenv("DB_NAME"));
+define('DB_USER', getenv("DB_USER"));
+define('DB_PASSWORD', getenv("DB_PASS"));
+define('DB_HOST', 'db');
+define('DB_CHARSET', 'utf8');
+define('DB_COLLATE', '');
+
+/**#@+
+ * Authentication Unique Keys and Salts.
+ *
+ * Change these to different unique phrases!
+ * You can generate these using the {@link https://api.wordpress.org/secret-key/1.1/salt/ WordPress.org secret-key service}
+ * You can change these at any point in time to invalidate all existing cookies. This will force all users to have to log in again.
+ *
+ * @since 2.6.0
+ */
+define('AUTH_KEY',         'put your unique phrase here');
+define('SECURE_AUTH_KEY',  'put your unique phrase here');
+define('LOGGED_IN_KEY',    'put your unique phrase here');
+define('NONCE_KEY',        'put your unique phrase here');
+define('AUTH_SALT',        'put your unique phrase here');
+define('SECURE_AUTH_SALT', 'put your unique phrase here');
+define('LOGGED_IN_SALT',   'put your unique phrase here');
+define('NONCE_SALT',       'put your unique phrase here');
+
+$table_prefix  = 'wp_';
+
+define('WP_DEBUG', false);
+
+/* That's all, stop editing! Happy blogging. */
+
+/** Absolute path to the WordPress directory. */
+if ( !defined('ABSPATH') )
+	define('ABSPATH', dirname(__FILE__) . '/');
+
+/** Sets up WordPress vars and included files. */
+require_once(ABSPATH . 'wp-settings.php');

data/workload_mapper/wordpress/container/Dockerfile

@@ -0,0 +1,25 @@
+FROM opensuse:13.2
+
+# see update.sh for why all "apt-get install"s have to stay as one long line
+RUN zypper -n --gpg-auto-import-keys refresh
+RUN zypper -n --gpg-auto-import-keys update
+RUN zypper -n --gpg-auto-import-keys install apache2-devel \
+mariadb-client \
+libmysqlclient-devel \
+php5 \
+php5-mcrypt \
+php5-gd \
+php5-mysql \
+apache2-mod_php5 \
+which
+
+RUN touch /etc/apache2/sysconfig.d/include.conf
+
+RUN mkdir /srv/www/wordpress
+WORKDIR /srv/www/wordpress
+ADD ./data /srv/www/wordpress
+
+ADD apache2/listen.conf /etc/apache2/listen.conf
+ADD apache2/wordpress_vhost.conf /etc/apache2/vhosts.d/
+RUN echo "LoadModule php5_module        /usr/lib64/apache2/mod_php5.so" >> /etc/apache2/sysconfig.d/loadmodule.conf
+RUN echo "variables_order = 'GPCSE'" >> /etc/php5/cli/php.ini

data/workload_mapper/wordpress/container/apache2/listen.conf

@@ -0,0 +1 @@
+Listen 8000

data/workload_mapper/wordpress/container/apache2/wordpress_vhost.conf

@@ -0,0 +1,21 @@
+<VirtualHost *:8000>
+  DocumentRoot /srv/www/wordpress
+  <Directory /srv/www/wordpress/>
+    Options Indexes FollowSymLinks MultiViews
+    AllowOverride None
+    Order allow,deny
+    Allow from all
+    Require all granted
+  </Directory>
+  <IfModule mod_php5.c>
+         <FilesMatch "\.ph(p[345]?|tml)$">
+             SetHandler application/x-httpd-php
+         </FilesMatch>
+         <FilesMatch "\.php[345]?s$">
+             SetHandler application/x-httpd-php-source
+         </FilesMatch>
+          DirectoryIndex index.php4
+          DirectoryIndex index.php5
+          DirectoryIndex index.php
+  </IfModule>
+</VirtualHost>

data/workload_mapper/wordpress/setup/setup.rb.erb

@@ -0,0 +1,67 @@
+#!/usr/bin/env ruby
+# Copyright (c) 2015 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 3 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.   See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, contact SUSE LLC.
+#
+# To contact SUSE about this file by physical or electronic mail,
+# you may find current contact information at www.suse.com
+require 'docker'
+require 'json'
+
+app_name = "<%= @name %>"
+web_env_vars = <%= @services["web"]["environment"].to_a.map { |ar| ar.join("=") } %>
+db_env_vars = <%= @services["db"]["environment"].to_a.map { |ar| ar.join("=") } %>
+db_workload = "<%= @workloads.find { |_, config| config["service"] == "db" }.first %>"
+
+def find_image(name)
+  Docker::Image.get(name)
+rescue
+  return false
+end
+
+def build_image(path, name)
+  if find_image(name)
+    puts "Found #{name} image, not building again"
+  else
+    puts "Building image #{name}"
+    Docker::Image.build_from_dir(path, t: name) do |output|
+      print JSON.parse(output)["stream"]
+    end
+  end
+end
+
+puts "Setting up #{db_workload}"
+if File.exists?("#{Dir.pwd}/#{db_workload}/data/configured.txt")
+  puts "Exiting as database has already been configured.\n" \
+    "If you want to re run the configuration just remove the files in #{Dir.pwd}/#{db_workload}/data/"
+  exit 1
+end
+
+db_image = "#{app_name}_db"
+build_image("./#{db_workload}", db_image)
+
+volumes = ["#{Dir.pwd}/#{db_workload}/data:/var/lib/mysql"]
+db = Docker::Container.create("Image" => db_image, "name" => "#{db_image}_0", "Env" => db_env_vars, "Binds" => volumes, "Tty" => true, "Stream" => true)
+db.tap(&:start).attach(tty: true) { |output| print output }
+
+web_image = "#{app_name}_web"
+build_image("./wordpress", web_image)
+
+db.start
+
+puts "Stopping db service"
+db.kill
+
+puts "removing containers"
+db.remove
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: machinery-tool
 version: !ruby/object:Gem::Version
-  version: 1.13.0
+  version: 1.14.0
 platform: ruby
 authors:
 - SUSE
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-15 00:00:00.000000000 Z
+date: 2015-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cheetah
@@ -227,9 +227,11 @@ email:
 - machinery@lists.suse.com
 executables:
 - machinery
-extensions: []
+extensions:
+- machinery-helper/Rakefile
 extra_rdoc_files: []
 files:
+- ".git_revision"
 - COPYING
 - NEWS
 - bin/machinery
@@ -239,11 +241,8 @@ files:
 - export_helpers/merge_users_and_groups.pl.erb
 - export_helpers/unmanaged_files_autoyast_excludes
 - export_helpers/unmanaged_files_kiwi_excludes
-- helpers/changed_files.sh
-- helpers/default_filters.json
-- helpers/filter-packages-for-build.yaml
-- helpers/inspector_files.rb
-- helpers/yum_repositories.py
+- filters/default_filters.json
+- filters/filter-packages-for-build.yaml
 - html/assets/angular-sanitize.min.js
 - html/assets/angular.min.js
 - html/assets/arrow_down.png
@@ -286,6 +285,8 @@ files:
 - html/assets/wheels_horizontal.png
 - html/comparison.html.haml
 - html/index.html.haml
+- inspect_helpers/changed_files.sh
+- inspect_helpers/yum_repositories.py
 - lib/analyze_config_file_diffs_task.rb
 - lib/array.rb
 - lib/autoyast.rb
@@ -303,6 +304,7 @@ files:
 - lib/copy_task.rb
 - lib/current_user.rb
 - lib/deploy_task.rb
+- lib/docker_system.rb
 - lib/element_filter.rb
 - lib/exceptions.rb
 - lib/export_task.rb
@@ -330,6 +332,7 @@ files:
 - lib/manifest.rb
 - lib/migration.rb
 - lib/mountpoints.rb
+- lib/move_task.rb
 - lib/object.rb
 - lib/remote_system.rb
 - lib/remove_task.rb
@@ -356,6 +359,14 @@ files:
 - lib/workload_mapper.rb
 - lib/workload_mapper_dsl.rb
 - lib/zypper.rb
+- machinery-helper/README.md
+- machinery-helper/Rakefile
+- machinery-helper/machinery_helper.go
+- machinery-helper/machinery_helper_test.go
+- machinery-helper/mountpoints.go
+- machinery-helper/mountpoints_test.go
+- machinery-helper/tar.go
+- machinery-helper/version.go
 - man/generated/machinery.1.gz
 - man/generated/machinery.1.html
 - plugins/changed_managed_files/changed_managed_files.md
@@ -478,6 +489,13 @@ files:
 - workload_mapper/rails/container/apache2/rails_app_vhost.conf
 - workload_mapper/rails/container/apache2/sysconfig_apache2
 - workload_mapper/rails/setup/setup.rb.erb
+- workload_mapper/wordpress/clue.rb
+- workload_mapper/wordpress/compose-template.yml
+- workload_mapper/wordpress/config/wp-config.php
+- workload_mapper/wordpress/container/Dockerfile
+- workload_mapper/wordpress/container/apache2/listen.conf
+- workload_mapper/wordpress/container/apache2/wordpress_vhost.conf
+- workload_mapper/wordpress/setup/setup.rb.erb
 homepage: http://machinery-project.org
 licenses:
 - GPL-3.0

data/helpers/inspector_files.rb

@@ -1,52 +0,0 @@
-# Copyright (c) 2013-2015 SUSE LLC
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of version 3 of the GNU General Public License as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.   See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, contact SUSE LLC.
-#
-# To contact SUSE about this file by physical or electronic mail,
-# you may find current contact information at www.suse.com
-
-
-# This script creates reference data for integration tests
-# by inspecting existing machines.
-# Therefor it generates files with information of each scope.
-
-# Usage: rake inspector_files[ip-adress,distribution]
-
-require "fileutils"
-require "cheetah"
-
-class ReferenceTestData
-  def initialize
-    @machinery = File.expand_path("../bin/machinery", File.dirname(__FILE__))
-    @test_data_path = File.expand_path("../spec/data/", File.dirname(__FILE__))
-  end
-
-  def inspect(ip_adress)
-    Cheetah.run(@machinery, "inspect", ip_adress, "-n", "referencetestdata", stdout: :capture)
-  end
-
-  def write(destination)
-    Inspector.all_scopes.each do |inspector|
-      output = Cheetah.run(
-        @machinery, "show", destination, "-s", inspector.gsub("_", "-"),
-        "referencetestdata", stdout: :capture
-      )
-      File.write(file_path(inspector, destination), output)
-    end
-  end
-
-  def file_path(inspector, destination)
-    FileUtils.mkdir_p("#{@test_data_path}/#{inspector}")
-    File.join(@test_data_path, inspector, destination)
-  end
-end