machinery-tool 1.12.0 → 1.13.0

data/plugins/unmanaged_files/unmanaged_files_inspector.rb

@@ -139,7 +139,6 @@ class UnmanagedFilesInspector < Inspector
     dep = depth - 1
     files = {}
     dirs = {}
-    excluded_files = []
 
     # compute command line
     cmd = ["find", dir, "-xdev", "-maxdepth", "1", "-maxdepth", depth.to_s]
@@ -178,11 +177,9 @@ class UnmanagedFilesInspector < Inspector
         broken_names = []
         if path.include?("\uFFFD")
           broken_names << "filename '#{path}'"
-          excluded_files << raw_path
         end
         if link.include?("\uFFFD")
           broken_names << "link target '#{link}'"
-          excluded_files << raw_link
         end
 
         warning = broken_names.join(" and ")
@@ -201,8 +198,9 @@ class UnmanagedFilesInspector < Inspector
       # dirs at maxdepth could be non-leafs all othere are leafs
       dirs[path] = path.count("/") == dep if type == "d"
     end
-    Machinery.logger.debug "get_find_data dir:#{dir} depth:#{depth} file:#{files.size} dirs:#{dirs.size} excluded:#{excluded_files}"
-    [files, dirs, excluded_files]
+    Machinery.logger.debug "get_find_data dir:#{dir} depth:#{depth} file:#{files.size}" \
+      " dirs:#{dirs.size}"
+    [files, dirs]
   end
 
   def max_depth
@@ -240,17 +238,8 @@ class UnmanagedFilesInspector < Inspector
 
     helper = MachineryHelper.new(@system)
     if helper_usable?(helper, options)
-      begin
-        helper.inject_helper
-        helper.run_helper(scope)
-      ensure
-        helper.remove_helper
-      end
-      scope.extracted = false
-
-      scope.files.delete_if { |f| file_filter.matches?(f.name) }
-
-      @description["unmanaged_files"] = scope
+      run_helper_inspection(helper, file_filter, do_extract, file_store_tmp, file_store_final,
+        scope)
     else
       run_inspection(file_filter, options, do_extract, file_store_tmp, file_store_final, scope)
     end
@@ -262,11 +251,6 @@ class UnmanagedFilesInspector < Inspector
         "Note: Using traditional inspection because there is no helper binary for" \
         " architecture '#{@system.arch}' available."
       )
-    elsif options[:extract_unmanaged_files]
-      Machinery::Ui.puts(
-        "Note: Using traditional inspection because file extraction is not" \
-        " supported by the helper binary."
-      )
     elsif options[:remote_user] && options[:remote_user] != "root"
       Machinery::Ui.puts(
         "Note: Using traditional inspection because only 'root' is supported as remote user."
@@ -279,6 +263,45 @@ class UnmanagedFilesInspector < Inspector
     false
   end
 
+  def run_helper_inspection(helper, filter, do_extract, file_store_tmp, file_store_final, scope)
+    begin
+      helper.inject_helper
+      helper.run_helper(scope)
+    ensure
+      helper.remove_helper
+    end
+
+    scope.files.delete_if { |f| filter.matches?(f.name) }
+
+    if do_extract
+      mount_points = MountPoints.new(@system)
+      excluded_trees = mount_points.remote + mount_points.special
+
+      file_store_tmp.remove
+      file_store_tmp.create
+
+      files = scope.files.select { |f| f.file? || f.link? }.map(&:name)
+      scope.retrieve_files_from_system_as_archive(@system, files, [])
+      show_extraction_progress(files.count)
+
+      scope.retrieve_trees_from_system_as_archive(@system,
+        scope.files.select(&:directory?).map(&:name), excluded_trees) do |count|
+        show_extraction_progress(files.count + count)
+      end
+
+      scope.files = extract_tar_metadata(scope.files, file_store_tmp.path)
+      file_store_final.remove
+      file_store_tmp.rename(file_store_final.store_name)
+      scope.scope_file_store = file_store_final
+      scope.extracted = true
+    else
+      file_store_final.remove
+      scope.extracted = false
+    end
+
+    @description["unmanaged_files"] = scope
+  end
+
   def run_inspection(file_filter, options, do_extract, file_store_tmp, file_store_final, scope)
     mount_points = MountPoints.new(@system)
 
@@ -325,8 +348,7 @@ class UnmanagedFilesInspector < Inspector
 
       # determine files and directories below find_dir until a certain depth
       depth = local_filesystems.include?(find_dir) ? start : max
-      files, dirs, excluded = get_find_data(find_dir, depth )
-      excluded_files += excluded
+      files, dirs = get_find_data(find_dir, depth)
       find_count += 1
       find_dir += "/" if find_dir.size > 1
       if !local_filesystems.empty?

data/plugins/users/schema/system-description-users.schema-v5.json

@@ -0,0 +1,61 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+
+  "type": "array",
+  "items": {
+    "type": "object",
+    "required": ["name", "password", "uid", "gid", "comment", "home", "shell"],
+    "properties": {
+      "name": {
+        "type": "string",
+        "minLength": 1
+      },
+      "password": {
+        "type": "string"
+      },
+      "uid": {
+        "type": ["integer", "null"],
+        "minimum": 0
+      },
+      "gid": {
+        "type": ["integer", "null"],
+        "minimum": 0
+      },
+      "comment": {
+        "type": "string"
+      },
+      "home": {
+        "type": "string"
+      },
+      "shell": {
+        "type": "string"
+      },
+      "encrypted_password": {
+        "type": "string"
+      },
+      "last_changed_date": {
+        "type": "integer"
+      },
+      "min_days": {
+        "type": "integer",
+        "minimum": 0
+      },
+      "max_days": {
+        "type": "integer",
+        "minimum": 0
+      },
+      "warn_days": {
+        "type": "integer",
+        "minimum": 0
+      },
+      "disable_days": {
+        "type": "integer",
+        "minimum": 0
+      },
+      "disabled_date": {
+        "type": "integer"
+      }
+    }
+  }
+}
+

data/schema/migrations/migrate4to5.rb

@@ -0,0 +1,28 @@
+# Copyright (c) 2013-2014 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 3 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.   See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, contact SUSE LLC.
+#
+# To contact SUSE about this file by physical or electronic mail,
+# you may find current contact information at www.suse.com
+
+class Migrate4To5 < Migration
+  desc <<-EOT
+    Schema version 5 adds a "environment" scope which stores information about the inspected system.
+  EOT
+
+  def migrate
+    @hash["environment"] = {
+      locale: "en_US.utf8"
+    }
+  end
+end

data/schema/system-description-global.schema-v5.json

@@ -0,0 +1,43 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+
+  "type": "object",
+  "required": ["meta"],
+  "properties": {
+    "meta": {
+      "required": ["format_version"],
+      "properties": {
+        "format_version": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "filters": {
+          "type": "object",
+          "required": ["inspect"],
+          "properties": {
+            "inspect": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          }
+        }
+      },
+      "additionalProperties": {
+        "type": "object",
+        "required": ["modified", "hostname"],
+        "properties": {
+          "modified": {
+            "type": "string",
+            "format": "date-time"
+          },
+          "hostname": {
+            "type": "string",
+            "format": "hostname"
+          }
+        }
+      }
+    }
+  }
+}

data/workload_mapper/docker-registry/clue.rb

@@ -0,0 +1,14 @@
+docker_registry_config = "/etc/registry/config.yml"
+if system.runs_service?("registry") && system.has_file?(docker_registry_config)
+  identify "docker-registry", "registry"
+  volume = "registry:/registry"
+  parameter "volumes", [volume]
+  used_port = /.*:(\d\d\d\d+)/.match(system.read_config(docker_registry_config, "addr")).
+    to_a.fetch(1, nil)
+  port = "#{used_port}:5000"
+  parameter "ports", [port]
+  parameter "links", ["web"]
+  cert = system.read_config(docker_registry_config, "rootcertbundle")
+  extract docker_registry_config, "data"
+  extract cert, "data"
+end

data/workload_mapper/docker-registry/compose-template.yml

@@ -0,0 +1,5 @@
+docker-registry:
+  build: ./docker-registry
+  ports: :ports
+  volumes: :volumes
+  links: :links

data/workload_mapper/docker-registry/container/Dockerfile

@@ -0,0 +1,12 @@
+FROM opensuse:13.2
+
+RUN zypper -n --gpg-auto-import-keys ar http://download.opensuse.org/repositories/Virtualization:/containers/openSUSE_13.2/ virt
+RUN zypper -n --gpg-auto-import-keys refresh
+RUN zypper -n --gpg-auto-import-keys update
+RUN zypper -n --gpg-auto-import-keys install docker docker-distribution-registry
+
+ADD ./data/etc/registry /etc/registry
+
+EXPOSE 5000
+ENTRYPOINT ["registry"]
+CMD ["/etc/registry/config.yml"]

data/workload_mapper/mariadb/clue.rb

@@ -0,0 +1,5 @@
+if system.runs_service?("mysql")
+  identify "mariadb", "db"
+  parameter "user", "dbuser"
+  parameter "password", SecureRandom.base64
+end

data/workload_mapper/mariadb/compose-template.yml

@@ -0,0 +1,9 @@
+mariadb:
+  build: ./mariadb
+  volumes:
+    - mariadb/data:/var/lib/mysql
+  environment:
+    DB_ROOT_PASS:
+    DB_USER: :user
+    DB_PASS: :password
+    DB_NAME:

data/workload_mapper/mariadb/container/Dockerfile

@@ -0,0 +1,13 @@
+FROM opensuse:13.2
+
+RUN zypper -n --gpg-auto-import-keys refresh
+RUN zypper -n --gpg-auto-import-keys update
+RUN zypper -n --gpg-auto-import-keys install mariadb pwgen psmisc net-tools
+
+ADD scripts /scripts
+RUN chmod 755 /scripts/*
+
+VOLUME ["/var/lib/mysql", "/var/log/mysql"]
+EXPOSE 3306
+
+CMD ["/bin/bash", "/scripts/start.sh"]

data/workload_mapper/mariadb/container/scripts/config_mariadb.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -e
+
+__mysql_config() {
+  echo "Running the mysql_config function."
+  mysql_install_db
+  chown -R mysql:mysql /var/lib/mysql
+  /usr/bin/mysqld_safe &
+  sleep 10
+}
+
+__start_mysql() {
+  printf "Running the start_mysql function.\n"
+  DB_ROOT_PASS="${DB_ROOT_PASS-$(pwgen -s -1 12)}"
+  DB_USER="${DB_USER-dbuser}"
+  DB_PASS="${DB_PASS-$(pwgen -s -1 12)}"
+  DB_NAME="${DB_NAME-db}"
+  printf "root password=%s\n" "$DB_ROOT_PASS"
+  printf "DB_NAME=%s\n" "$DB_NAME"
+  printf "DB_USER=%s\n" "$DB_USER"
+  printf "DB_PASS=%s\n" "$DB_PASS"
+  mysqladmin -u root password "$DB_ROOT_PASS"
+  mysql -uroot -p"$DB_ROOT_PASS" <<-EOF
+	DELETE FROM mysql.user WHERE user = '$DB_USER';
+	FLUSH PRIVILEGES;
+	CREATE USER '$DB_USER'@'localhost' IDENTIFIED BY '$DB_PASS';
+	GRANT ALL PRIVILEGES ON *.* TO '$DB_USER'@'localhost' WITH GRANT OPTION;
+	CREATE USER '$DB_USER'@'%' IDENTIFIED BY '$DB_PASS';
+	GRANT ALL PRIVILEGES ON *.* TO '$DB_USER'@'%' WITH GRANT OPTION;
+	CREATE DATABASE $DB_NAME;
+EOF
+
+  killall mysqld
+  sleep 10
+}
+
+# Call all functions
+DB_FILES=$(echo /var/lib/mysql/*)
+DB_FILES="${DB_FILES#/var/lib/mysql/\*}"
+DB_FILES="${DB_FILES#/var/lib/mysql/lost+found}"
+if [ -z "$DB_FILES" ]; then
+  printf "Initializing empty /var/lib/mysql...\n"
+  __mysql_config
+  __start_mysql
+fi
+
+touch /var/lib/mysql/configured.txt
+kill 1

data/workload_mapper/mariadb/container/scripts/start.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+if [ -r /var/lib/mysql/configured.txt ]; then
+  rm -f /run/mysqld/mysqld.sock
+  exec /usr/bin/mysqld_safe
+else
+  /scripts/config_mariadb.sh
+fi
+

data/workload_mapper/rails/clue.rb

@@ -0,0 +1,16 @@
+if system.runs_service?("apache2") && system.has_file?("/usr/bin/rails")
+  apache_vhosts = system["unmanaged_files"].files.select { |f| /vhosts\.d/ =~ f.name }.map(&:name)
+  apache_vhosts.each do |vhost|
+    rails_env = system.read_config(vhost, "PassengerAppEnv")
+    rails_public = system.read_config(vhost, "DocumentRoot")
+
+    if rails_env && !rails_env.empty? && /public/.match(rails_public)
+      identify "rails", "web"
+      parameter "ports", ["3000:3000"]
+      parameter "links", ["db"]
+      rails_root = File.join(rails_public.gsub(/\/public/, ""), "")
+      extract rails_root, "data"
+      break # for now we only handle the first rails app that was found
+    end
+  end
+end

data/workload_mapper/rails/compose-template.yml

@@ -0,0 +1,5 @@
+rails:
+  build: ./rails
+  ports: :ports
+  links: :links
+  command: "/usr/sbin/apache2ctl -D FOREGROUND"

data/workload_mapper/rails/config/config/database.yml

@@ -0,0 +1,7 @@
+development:
+  adapter: mysql2
+  database: portus_development
+  encoding: utf8
+  host: db
+  username: <%= ENV['DB_USER'] %>
+  password: <%= ENV['DB_PASS'] %>

data/workload_mapper/rails/container/Dockerfile

@@ -0,0 +1,48 @@
+FROM opensuse:13.2
+
+# see update.sh for why all "apt-get install"s have to stay as one long line
+RUN zypper -n --gpg-auto-import-keys refresh
+RUN zypper -n --gpg-auto-import-keys update
+RUN zypper -n --gpg-auto-import-keys install ca-certificates-mozilla \
+apache2-devel \
+gcc \
+gcc-c++ \
+git-core \
+libcurl-devel \
+mariadb-client \
+libmysqlclient-devel \
+libopenssl-devel \
+libstdc++-devel \
+libxml2-devel \
+libxslt-devel \
+make \
+nodejs \
+patch \
+ruby2.1-devel \
+rubygem-bundler \
+zlib-devel \
+which
+
+RUN touch /etc/apache2/sysconfig.d/include.conf
+
+RUN gem install passenger -v 5.0.7
+RUN passenger-install-apache2-module.ruby2.1 -a
+
+#variable?
+RUN mkdir /srv/www/rails
+WORKDIR /srv/www/rails
+ADD ./data /srv/www/rails
+RUN chown -R wwwrun:www  /srv/www/rails
+
+RUN bundle config build.nokogiri --use-system-libraries
+RUN bundle install
+
+ADD apache2/sysconfig_apache2 /etc/sysconfig/apache2
+ADD apache2/httpd.conf.local /etc/apache2/httpd.conf.local
+ADD apache2/listen.conf /etc/apache2/listen.conf
+ADD apache2/rails_app_vhost.conf /etc/apache2/vhosts.d/
+RUN cat /etc/apache2/httpd.conf.local >> /etc/apache2/httpd.conf
+
+ENV COMPOSE=1
+
+EXPOSE 3000