comfy 0.2.0

data/lib/comfy/creator.rb

@@ -0,0 +1,200 @@
+require 'fileutils'
+require 'mixlib/shellout'
+require 'tmpdir'
+require 'json'
+require 'json-schema'
+require 'cloud-appliance-descriptor'
+
+# Creates given virtual machine images.
+class Comfy::Creator
+  attr_accessor :data
+
+  TIMESTAMP_FORMAT = '%Y%m%d%H%M'
+
+  NEEDLE_REPLACEMENTS = {
+    :$v => lambda { |instance| instance.send(:version_string) },
+    :$t => lambda { |instance| Time.new.strftime(TIMESTAMP_FORMAT) },
+    :$n => lambda { |instance| instance.data[:distro]['name'] },
+    :$g => lambda { |instance| instance.data[:groups].join(',') }
+  }
+  REPLACEMENT_REGEX = /\$[a-zA-Z]/
+
+  # Creates a creator instance.
+  #
+  # @param options [Hashie::Mash] hash-like structure with options
+  def initialize(options)
+    @data = options.clone
+  end
+
+  # Method representing thw whole creation process. Prepares enviroment,
+  # prepare files and starts packer job.
+  def create
+    logger.info('Preparing for image creation...')
+
+    data[:server_dir] = Dir.mktmpdir('comfy')
+    logger.debug("Server root directory: #{data[:server_dir]}")
+
+    prepare_data
+    logger.debug("Prepared data: #{data}")
+
+    templater = Comfy::Templater.new data
+    templater.prepare_files
+
+    packer_file = "#{data[:server_dir]}/#{data[:distribution]}.packer"
+    run_packer(packer_file)
+
+    # let's create cloud appliance descriptor files
+    if data[:description]
+      data[:formats].each do |format|
+        name = data[:distribution]
+        major = data[:distro][:version]['major_version']
+        minor = data[:distro][:version]['minor_version']
+        dir = File.join(data[:'output-dir'], "comfy_#{name}-#{major}.#{minor}_#{format}/")
+        File.write(File.join(dir, "#{data[:identifier]}.json"), description(format))
+      end
+    end
+  end
+
+  # Cleans everything from temporary directory
+  def clean
+    if data[:server_dir]
+      logger.debug("Cleaning temporary directory #{data[:server_dir]}.")
+      FileUtils.remove_dir(data[:server_dir])
+    end
+  end
+
+  private
+
+  # Method wrapping usage of packer tool.
+  #
+  # @param packer_file descriptor file with info for packer processing.
+  def run_packer(packer_file)
+    logger.info("Calling Packer - building distribution: '#{data[:distribution]}'.")
+    packer = Mixlib::ShellOut.new("packer validate #{packer_file}")
+    packer.run_command
+
+    fail Comfy::Errors::PackerValidationError, "Packer validation failed for distribution '#{data[:distribution]}': #{packer.stdout}" if packer.error?
+
+    packer = Mixlib::ShellOut.new("packer build -parallel=false #{packer_file}", timeout: 5400)
+    packer.live_stream = logger
+    packer.run_command
+
+    fail Comfy::Errors::PackerExecutionError, "Packer finished with error for distribution '#{data[:distribution]}': #{packer.stderr}" if packer.error?
+
+    logger.info("Packer finished successfully for distribution '#{data[:distribution]}'")
+  end
+
+  # Preparation of various data. Method prepares description file for packer and distribution
+  # preseed / kickstart file.
+  def prepare_data
+    description_file = "#{data[:'template-dir']}/#{data[:distribution]}/#{data[:distribution]}.description"
+    JSON::Validator.validate!(Comfy::DESCRIPTION_SCHEMA_FILE, description_file)
+
+    description = File.read(description_file)
+    data[:distro] = JSON.parse(description)
+    logger.debug("Data from description file: #{data[:distro]}")
+
+    data[:distro][:version] = choose_version
+    logger.debug("Version selected for build: #{data[:distro][:version]}")
+
+    data[:provisioners] = {}
+    data[:provisioners][:scripts] = Dir.glob(File.join(data[:'template-dir'], data[:distribution], 'scripts', '*'))
+    data[:provisioners][:files] = Dir.glob(File.join(data[:'template-dir'], data[:distribution], 'files', '*'))
+
+    data[:password] = password
+    logger.debug("Temporary password: '#{data[:password]}'")
+
+    data[:identifier] = replace_needles(data[:identifier])
+  end
+
+  # Choose_version is a method that selects required version from available versions.
+  def choose_version
+    version = data[:version]
+
+    available_versions = []
+    data[:distro]['versions'].each do |v|
+      available_versions << { major: v['major_version'].to_i, minor: v['minor_version'].to_i, patch: v['patch_version'].to_i, version: v }
+    end
+    available_versions.sort_by! { |v| [v[:major], v[:minor], v[:patch]] }.reverse!
+
+    return available_versions.first[:version] if version == :newest
+
+    version_parts = version.split('.')
+    fail Comfy::Errors::InvalidDistributionVersionError, "Version '#{version}' is not a valid distribution version" if version_parts.size > 3
+
+    version_parts.map! do |part|
+      fail Comfy::Errors::InvalidDistributionVersionError, "Version '#{version}' is not a valid distribution version" unless part =~ /\A\d+\z/
+
+      part.to_i
+    end
+
+    selected = available_versions.select { |v| v[:major] == version_parts[0] }
+    if version_parts.size > 1
+      selected = selected.select { |v| v[:minor] == version_parts[1] }
+
+      if version_parts.size > 2
+        selected = selected.select { |v| v[:patch] == version_parts[2] }
+      end
+    end
+
+    fail Comfy::Errors::NoSuchDistributionVersionError, "No version '#{version}' available for distribution '#{data[:distribution]}'" if selected.empty?
+
+    selected.sort_by { |v| [v[:major], v[:minor], v[:patch]] }.reverse.first[:version]
+  end
+
+  # Method generating a temporary random password used while creating image.
+  #
+  # @return [String] password
+  def password
+    o = [('a'..'z'), ('A'..'Z')].map(&:to_a).flatten
+    (0...100).map { o[rand(o.length)] }.join
+  end
+
+  # Description returns cloud appliance descriptor JSON. It uses information gathered from command line arguments
+  # and the config file.
+  #
+  # @param builder [Symbol] builder used in the description of the cloud appliance descriptor
+  #
+  # @return [Json] appliance descriptor in Json format
+  def description(builder)
+    # FIXME? mapping platforms/builders to formats is hardcoded for now, nothing else is supported
+    formats = { 'virtualbox' => 'ova', 'qemu' => 'qcow2' }
+    vm_dir = "comfy_#{data[:distribution]}-#{data[:distro][:version]['major_version']}.#{data[:distro][:version]['minor_version']}_#{builder}"
+    vm_name = "#{vm_dir}.#{formats[builder]}"
+    disk_path = File.join(data[:'output-dir'],vm_dir,vm_name)
+
+    os = Cloud::Appliance::Descriptor::Os.new distribution: data[:distribution], version: version_string
+    disk = Cloud::Appliance::Descriptor::Disk.new type: :os, format: formats[builder], path: disk_path
+
+    appliance = Cloud::Appliance::Descriptor::Appliance.new action: :registration, os: os, disks: [disk]
+    appliance.title = data[:distro]['name']
+    appliance.identifier = data[:identifier]
+    appliance.version = Time.new.strftime(TIMESTAMP_FORMAT)
+    appliance.groups = data[:groups]
+
+    appliance.to_json
+  end
+
+  # Replace needles in the argument.
+  # Replacements are picked from NEEDLE_REPLACEMENTS constant.
+  #
+  # @param [String] format_string string with needles to be replaced
+  #
+  # @return [String] format_string with all needles replaced
+  def replace_needles(format_string)
+    format_string.gsub(REPLACEMENT_REGEX) do |match|
+      NEEDLE_REPLACEMENTS.key?(match.to_sym) ? NEEDLE_REPLACEMENTS[match.to_sym].call(self) : match
+    end
+  end
+
+  # Simple method used to return the version string
+  #
+  # @return [String] string which contains major, minor, and patch version (if possible).
+  def version_string
+    result = []
+    result << data[:distro][:version]['major_version']
+    result << data[:distro][:version]['minor_version']
+    result << data[:distro][:version]['patch_version']
+    result.compact.join('.')
+  end
+end

data/lib/comfy/errors/invalid_distribution_version_error.rb

@@ -0,0 +1 @@
+class Comfy::Errors::InvalidDistributionVersionError < StandardError; end

data/lib/comfy/errors/no_such_distribution_version_error.rb

@@ -0,0 +1 @@
+class Comfy::Errors::NoSuchDistributionVersionError < StandardError; end

data/lib/comfy/errors/packer_error.rb

@@ -0,0 +1 @@
+class Comfy::Errors::PackerError < StandardError; end

data/lib/comfy/errors/packer_execution_error.rb

@@ -0,0 +1 @@
+class Comfy::Errors::PackerExecutionError < Comfy::Errors::PackerError; end

data/lib/comfy/errors/packer_validation_error.rb

@@ -0,0 +1 @@
+class Comfy::Errors::PackerValidationError < Comfy::Errors::PackerError; end

data/lib/comfy/errors.rb

@@ -0,0 +1,4 @@
+module Comfy::Errors
+  require File.join(File.dirname(__FILE__), "#{self.name.demodulize.underscore}", 'packer_error')
+  Dir.glob(File.join(File.dirname(__FILE__), "#{self.name.demodulize.underscore}", '*.rb')) { |error_file| require error_file.chomp('.rb') }
+end

data/lib/comfy/extensions/yell.rb

@@ -0,0 +1,8 @@
+require 'yell'
+
+# Monkeypatch for Yell
+class Yell::Logger
+  def <<(x)
+    info x.strip
+  end
+end

data/lib/comfy/settings.rb

@@ -0,0 +1,15 @@
+require 'settingslogic'
+
+class Comfy::Settings < Settingslogic
+  CONFIGURATION = 'comfy.yml'
+
+  # three possible configuration file locations in order by preference
+  # if configuration file is found rest of the locations are ignored
+  source "#{ENV['HOME']}/.comfy/#{CONFIGURATION}"\
+ if File.exist?("#{ENV['HOME']}/.comfy/#{CONFIGURATION}")
+  source "/etc/comfy/#{CONFIGURATION}"\
+ if File.exist?("/etc/comfy/#{CONFIGURATION}")
+  source "#{File.dirname(__FILE__)}/../../config/#{CONFIGURATION}"
+
+  namespace 'production'
+end

data/lib/comfy/templater.rb

@@ -0,0 +1,62 @@
+require 'tempfile'
+require 'erb'
+require 'fileutils'
+require 'tmpdir'
+
+# Class used for preparing and filling file templates.
+class Comfy::Templater
+  attr_reader :data
+
+  # Creates an instance of Templater.
+  #
+  # @param data [Hash] prepared data with distro, provisioners, files, password and identifier info.
+  def initialize(data)
+    @data = data
+  end
+
+  # Prepares *.json and *.cfg files from templates for selected distribution
+  def prepare_files
+    prepare_file('cfg')
+    prepare_file('packer', true)
+  end
+
+  private
+
+  # Method prepares .erb file with given data.
+  #
+  # @param name [String] type of file for preparation.
+  # @param packer [Boolean] (implicite value = false).
+  def prepare_file(name, packer = false)
+    logger.debug("Creating temporary #{name} file...")
+    tmp = Tempfile.new("comfy_#{name}")
+    logger.debug("Temporary file '#{tmp.path}' was created.")
+
+    output = File.join(data[:server_dir], "#{data[:distribution]}.#{name}")
+
+    logger.debug("Writing to temporary #{name} file...")
+    template_path = File.join(data[:'template-dir'], data[:distribution], "#{data[:distribution]}.#{name}.erb")
+    template_path = File.join(data[:'template-dir'], 'packer.erb') if packer
+    write_to_tmp(tmp, populate_template(template_path))
+
+    logger.debug("Copying #{name} file to its proper location...")
+    FileUtils.cp(tmp.path, output)
+
+    logger.debug("Cleaning temporary #{name} file...")
+    tmp.close(true)
+  end
+
+  def write_to_tmp(tmp, data)
+    tmp.write(data)
+    tmp.flush
+  end
+
+  # Actual filling of .erb fils with given info.
+  #
+  # @param template [String] path to template file.
+  def populate_template(template)
+    logger.debug("Populating template '#{template}'")
+    erb = ERB.new(File.read(template), nil, '-')
+    erb.filename = template
+    erb.result(binding)
+  end
+end

data/lib/comfy/version.rb

@@ -0,0 +1,3 @@
+module Comfy
+  VERSION = '0.2.0'
+end

data/lib/comfy.rb

@@ -0,0 +1,18 @@
+require 'comfy/settings'
+
+module Comfy
+  GEM_DIR = File.realdirpath(File.join(File.dirname(__FILE__), '..'))
+  DESCRIPTION_SCHEMA_FILE = File.join(GEM_DIR, 'schema', 'distribution_descriptor.schema')
+  TEMPLATE_DIR = Comfy::Settings['template-dir'] || File.join(GEM_DIR, 'lib', 'templates')
+  PACKER_FILE = File.join(TEMPLATE_DIR, 'packer.erb')
+
+  require 'active_support/all'
+
+  require 'comfy/extensions/yell'
+
+  require 'comfy/command_executioner'
+  require 'comfy/version'
+  require 'comfy/templater'
+  require 'comfy/creator'
+  require 'comfy/errors'
+end

data/lib/templates/centos/centos.cfg.erb

@@ -0,0 +1,47 @@
+### Kickstart
+## Fresh install, not an upgrade
+install
+url --url=http://mirror.nextlayer.at/centos/<%= @data[:distro][:version]['major_version'] %>.<%= @data[:distro][:version]['minor_version'] %>.<%= @data[:distro][:version]['patch_version'] %>/os/x86_64/
+lang en_US.UTF-8
+keyboard us
+timezone UTC
+## Disk Partitioning - how the boot loader should be installed, delete all partitions/mbr, then create
+bootloader --location=mbr
+zerombr
+clearpart --all --initlabel
+part  / --size=1  --grow --fstype ext4
+## Don't use GUI
+text
+## Don't configure X
+skipx
+## Firstboot - the druid that helps you to set up the system after install - disabled
+firstboot --disabled
+## Authentication
+## Sets up the authentication options - use shadow passwords, encryption for user passwords
+authconfig --enableshadow --passalgo=sha512
+auth --useshadow --enablemd5
+## Sets the system's root password
+rootpw <%= @data[:password] %>
+## Configure NICs - use dhcp - disable IPv6
+network --device=eth0 --bootproto dhcp --onboot=yes --noipv6
+network --device=eth1 --bootproto dhcp --onboot=yes --noipv6
+## Firewall - enable and open ssh port
+firewall --enabled --service=ssh
+## Selinux - disable
+selinux --disabled
+## Reboot the machine after the install
+reboot
+%packages --nobase
+## Add basic packages groups
+@core
+## Add basic packages
+wget
+curl
+bzip2
+## Add extra packages
+openssh-clients
+openssh-server
+%end
+%post
+/usr/bin/yum -y install sudo
+%end

data/lib/templates/centos/centos.description

@@ -0,0 +1,20 @@
+{
+  "name": "CentOS",
+  "versions": [{
+      "major_version": "7",
+      "minor_version": "1",
+      "patch_version": "1503",
+      "iso_url": "http://merlin.fit.vutbr.cz/mirrors/centos/7/isos/x86_64/CentOS-7-x86_64-NetInstall-1503.iso",
+      "iso_checksum": "498bb78789ddc7973fe14358822eb1b48521bbaca91c17bd132c7f8c903d79b3"
+    }],
+  "boot_command": "linux ks=http://{{ .HTTPIP }}:{{ .HTTPPort }}",
+  "qemu": {
+    "accelerator": "kvm",
+    "qemuargs": [ [ "-m", "1024M" ] ]
+  },
+  "virtualbox": {
+    "guest_os_type": "RedHat_64",
+    "vboxmanage": [ ["modifyvm", "{{.Name}}", "--memory", "1024"] ],
+    "guest_additions_mode": "disable"
+  }
+}

data/lib/templates/centos/files/10-ipv6.conf

@@ -0,0 +1,5 @@
+net.ipv6.conf.all.disable_ipv6 = 1
+net.ipv6.conf.default.disable_ipv6 = 1
+net.ipv6.conf.lo.disable_ipv6 = 1
+net.ipv6.conf.eth0.disable_ipv6 = 1
+net.ipv6.conf.eth1.disable_ipv6 = 1

data/lib/templates/centos/files/cloud.cfg

@@ -0,0 +1,101 @@
+# If this is set, 'root' will not be able to ssh in and they 
+# will get a message to login instead as the above $user (ubuntu)
+disable_root: False
+user: root
+ssh_pwauth: False
+ssh_deletekeys: True
+ssh_genkeytypes: ['rsa', 'dsa']
+ssh_svcname: sshd
+
+# This will cause the set+update hostname module to not operate (if true)
+preserve_hostname: false
+cc_ready_cmd: ['/bin/true']
+mount_default_fields: [~, ~, 'auto', 'defaults,nofail', '0', '2']
+syslog_fix_perms: ~
+manage_etc_hosts: True
+
+# Update and upgrade system on first boot
+apt_preserve_sources_list: True
+package_update: True
+package_upgrade: True
+package_reboot_if_required: True
+
+# work only with OpenNebula, use network based datasource,
+# so that we can successfully resolve IPv4 based hostname
+disable_ec2_metadata: True
+datasource_list: ['OpenNebula']
+datasource:
+  OpenNebula:
+    dsmode: net
+
+# The modules that run in the 'init' stage
+cloud_init_modules:
+ - migrator
+ - seed_random
+ - bootcmd
+ - write-files
+ - growpart
+ - resizefs
+ - set_hostname
+ - update_hostname
+ - update_etc_hosts
+ - ca-certs
+ - rsyslog
+ - users-groups
+ - ssh
+
+# The modules that run in the 'config' stage
+cloud_config_modules:
+# Emit the cloud config ready event
+# this can be used by upstart jobs for 'start on cloud-config'.
+ - emit_upstart
+ - disk_setup
+ - mounts
+ - ssh-import-id
+ - locale
+ - set-passwords
+ - grub-dpkg
+ - apt-pipelining
+ - apt-configure
+ - package-update-upgrade-install
+ - landscape
+ - timezone
+ - puppet
+ - chef
+ - salt-minion
+ - mcollective
+ - disable-ec2-metadata
+ - runcmd
+ - byobu
+
+# The modules that run in the 'final' stage
+cloud_final_modules:
+ - rightscale_userdata
+ - scripts-per-once
+ - scripts-per-boot
+ - scripts-per-instance
+ - scripts-user
+ - ssh-authkey-fingerprints
+ - keys-to-console
+ - phone-home
+ - final-message
+ - power-state-change
+
+# System and/or distro specific settings
+# (not accessible to handlers/transforms)
+system_info:
+   # This will affect which distro class gets used
+   distro: rhel
+   # Other config here will be given to the distro class and/or path classes
+   paths:
+      cloud_dir: /var/lib/cloud/
+      templates_dir: /etc/cloud/templates/
+      upstart_dir: /etc/init/
+   package_mirrors:
+     - arches: [default]
+       failsafe:
+         primary: http://http.us.debian.org/debian/
+         security: http://security.debian.org/
+   ssh_svcname: sshd
+
+# vim:syntax=yaml

data/lib/templates/centos/files/fail2ban.local

@@ -0,0 +1,3 @@
+[Definition]
+
+logtarget = SYSLOG

data/lib/templates/centos/files/getty@ttyS0.service

@@ -0,0 +1,47 @@
+#  This file is part of systemd.
+#
+#  systemd is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU Lesser General Public License as published by
+#  the Free Software Foundation; either version 2.1 of the License, or
+#  (at your option) any later version.
+
+[Unit]
+Description=Getty on %I
+Documentation=man:agetty(8) man:systemd-getty-generator(8)
+Documentation=http://0pointer.de/blog/projects/serial-console.html
+After=systemd-user-sessions.service plymouth-quit-wait.service
+After=rc-local.service
+
+# If additional gettys are spawned during boot then we should make
+# sure that this is synchronized before getty.target, even though
+# getty.target didn't actually pull it in.
+Before=getty.target
+IgnoreOnIsolate=yes
+
+# On systems without virtual consoles, don't start any getty. Note
+# that serial gettys are covered by serial-getty@.service, not this
+# unit.
+ConditionPathExists=/dev/tty0
+
+[Service]
+# the VT is cleared by TTYVTDisallocate
+ExecStart=-/sbin/agetty --autologin root --noclear %I $TERM
+Type=idle
+Restart=always
+RestartSec=0
+UtmpIdentifier=%I
+TTYPath=/dev/%I
+TTYReset=yes
+TTYVHangup=yes
+TTYVTDisallocate=yes
+KillMode=process
+IgnoreSIGPIPE=no
+SendSIGHUP=yes
+
+# Unset locale for the console getty since the console has problems
+# displaying some internationalized messages.
+Environment=LANG= LANGUAGE= LC_CTYPE= LC_NUMERIC= LC_TIME= LC_COLLATE= LC_MONETARY= LC_MESSAGES= LC_PAPER= LC_NAME= LC_ADDRESS= LC_TELEPHONE= LC_MEASUREMENT= LC_IDENTIFICATION=
+
+[Install]
+WantedBy=getty.target
+Alias=getty@ttys0.service

data/lib/templates/centos/files/grub

@@ -0,0 +1,11 @@
+GRUB_TIMEOUT=5
+GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
+GRUB_DEFAULT=saved
+GRUB_DISABLE_SUBMENU=true
+GRUB_TERMINAL_OUTPUT="console"
+GRUB_CMDLINE_LINUX="vconsole.keymap=us crashkernel=auto  vconsole.font=latarcyrheb-sun16 rhgb quiet net.ifnames=0 biosdevname=0 console=ttys0,115200n8 console=tty0"
+#GRUB_CMDLINE_LINUX="console=ttys0,115200n8 console=tty0"
+GRUB_DISABLE_RECOVERY="true"
+GRUB_TERMINAL="serial"
+GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"
+

data/lib/templates/centos/files/iptables-multiport.local

@@ -0,0 +1,6 @@
+[Definition]
+
+actionban = iptables -I fail2ban-<name> 1 -s <ip> -j REJECT
+
+actionunban = iptables -D fail2ban-<name> -s <ip> -j REJECT
+

data/lib/templates/centos/files/jail.local

@@ -0,0 +1,17 @@
+[DEFAULT]
+
+# Seznam vygenerovany skriptem /software/meta-admin/scripts/get_nodes_ips
+# Vygeneruje pouze C site, tzn. je tam o nekolik set hostu vic nez ve skutecnosti
+
+ignoreip = 127.0.0.1 147.228.1.0/24 147.251.17.0/24 147.228.240.0/24 147.228.241.0/24 147.231.11.0/24 147.231.18.0/24 147.251.11.0/24 147.251.252.0/24 147.251.254.0/24 147.251.3.0/24 147.251.84.0/24 147.251.9.0/24 195.113.0.0/24 195.113.123.0/24 195.113.209.0/24 195.113.214.0/24 78.128.210.0/24
+
+[ssh]
+
+enabled = true
+port    = ssh
+filter  = sshd
+logpath  = /var/log/auth.log
+maxretry = 100
+findtime = 86400
+bantime  = 1209600
+