falkorlib 0.8.5 → 0.8.6

data/templates/mkdocs/mkdocs.yml.erb

@@ -2,12 +2,13 @@ site_name: <%= config[:sitename] %>
 nav:
 - Home: 'index.md'
 - Organization: 'layout.md'
-- Getting Started:
-  - Pre-Requisites: 'setup/preliminaries.md'
-  - Installation: 'setup/install.md'
+- Pre-Requisites and Repository Setup: 'setup.md'
+<% if File.exists?(File.join(config[:rootdir], 'Vagrantfile')) %>
+- Overview of the Vagrant configuration: 'vagrant.md'
+<% end %>
 - Contributing:
   - Overview: 'contributing/index.md'
   - Directory Layout: 'layout.md'
-  - Repository Setup and Developments: 'contributing/setup.md'
+  - Repository Setup and Git Developments: 'contributing/setup.md'
   - Semantic Versioning: 'contributing/versioning.md'
 theme: readthedocs

data/templates/vagrant/Vagrantfile.erb

@@ -1,6 +1,6 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
-# Time-stamp: <Fri 2018-04-27 16:53 svarrette>
+# Time-stamp: <Mon 2020-04-20 15:32 svarrette>
 ###########################################################################################
 #             __     __                          _    __ _ _
 #             \ \   / /_ _  __ _ _ __ __ _ _ __ | |_ / _(_) | ___
@@ -11,7 +11,6 @@
 ###########################################################################################
 require 'yaml'
 require 'ipaddr'
-require 'deep_merge'
 require 'pp'
 require 'erb'
 
@@ -19,17 +18,19 @@ require 'erb'
 # For more information on the below plugins:
 # - https://github.com/oscar-stack/vagrant-hosts
 # - https://github.com/dotless-de/vagrant-vbguest
-# - https://github.com/emyl/vagrant-triggers
 # - https://github.com/fgrehm/vagrant-cachier
+# - https://github.com/vagrant-libvirt/vagrant-libvirt
 # Terminal-table is a nice ruby gem for automatically print tables with nice layout
 ###
 [ 'vagrant-hosts',
   'vagrant-vbguest',
-  'vagrant-triggers',
   'vagrant-cachier',
+  # 'vagrant-libvirt',
+  'deep_merge',
   'terminal-table' ].each do |plugin|
   abort "Install the  '#{plugin}' plugin with 'vagrant plugin install #{plugin}'" unless Vagrant.has_plugin?("#{plugin}")
 end
+require 'deep_merge'
 require 'terminal-table'
 
 ### Global variables ###
@@ -39,6 +40,8 @@ VAGRANTFILE_API_VERSION = "2"
 # Eventually a local YAML configuration for the deployment
 TOP_SRCDIR  = File.expand_path File.dirname(__FILE__)
 TOP_CONFDIR = File.join(TOP_SRCDIR, 'vagrant')
+TOP_PUPPETDIR = File.join(TOP_CONFDIR, 'puppet')
+TOP_PUPPET_MODULEDIR = File.join(TOP_PUPPETDIR, 'modules')
 config_file = File.join(TOP_CONFDIR, 'config.yaml')
 
 #SHARED_DIR  = File.join('vagrant', 'shared')
@@ -61,6 +64,7 @@ DEFAULT_SETTINGS = {
   },
   # Default Boxes
   :boxes => {
+    :centos8  => '<%= config[:boxes][:centos8] %>',
     :centos7  => '<%= config[:boxes][:centos7] %>',
     :debian8  => '<%= config[:boxes][:debian8] %>',
     :ubuntu14 => '<%= config[:boxes][:ubuntu14] %>'
@@ -72,7 +76,9 @@ DEFAULT_SETTINGS = {
   #   :os: <os>       # from the configured boxes
   #   :ram: <ram>
   #   :vcpus: <vcpus>
-  #   :role: <role>   # supported: [ 'XX', 'YY' ]
+  #   :role: <role>    # Puppet role
+  #   :puppet_modules => [ ... ]
+  #   :provider => <provider>  # if not virtualbox (Ex: 'libvirt')
   :vms => {
     # IF in single mode, below is the definition of the box to deploy
     'default' => {
@@ -84,7 +90,7 @@ DEFAULT_SETTINGS = {
 
 # List of default provisioning scripts
 DEFAULT_PROVISIONING_SCRIPTS = [
-  "vagrant/bootstrap.sh"
+  "vagrant/scripts/bootstrap.sh"
 ]
 
 # Load the settings (eventually overwritten using values from the yaml file 'config/vagrant.yaml')
@@ -96,13 +102,13 @@ if File.exist?(config_file)
 end
 #puts settings.to_yaml
 # abort 'end'
-#pp settings
 abort "Undefined settings" if settings.nil?
 
 ############################################################
 # Complete configuration of the boxes to deploy
-defaults = settings[:defaults]
-network  = settings[:network]
+defaults   = settings[:defaults]
+network    = settings[:network]
+puppet_dir = File.join('vagrant', 'puppet') # Relative path to Local puppet directory
 
 
 ############################################################
@@ -128,6 +134,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   ## Uncomment the below part if you want and additional shared directory
   # config.vm.synced_folder "vagrant/shared", "/shared", mount_options: ['dmode=777','fmode=777'],
   #                         type: "virtualbox" # Shared directory for users
+  if Dir.exist?(File.join(TOP_PUPPETDIR, 'hieradata'))
+    config.vm.synced_folder "#{puppet_dir}/hieradata", "/tmp/vagrant-puppet/hieradata", type: "virtualbox"
+  end
 
   # network settings
   ipaddr   = IPAddr.new network[:range]
@@ -137,7 +146,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   # cosmetics for the post-up message
   __table = {
     :title    => "Virtual environment deployed on Vagrant",
-    :headings => [ 'Name', 'Hostname', 'OS', 'vCPU/RAM', 'Description', 'IP' ],
+    :headings => [ 'Name', 'Hostname', 'OS', 'vCPU/RAM', 'Role', 'Description', 'IP' ],
     :rows => [],
   }
 
@@ -146,12 +155,15 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   settings[:vms].each do |name, node|
     hostname = node[:hostname] ? node[:hostname] : name
     domain   = network[:domain]
-    fqdn  =    "#{hostname}.#{domain}"
-    os    =    node[:os]       ? node[:os].to_sym : defaults[:os].to_sym
-    ram   =    node[:ram]      ? node[:ram]       : defaults[:ram]
-    vcpus =    node[:vcpus]    ? node[:vcpus]     : defaults[:vcpus]
-    role  =    node[:role]     ? node[:role]      : 'default'
-    desc  =    node[:desc]     ? node[:desc]      : 'n/a'
+    fqdn     = "#{hostname}.#{domain}"
+    boxname  = defaults[:os].to_s.downcase.gsub(/([^\d]+)(\d+)/, '\\1-\\2')
+    name     = boxname if name == 'default'
+    os       = node[:os]       ? node[:os].to_sym : defaults[:os].to_sym
+    ram      = node[:ram]      ? node[:ram]       : defaults[:ram]
+    vcpus    = node[:vcpus]    ? node[:vcpus]     : defaults[:vcpus]
+    role     = node[:role]     ? node[:role]      : 'default'
+    desc     = node[:desc]     ? node[:desc]      : 'n/a'
+    puppet_modules = node[:puppet_modules] ? node[:puppet_modules] : []
 
     abort "Non-existing box OS '#{os}' for the VM '#{name}'" if  settings[:boxes][os.to_sym].nil?
     abort "Empty IP address range" if ip_range.empty?
@@ -171,19 +183,52 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
       end
 
       # role specialization
-      if role == 'XX'
-        c.vm.provision "shell" do |s|
-          s.path = "scripts/XX_install.sh"
-          s.args = [ '--debug' ]
-          s.keep_color = true
-        end
+      # if role == 'XX'
+      #   c.vm.provision "shell" do |s|
+      #     s.path = "scripts/XX_install.sh"
+      #     s.args = [ '--debug' ]
+      #     s.keep_color = true
+      #   end
+      # end
+
+      ########## PUPPET PROVISIONNING #########
+      # Install required puppet modules, if any
+      puppet_modules.each do |mod|
+        config.vm.provision "shell",
+                            inline: "puppet module install --target-dir #{TOP_PUPPET_MODULEDIR} #{mod}"
       end
-    end
-    __table[:rows] << [ name, fqdn, os.to_sym, "#{vcpus}/#{ram}", desc, ip]
-  end  # settings
 
-  config.trigger.after :up do
-    puts Terminal::Table.new __table
-  end
+      # A role fact MUST be defined (default: 'default') as it is used in Hiera config
+      c.vm.provision "shell",
+                     inline: "echo '{ \"role\": \"#{role}\" }' > /opt/puppetlabs/facter/facts.d/custom.json",
+                     keep_color: true
+      ################ Puppet install ##############
+      c.vm.provision "puppet" do |puppet|
+        puppet.options = [
+          '--verbose',
+          '-t',
+          # '--debug',
+          "--yamldir /vagrant/#{puppet_dir}/hieradata",
+        ]
+        # Does not work :(
+        # puppet.facter = { "role" => "#{role}" }
+        puppet.hiera_config_path = "#{puppet_dir}/hiera.yaml"
+        puppet.module_path       = [ "#{puppet_dir}/site", "#{puppet_dir}/modules" ]
+        puppet.manifests_path    = "#{puppet_dir}/manifests"
+        puppet.manifest_file     = File.exists?("/vagrant/#{puppet_dir}/manifests/#{role}.pp") ? "#{role}.pp" : 'default.pp'
+      end # puppet
+
+      __table[:rows] << [ name, fqdn, os.to_sym, "#{vcpus}/#{ram}", role, desc, ip]
+
+      if settings[:vms].keys.last == name
+        c.trigger.after :up do |trigger|
+          trigger.info = (Terminal::Table.new __table).to_s
+          trigger.warn = <<-EOF
+- Virtual Vagrant infrastructure successfully deployed!
+EOF
+        end
+      end
 
+    end # config.vm.define
+  end  # settings
 end

data/templates/vagrant/vagrant/config.yaml.sample

@@ -1,5 +1,5 @@
 # -*- mode: yaml; -*-
-# Time-stamp: <Fri 2018-04-27 16:47 svarrette>
+# Time-stamp: <Sat 2020-04-18 18:00 svarrette>
 ################################################################################
 # Complementary configuration for Vagrant
 # You can overwrite here the default settings defined in ../Vagrantfile and
@@ -11,6 +11,7 @@
 # Format:
 #   :<os><version>: <username>/<box> # see https://vagrantcloud.com
 # :boxes:
+#   :centos8: 'centos/8'
 #   :centos7: 'centos/7'
 #   :debian8: 'debian/contrib-jessie64'
 #   :ubuntu14: 'ubuntu/trusty64'
@@ -28,8 +29,6 @@
 # :network:
 #   :domain: 'vagrant.dev'       # network domain to use
 #   :range: '10.10.1.0/24'       # IP range to use
-#   :client_ip_start_offset: 100 # Note: compute nodes will have xx.xx.xx.254
-#   # client/compute nodes VMs will start on xx.xx.xx.<client_ip_start_offset+1>
 
 #___________________________________________________________
 # VMs / Vagrant boxes to define apart from the compute nodes
@@ -40,7 +39,10 @@
 #   :os: <os>       # from the configured boxes
 #   :ram: <ram>
 #   :vcpus: <vcpus>
-#   :role: <role>   # supported: [ 'controller', 'frontend' ]
+#   :role: <role>
+#   :puppet_modules:
+#   - '<name1>-<mod1>'
+#   - '<name2>-<mod2>'
 #
 # :vms:
 #   'anothervm':
@@ -49,3 +51,5 @@
 #     :vcpus: 2
 #     :desc: 'VM #2 (secondary)'
 #     :role: role2
+#     :puppet_modules:
+#     - ...

data/templates/vagrant/vagrant/puppet/Makefile

@@ -0,0 +1,40 @@
+##################################################################################
+# Makefile - Configuration file for GNU make (http://www.gnu.org/software/make/)
+# Time-stamp: <Wed 2020-04-15 13:13 svarrette>
+#
+# Copyright (c) 2018 Sebastien Varrette <Sebastien.Varrette@uni.lu>
+#
+############################## Variables Declarations ############################
+
+CURDIR = $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
+
+MODULES_DIR   = modules
+MANIFESTS_DIR = manifests
+HIERADATA_DIR = hieradata
+
+MODULES = $(sort $(dir $(wildcard $(MODULES_DIR)/*/)))
+
+# all: setup
+
+# .PHONY: setup
+# setup:
+# 	r10k puppetfile install -v
+# 	$(MAKE) check
+
+# check:
+# 	puppet module --modulepath $(CURDIR)/modules --color true list --tree
+
+info:
+	@echo "MODULES = $(MODULES)"
+
+clean:
+	@if [ -f "$(HIERADATA_DIR)/custom.yaml" ]; then \
+		echo "=> remove custom hiera"; \
+		rm -f $(HIERADATA_DIR)/custom.yaml;\
+	fi
+
+distclean: clean
+	@if [ -n "$(MODULES)" ]; then \
+		echo "=> remove puppet modules"; \
+		echo "(simulating) rm -rf $(MODULES_DIR)/*";\
+	fi

data/templates/vagrant/vagrant/puppet/hiera.yaml

@@ -0,0 +1,36 @@
+# -*- mode: yaml; -*-
+# Time-stamp: <Wed 2020-04-15 10:29 svarrette>
+################################################################################
+#             _   _ _                   ____             __ _
+#            | | | (_) ___ _ __ __ _   / ___|___  _ __  / _(_) __ _
+#            | |_| | |/ _ \ '__/ _` | | |   / _ \| '_ \| |_| |/ _` |
+#            |  _  | |  __/ | | (_| | | |__| (_) | | | |  _| | (_| |
+#            |_| |_|_|\___|_|  \__,_|  \____\___/|_| |_|_| |_|\__, |
+#                                                             |___/
+################################################################################
+# Puppet Hiera configuration, which is used to configure the hierarchy, which
+# backend(s) to use, and settings for each backend.
+# - Doc: https://docs.puppet.com/puppet/latest/hiera_config_yaml_5.html
+#
+---
+version: 5      # below version 5 are deprecated starting puppet 4.9
+
+### default datadir and backend for hierarchy levels.
+defaults:  # Used for any hierarchy level that omits these keys.
+  datadir: hieradata  # Should be absolute with vagrant
+  data_hash: yaml_data  # Use the built-in YAML backend.
+
+hierarchy:
+  #______________________
+  - name: "Custom data"
+    path: "custom.yaml"
+  #___________________________
+  - name: "Role Specific data"
+    path: "roles/%{::role}.yaml"
+  #______________________
+  - name: "Local settings (eventually from vagrant config)"
+    path: "locals.yaml"
+  #______________________
+  - name: "Default data"  # Human-readable name.
+    path: "defaults.yaml"  # File path, relative to datadir.
+    #              ^^^^^ IMPORTANT: include the file extension!

data/templates/vagrant/vagrant/puppet/hieradata/.gitignore

@@ -0,0 +1,2 @@
+locals.yaml
+custom.yaml

data/templates/vagrant/vagrant/puppet/hieradata/defaults.yaml

@@ -0,0 +1,4 @@
+# Default Hiera values for Puppet
+---
+
+msg: "Vagrant provisioning with Hiera enabled"

data/templates/vagrant/vagrant/puppet/hieradata/roles/example.yaml

@@ -0,0 +1,13 @@
+# -*- mode: yaml -*-
+# Time-stamp: <Wed 2020-04-15 16:28 svarrette>
+###########################################################################
+# Puppet hiera settings for 'example' role
+---
+
+# Profiles key may be used to include profile classes
+profiles:
+- '::profiles::pxe::server'
+
+msg: 'Provisioning Sample PXE server'
+
+pxe::syslinux_version: 'system'

data/templates/vagrant/vagrant/puppet/manifests/default.pp

@@ -0,0 +1,23 @@
+# File::      <tt>default.pp</tt>
+# Author::    UL HPC Team (<hpc-team@uni.lu>)
+# Copyright:: Copyright (c) 2020 Sebastien Varrette
+# License::   Apache-2.0
+#
+# ------------------------------------------------------------------------------
+node default {
+  # Setup pre and post run stages
+  # Typically these are only needed in special cases but are good to have
+  stage { ['pre', 'post']: }
+  Stage['pre'] -> Stage['main'] -> Stage['post']
+  Package{ ensure => 'present' }
+
+  # Check that the hiera configuration is working...
+  # if not the puppet provisioning will fail.
+  $msg=lookup('msg')
+  notice("Role: ${::role}")
+  notice("Welcome Message: '${msg}'")
+
+  if (lookup('profiles', undef, undef, false)) {
+    lookup('profiles', {merge => unique}).contain
+  }
+}

data/templates/vagrant/vagrant/puppet/modules/.gitignore

@@ -0,0 +1,3 @@
+*
+
+!.gitignore

data/templates/vagrant/vagrant/puppet/site/profiles/manifests/pxe/server.pp

@@ -0,0 +1,12 @@
+# -*- mode: puppet; -*-
+# Time-stamp: <Wed 2020-04-15 16:28 svarrette>
+###########################################################################################
+# Profile (base) class used for a PXE server general settings
+#
+# Documentation on puppet/pxe class: https://forge.puppet.com/puppet/pxe
+
+class profiles::pxe::server
+{
+  include ::tftp
+  include ::pxe
+}

data/templates/vagrant/vagrant/{bootstrap.sh → scripts/bootstrap.sh}

@@ -1,13 +1,13 @@
 #!/usr/bin/env bash
-# Time-stamp: <Fri 2018-04-27 15:53 svarrette>
+# Time-stamp: <Sun 2020-04-19 12:39 svarrette>
 ###########################################################################################
 # __     __                          _     ____              _       _
 # \ \   / /_ _  __ _ _ __ __ _ _ __ | |_  | __ )  ___   ___ | |_ ___| |_ _ __ __ _ _ __
 #  \ \ / / _` |/ _` | '__/ _` | '_ \| __| |  _ \ / _ \ / _ \| __/ __| __| '__/ _` | '_ \
-#   \ V / (_| | (_| | | | (_| | | | | |_  | |_) | (_) | (_) | |_\__ \ |_| | | (_| | |_) |
+    #   \ V / (_| | (_| | | | (_| | | | | |_  | |_) | (_) | (_) | |_\__ \ |_| | | (_| | |_) |
 #    \_/ \__,_|\__, |_|  \__,_|_| |_|\__| |____/ \___/ \___/ \__|___/\__|_|  \__,_| .__/
 #              |___/                                                              |_|
-#              Copyright (c) 2017 Sebastien Varrette <sebastien.varrette@uni.lu>
+#            Copyright (c) 2017-2020 Sebastien Varrette <sebastien.varrette@uni.lu>
 ###########################################################################################
 # (prefered) way to see a Vagrant box configured.
 #
@@ -23,21 +23,15 @@ STARTDIR="$(pwd)"
 SCRIPTFILENAME=$(basename $0)
 SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-
 MOTD="/etc/motd"
 DOTFILES_DIR='/etc/dotfiles.d'
 DOTFILES_URL='https://github.com/ULHPC/dotfiles.git'
 EXTRA_PACKAGES=
 
-# List of default packages to install
-COMMON_DEFAULT_PACKAGES="wget figlet git screen bash-completion rsync nmap vim python-pip htop direnv"
-
-# Easybuild
-#export EASYBUILD_MODULES_TOOL=Lmod
-export EASYBUILD_MODULE_NAMING_SCHEME=CategorizedModuleNamingScheme
-EB_INSTALL_SCRIPT='/tmp/bootstrap_eb.py'
-EB_INSTALL_SCRIPT_URL='https://raw.githubusercontent.com/easybuilders/easybuild-framework/develop/easybuild/scripts/bootstrap_eb.py'
+TITLE=$(hostname -s)
 
+# List of default packages to install
+COMMON_DEFAULT_PACKAGES="ruby wget figlet git screen bash-completion rsync vim htop net-tools mailx"
 
 ######
 # Print information in the following form: '[$2] $1' ($2=INFO if not submitted)
@@ -69,28 +63,26 @@ EOF
 
 #######################  Per OS Bootstrapping function ##########################
 setup_redhat() {
+    info "fix screen title"
+    touch    /etc/sysconfig/bash-prompt-screen
+    chmod +x /etc/sysconfig/bash-prompt-screen
+
     info "Running yum update"
     yum update -y  >/dev/null
 
     info "Installing default packages"
     yum install -y epel-release
-    yum install -y ${COMMON_DEFAULT_PACKAGES} bind-utils ${EXTRA_PACKAGES}  >/dev/null
-
-    info "Uninstalling (eventually) existing Puppet installation"
-    yum erase -y puppet puppetlabs-release >/dev/null
-
-    info "Adding repo for Puppet 4"
-    rpm -ivh https://yum.puppetlabs.com/puppetlabs-release-pc1-el-$1.noarch.rpm
+    yum install -y ${COMMON_DEFAULT_PACKAGES} ruby-devel gdbm-devel bind-utils ${EXTRA_PACKAGES}  >/dev/null
 
     sleep 1
-    info "Installing Puppet and its dependencies"
-    yum install -y puppet-agent >/dev/null
-
-    info "installing Environment modules and  LMod"
-    yum install -y environment-modules Lmod
-
     yum groupinstall -y "Development Tools"
-    yum install -y openssl-devel libssl-dev libopenssl-devel ncurses-devel libibverbs-dev libibverbs-devel, rdma-core-devel bzip2-devel readline-devel libsqlite3x-devel
+
+    # get OS version
+    os_version=$(rpm -q --qf "%{VERSION}" $(rpm -q --whatprovides redhat-release))
+    info "Adding Puppet Labs repo and installing Puppet for RHEL/CentOS ${os_version}"
+    # Get the major version
+    yum install -y https://yum.puppetlabs.com/puppet-release-el-$(echo $os_version | cut -d '.' -f 1).noarch.rpm
+    yum install -y puppet
 }
 
 setup_apt() {
@@ -190,8 +182,14 @@ setup_motd() {
 EOF
     if [ -n "${has_figlet}" ]; then
         cat <<EOF >> ${motd}
-$(${has_figlet} -w 80 -c "Virtual $(hostname -s)")
+$(${has_figlet} -w 100 -c "Virt. ${TITLE}")
 EOF
+        if [ -n "${SUBTITLE}" ]; then
+            echo "$(${has_figlet} -w 100 -c \"${SUBTILE}\")" >> ${motd}
+        fi
+    fi
+    if [ -n "${DESC}" ]; then
+        echo "${DESC}" >> ${motd}
     fi
     cat <<EOF >> ${motd}
 ================================================================================
@@ -202,7 +200,6 @@ EOF
 EOF
 }
 
-
 ######################################################################################
 [ $UID -gt 0 ] && error "You must be root to execute this script (current uid: $UID)"
 
@@ -228,8 +225,8 @@ case "$OSTYPE" in
     *)        echo "unknown: $OSTYPE"; exit 1;;
 esac
 
-[ -f /usr/bin/puppet ] || ln -s /opt/puppetlabs/puppet/bin/puppet /usr/bin/puppet
-[ -f /usr/bin/facter ] || ln -s /opt/puppetlabs/puppet/bin/facter /usr/bin/facter
+# [ -f /usr/bin/puppet ] || ln -s /opt/puppetlabs/puppet/bin/puppet /usr/bin/puppet
+# [ -f /usr/bin/facter ] || ln -s /opt/puppetlabs/puppet/bin/facter /usr/bin/facter
 
 setup_dotfiles
 setup_motd