propro 0.1.0

data/ext/bash/lib/propro.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+
+set -e
+set -u
+
+export PROPRO_LOG_FILE="/root/provision.log"
+export PROPRO_FULL_LOG_FILE="/root/full_provision.log"
+export PROPRO_LOG_USE_COLOR="yes"
+export PROPRO_DISABLE_LOG="no"
+
+>$PROPRO_FULL_LOG_FILE
+exec > >(tee $PROPRO_FULL_LOG_FILE)
+exec 2>&1
+
+function log {
+  echo -e "$1"
+
+  if is-yes $PROPRO_DISABLE_LOG; then
+    return 0
+  fi
+
+  if [ $PROPRO_LOG_FILE ]; then
+    touch $PROPRO_LOG_FILE
+    echo -e "$1" >> $PROPRO_LOG_FILE
+  fi
+}
+
+# $1 text
+function section {
+  local msg="#### $1"
+  log ""
+  if is-yes $PROPRO_LOG_USE_COLOR; then
+    log "\e[32m\e[1m$msg\e[0m"
+  else
+    log "$msg"
+  fi
+}
+
+# $1 text
+function announce {
+  if is-yes $PROPRO_LOG_USE_COLOR; then
+    log "\e[34m\e[1m--->\e[0m $1"
+  else
+    log "---> $1"
+  fi
+}
+
+# $1 text
+function announce-item {
+  if is-yes $PROPRO_LOG_USE_COLOR; then
+    log "     - \e[36m$1\e[0m"
+  else
+    log "     - $1"
+  fi
+}
+
+function finished {
+  if is-yes $PROPRO_LOG_USE_COLOR; then
+    log "\e[35m\e[1m     Fin.\e[0m"
+  else
+    log "     Fin."
+  fi
+  log ""
+}
+
+function get-tmp-dir {
+  mktemp -d
+}
+
+# $1 "yes" or "no"
+function is-yes {
+  if [ $1 == "yes" ]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+# $1 "yes" or "no"
+function is-no {
+  if [ $1 == "no" ]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+# $1 comma separated list
+#
+# example:
+# > $ csl-to-wsl "item1,item2,item3"
+# > item1 item2 item3
+function csl-to-wsl {
+  echo "$1" | sed 's/,/ /g'
+}
+
+# $1 path or relative uri
+#
+# example:
+# > $ path-to-id example.com/neat/stuff
+# > example_com_neat_stuff
+function path-to-id {
+  echo "$1" | sed -r 's/[-\.:\/\]/_/g'
+}

data/ext/bash/lib/redis.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+export REDIS_VERSION="2.8.4" # @specify
+export REDIS_USER="redis"
+export REDIS_CONF_FILE="/etc/redis.conf"
+export REDIS_DATA_DIR="/var/lib/redis"
+export REDIS_FORCE_64BIT="no" # @specify Force 64bit build even if available memory is lte 4GiB
+REDIS_URL="http://download.redis.io/releases/redis-$REDIS_VERSION.tar.gz"
+
+function redis-install {
+  local tmpdir=$(get-tmp-dir)
+  cd "$tmpdir"
+
+  announce "Download $REDIS_VERSION"
+  download $REDIS_URL
+
+  announce "Extract"
+  extract redis-$REDIS_VERSION.tar.gz
+  cd redis-$REDIS_VERSION
+
+  if [ $(get-ram-bytes) -gt 4294967296 ] || is-yes $REDIS_FORCE_64BIT; then
+    announce "Compile"
+    make
+  else
+    announce "Compile (32bit, available memory <= 4GiB)"
+    install-packages libc6-dev-i386
+    make 32bit
+  fi
+
+  announce "Install $REDIS_VERSION"
+  make install
+
+  announce "Add Redis user: $REDIS_USER"
+  useradd -r $REDIS_USER
+
+  announce "Create Redis directories"
+  as-user-mkdir $REDIS_USER $REDIS_DATA_DIR
+
+  announce "Copy Redis config to $REDIS_CONF_FILE"
+  cp ./redis.conf $REDIS_CONF_FILE
+
+  cd ~/
+  rm -rf "$tmpdir"
+
+  announce "Update Redis config"
+  tee -a $REDIS_CONF_FILE <<EOT
+syslog-enabled yes
+syslog-ident redis
+dir $REDIS_DATA_DIR
+EOT
+
+  announce "Create upstart for Redis"
+  tee /etc/init/redis.conf <<EOT
+description "Redis"
+start on runlevel [23]
+stop on shutdown
+exec sudo -u $REDIS_USER /usr/local/bin/redis-server $REDIS_CONF_FILE
+respawn
+EOT
+}

data/ext/bash/lib/rvm.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# requires app.sh
+export RVM_CHANNEL="stable"
+RVM_REQUIRED_PACKAGES="curl gawk g++ gcc make libc6-dev libreadline6-dev zlib1g-dev libssl-dev libyaml-dev libsqlite3-dev sqlite3 autoconf libgdbm-dev libncurses5-dev automake libtool bison pkg-config libffi-dev"
+
+# $1 unix user
+# $2 ruby version
+function rvm-install-for-user {
+  section "RVM"
+  install-packages $RVM_REQUIRED_PACKAGES
+
+  announce "Install RVM for user $1"
+  su - $1 -c "curl -L https://get.rvm.io | bash -s $RVM_CHANNEL"
+  su - $1 -c "rvm autolibs read-fail"
+
+  announce "Install Ruby $2 for user $1"
+  su - $1 -c "rvm install $2"
+
+  announce "Set Ruby $2 as default for user $1"
+  su - $1 -c "rvm --default use $2"
+}

data/ext/bash/lib/system.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+export SYSTEM_SHMALL_PERCENT="0.75" # @specify
+export SYSTEM_SHMMAX_PERCENT="0.5" # @specify
+export SYSTEM_BASE_PACKAGES="curl vim-nox less htop build-essential openssl git tree python-software-properties"
+export SYSTEM_TIMEZONE="Etc/UTC" # @specify
+export SYSTEM_LOCALE="en_US.UTF-8" # @specify
+export SYSTEM_SOURCES_PG_KEY_URL="http://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc"
+
+function system-configure-shared-memory {
+  announce "Configuring shared memory"
+  install-packages bc
+
+  local shmall=$(get-kernel-shmall $SYSTEM_SHMALL_PERCENT)
+  local shmmax=$(get-kernel-shmmax $SYSTEM_SHMMAX_PERCENT)
+
+  sysctl -w kernel.shmall=$shmall
+  sysctl -w kernel.shmmax=$shmmax
+  tee -a /etc/sysctl.conf <<EOT
+
+kernel.shmall = $shmall
+kernel.shmmax = $shmmax
+EOT
+}
+
+function system-install-packages {
+  install-packages $SYSTEM_BASE_PACKAGES
+}
+
+function system-configure-timezone {
+  announce "Set timezone to $SYSTEM_TIMEZONE"
+  set-timezone $SYSTEM_TIMEZONE
+}
+
+function system-configure-locale {
+  announce "Set locale to $SYSTEM_LOCALE"
+  set-locale $SYSTEM_LOCALE
+}
+
+function system-upgrade {
+  announce "Update and upgrade system packages"
+  upgrade-system
+}
+
+function system-add-pg-source {
+  announce "Add PostgreSQL sources:"
+  tee /etc/apt/sources.list.d/pgdg.list <<EOT
+deb http://apt.postgresql.org/pub/repos/apt/ $(release-codename)-pgdg main
+EOT
+
+  announce-item "apt.postgresql.org"
+  add-source-key $SYSTEM_SOURCES_PG_KEY_URL
+  update-sources
+}
+
+function system-install-sources {
+  system-add-pg-source
+}

data/ext/bash/lib/ubuntu.sh

@@ -0,0 +1,175 @@
+#!/usr/bin/env bash
+function get-processor-count {
+  nproc
+}
+
+function release-codename {
+  lsb_release -c -s
+}
+
+# $@ package names
+function install-packages {
+  announce "Installing packages:"
+  for package in $@; do
+    announce-item "$package"
+  done
+  aptitude -q -y -o Dpkg::Options::="--force-confnew" install $@
+}
+
+function get-archtype {
+  if [ $(getconf LONG_BIT) == 32 ]; then
+    echo 'x86'
+  else
+    echo 'x64'
+  fi
+}
+
+function update-sources {
+  apt-get -qq -y update
+}
+
+function add-repository {
+  add-apt-repository -y $1
+}
+
+# $1 unix user
+# $2 service name
+# $3 service args
+function add-sudoers-entries {
+  for event in start status stop reload restart; do
+    if [ $3 ]; then
+      tee -a /etc/sudoers.d/$2.entries <<EOT
+$1 ALL=NOPASSWD: /sbin/$event $2 $3
+EOT
+    else
+      tee -a /etc/sudoers.d/$2.entries <<EOT
+$1 ALL=NOPASSWD: /sbin/$event $2
+EOT
+    fi
+  done
+}
+
+function reboot-system {
+  shutdown -r now
+}
+
+# $1 package name
+function reconfigure-package {
+  dpkg-reconfigure -f noninteractive $1
+}
+
+# $1 key URL
+function add-source-key {
+  wget --quiet -O - $1 | apt-key add -
+}
+
+# $@ files to extract
+function extract {
+  tar xzf $@
+}
+
+# $1 URL to download
+function download {
+  wget -nv $1
+}
+
+function get-ram-bytes {
+  free -m -b | awk '/^Mem:/{print $2}'
+}
+
+function get-page-size {
+  getconf PAGE_SIZE
+}
+
+function get-ram-pages {
+  echo "$(get-ram-bytes) / $(get-page-size)" | bc
+}
+
+# $1 shmall percent
+function get-kernel-shmall {
+  echo "($(get-ram-pages) * $1) / 1" | bc
+}
+
+# $1 shmmax percent
+function get-kernel-shmmax {
+  echo "($(get-ram-bytes) * $1) / 1" | bc
+}
+
+# $1 unix user
+# $2 path
+function as-user-mkdir {
+  mkdir -p $2
+  chown $1:$1 $2
+}
+
+function upgrade-system {
+  update-sources
+  apt-get -qq -y install aptitude
+  aptitude -q -y -o Dpkg::Options::="--force-confnew" full-upgrade
+}
+
+# $1 timezone
+function set-timezone {
+  echo $1 > /etc/timezone
+  reconfigure-package tzdata
+}
+
+# $1 locale eg: en_US.UTF-8
+function set-locale {
+  export LANGUAGE=$1
+  export LANG=$1
+  export LC_ALL=$1
+  locale-gen $1
+  reconfigure-package locales
+  update-locale
+}
+
+# $1 hostname
+function set-hostname {
+  echo $1 > /etc/hostname
+  hostname -F /etc/hostname
+}
+
+# $1 unix user
+# $2 unix group
+# $3 password
+function add-user {
+  if [ $2 ]; then
+    announce "Adding $1 user to group $2"
+    useradd -m -s /bin/bash -g $2 $1
+  else
+    announce "Adding $1 user"
+    useradd -m -s /bin/bash $1
+  fi
+
+  if [ $3 ]; then
+    announce "Setting password for $1 user"
+    echo "$1:$3" | chpasswd
+  fi
+}
+
+# $1 unix user
+# $2 github usernames for public keys
+function add-pubkeys-from-github {
+  announce "Installing public keys for $1 from GitHub users:"
+
+  local ssh_dir="/home/$1/.ssh"
+  local keys_file="$ssh_dir/authorized_keys"
+
+  mkdir -p $ssh_dir
+  touch $keys_file
+
+  for user in $2; do
+    announce-item "$user"
+    local url="https://github.com/$user.keys"
+    tee -a $keys_file <<EOT
+# $url
+$(wget -qO- $url)
+
+EOT
+  done
+
+  chmod 700 $ssh_dir
+  chmod 600 $keys_file
+  chown -R $1 $ssh_dir
+}

data/ext/bash/vagrant/nginx.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+function provision-vagrant-nginx {
+  section "Nginx"
+  nginx-install
+  nginx-configure
+  nginx-conf-add-gzip
+  nginx-conf-add-mimetypes
+
+  announce "Adding Nginx config for Vagrant"
+  tee "$NGINX_SITES_DIR/vagrant.conf" <<EOT
+upstream rack_app {
+  server 127.0.0.1:3000 fail_timeout=0;
+}
+
+server {
+  root $VAGRANT_DATA_DIR/public;
+
+  access_log /dev/null;
+  error_log /dev/null;
+
+  try_files \$uri/index.html \$uri.html \$uri @upstream_app;
+
+  location @upstream_app {
+    proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+    proxy_set_header Host \$http_host;
+    proxy_redirect off;
+    proxy_pass http://rack_app;
+  }
+}
+EOT
+}

data/ext/bash/vagrant/node.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+function provision-vagrant-node {
+  section "Node.js"
+  node-install
+}

data/ext/bash/vagrant/pg.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+function vagrant-pg-create-user {
+  announce "Create database user: $VAGRANT_USER"
+  su - $PG_USER -c "createuser -s $VAGRANT_USER"
+}
+
+function provision-vagrant-pg {
+  section "PostgreSQL Server"
+  pg-install-packages
+  pg-tune
+  vagrant-pg-create-user
+}

data/ext/bash/vagrant/redis.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+function provision-vagrant-redis {
+  section "Redis"
+  redis-install
+}

data/ext/bash/vagrant/rvm.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+export VAGRANT_RVM_RUBY_VERSION="2.0.0" # @specify
+
+function provision-vagrant-rvm {
+  rvm-install-for-user $VAGRANT_USER $VAGRANT_RVM_RUBY_VERSION
+}

data/ext/bash/vagrant/system.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+function vagrant-system-install-user-aliases {
+  announce "Installing helper aliases for user: $VAGRANT_USER"
+  tee -a /home/$VAGRANT_USER/.profile <<EOT
+alias be="bundle exec"
+alias r="bin/rails"
+alias v="cd $VAGRANT_DATA_DIR"
+cd $VAGRANT_DATA_DIR
+EOT
+}
+
+function vagrant-system-purge-grub-menu-config {
+  ucf --purge /boot/grub/menu.lst
+}
+
+function provision-vagrant-system {
+  section "Vagrant System"
+  vagrant-system-purge-grub-menu-config
+  system-upgrade
+  system-configure-timezone
+  system-configure-locale
+  system-install-packages
+  system-configure-shared-memory
+  system-install-sources
+  vagrant-system-install-user-aliases
+}

data/ext/bash/vagrant.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+export VAGRANT_USER="vagrant"
+export VAGRANT_DATA_DIR="/vagrant"

data/ext/bash/vps/system.sh

@@ -0,0 +1,156 @@
+#!/usr/bin/env bash
+export VPS_SYSTEM_HOSTNAME="" # @require
+export VPS_SYSTEM_FQDN="" # @require
+export VPS_SYSTEM_ADMIN_AUTHORIZED_GITHUB_USERS="" # @require
+export VPS_SYSTEM_ADMIN_SUDO_PASSWORD="" # @require
+export VPS_SYSTEM_PRIVATE_IP="" # @specify
+export VPS_SYSTEM_ADMIN_USER="admin" # @specify
+export VPS_SYSTEM_PRIVATE_NETMASK="255.255.128.0"
+export VPS_SYSTEM_ALLOW_PORTS="www 443 ssh"
+export VPS_SYSTEM_LIMIT_PORTS="ssh"
+export VPS_SYSTEM_ALLOW_PRIVATE_IPS="" # @specify
+export VPS_SYSTEM_ALLOW_PRIVATE_PORTS="5432 6379" # Postgres & Redis
+export VPS_SYSTEM_GET_PUBLIC_IP_SERVICE_URL="http://ipecho.net/plain"
+
+function get-vps-system-public-ip {
+  wget -qO- $VPS_SYSTEM_GET_PUBLIC_IP_SERVICE_URL
+}
+
+function get-vps-system-default-gateway {
+  ip route | awk '/default/ { print $3 }'
+}
+
+function vps-system-configure-hostname {
+  announce "Set hostname to $VPS_SYSTEM_HOSTNAME"
+  set-hostname $VPS_SYSTEM_HOSTNAME
+}
+
+function vps-system-configure-sshd {
+  announce "Configure sshd:"
+  announce-item "disable root login"
+  announce-item "disable password auth"
+  tee /etc/ssh/sshd_config <<EOT
+Port 22
+Protocol 2
+HostKey /etc/ssh/ssh_host_rsa_key
+HostKey /etc/ssh/ssh_host_dsa_key
+HostKey /etc/ssh/ssh_host_ecdsa_key
+UsePrivilegeSeparation yes
+KeyRegenerationInterval 3600
+ServerKeyBits 768
+SyslogFacility AUTH
+LogLevel INFO
+LoginGraceTime 120
+PermitRootLogin no
+StrictModes yes
+RSAAuthentication yes
+PubkeyAuthentication yes
+IgnoreRhosts yes
+RhostsRSAAuthentication no
+HostbasedAuthentication no
+PermitEmptyPasswords no
+ChallengeResponseAuthentication no
+PasswordAuthentication no
+X11Forwarding yes
+X11DisplayOffset 10
+PrintMotd no
+PrintLastLog yes
+TCPKeepAlive yes
+AcceptEnv LANG LC_*
+Subsystem sftp /usr/lib/openssh/sftp-server
+UsePAM yes
+EOT
+
+  announce "Restart sshd"
+  service ssh restart
+}
+
+function vps-system-configure-firewall {
+  section "Firewall"
+  install-packages ufw
+
+  announce "Configuring firewall:"
+  ufw default deny
+  ufw logging on
+
+  for port in $VPS_SYSTEM_ALLOW_PORTS; do
+    announce-item "allow $port"
+    ufw allow $port
+  done
+
+  for port in $VPS_SYSTEM_LIMIT_PORTS; do
+    announce-item "limit $port"
+    ufw limit $port
+  done
+
+  for local_ip in $VPS_SYSTEM_ALLOW_PRIVATE_IPS; do
+    for port in $VPS_SYSTEM_ALLOW_PRIVATE_PORTS; do
+      announce-item "allow $port from $local_ip"
+      ufw allow $port from $local_ip
+    done
+  done
+
+  echo 'y' | ufw enable
+}
+
+function vps-system-configure-admin-user {
+  announce "Adding admin user: $VPS_SYSTEM_ADMIN_USER"
+  add-user $VPS_SYSTEM_ADMIN_USER sudo $VPS_SYSTEM_ADMIN_SUDO_PASSWORD
+  add-pubkeys-from-github $VPS_SYSTEM_ADMIN_USER "$VPS_SYSTEM_ADMIN_AUTHORIZED_GITHUB_USERS"
+}
+
+function vps-system-configure-interfaces {
+  announce "Resolving extenal IP address"
+
+  local ip_addr=$(get-vps-system-public-ip)
+  local gateway=$(get-vps-system-default-gateway)
+  local fqdn="$ip_addr $VPS_SYSTEM_HOSTNAME $VPS_SYSTEM_FQDN"
+
+  announce "Setting FQDN: $fqdn"
+  echo "$fqdn" >> /etc/hosts
+
+  announce "Writing /etc/network/interfaces"
+  tee /etc/network/interfaces <<EOT
+auto lo
+iface lo inet loopback
+
+auto eth0 eth0:0 eth0:1
+
+# Public interface
+iface eth0 inet static
+ address $ip_addr
+ netmask 255.255.255.0
+ gateway $gateway
+EOT
+
+  if [ $VPS_SYSTEM_PRIVATE_IP ]; then
+    tee -a /etc/network/interfaces <<EOT
+
+# Private interface
+iface eth0:1 inet static
+ address $VPS_SYSTEM_PRIVATE_IP
+ netmask $VPS_SYSTEM_PRIVATE_NETMASK
+EOT
+  fi
+
+  announce "Restart networking"
+  /etc/init.d/networking restart
+
+  announce "Removing DHCP"
+  aptitude -q -y remove isc-dhcp-client dhcp3-client dhcpcd
+}
+
+function provision-vps-system {
+  section "VPS System"
+  system-upgrade
+  system-configure-timezone
+  vps-system-configure-hostname
+  system-configure-locale
+  system-install-packages
+  system-configure-shared-memory
+  system-install-sources
+  vps-system-configure-admin-user
+  vps-system-configure-interfaces
+  vps-system-configure-sshd
+  vps-system-configure-firewall
+}

data/lib/propro/cli/templates/init.tt

@@ -0,0 +1,21 @@
+<%= Propro.comment_banner %>
+#
+# Example provisioner for <%= @desc %>
+#
+
+<%- @paths.each do |path| -%>
+source :<%= path %>
+<%- end -%>
+
+<%- @sources.each do |source| -%>
+<%- has_no_exports = source.specified_exports.empty? -%>
+<%- next if has_no_exports && !source.can_provision? -%>
+# <%= source.name %>
+<%- source.specified_exports.each do |export| -%>
+<%= export.to_ruby %>
+<%- end -%>
+<%- if source.can_provision? -%>
+provision "<%= source.provisioner %>"
+<%- end -%>
+
+<%- end -%>