stork 0.1.0.pre → 0.2.0.pre

data/bin/storkctl

@@ -18,16 +18,28 @@ require 'ostruct'
 class StorkCtlApp
   def initialize
     @options = OpenStruct.new
-    @options.config = "/etc/stork/config.rb"
+    @options.config = ENV['STORK_CONFIG'] || "/etc/stork/config.rb"
+    @options.daemonize = false
+    @options.debug = false
     @options.command = :start
   end
 
   def parse
     opts = OptionParser.new do |o|
       o.banner = "Usage: storkctl start|stop|restart [options]"
-      o.on("-c", "--config FILE", "Read configuration from the specified file") do |config|
+      o.on("-c", "--config FILE", "Read configuration from the specified file.") do |config|
         @options.config = config
       end
+      o.on("-d", "--daemonize", "Daemonize stork.") do
+        @options.daemonize = true
+      end
+      o.on("-D", "--debug", "Turn on debug output") do
+        @options.debug = true
+      end
+      o.on("-h", "--help", "Print this help message") do
+        puts opts.help
+        exit 1
+      end
     end
 
     action = :"#{ARGV.shift}"
@@ -35,28 +47,46 @@ class StorkCtlApp
       puts opts.help
       exit 1
     end
+    
     @options.command = action
     opts.parse!
+    load_or_create_config
   end
 
-  def build
-    @configuration = Stork::Configuration.from_file(@options.config)
-    @collection = Stork::Builder.load(@configuration).collection
+  def load_or_create_config
+    unless File.exist?(@options.config)
+      Stork::Configuration.to_file(@options.config)
+    end
+    Stork::Configuration.from_file(@options.config)
   end
 
   def run
+    @control = Stork::Server::Control.new(@options)
+
     case @options.command
     when :start
-      Stork::Server::Control.start(@configuration, @collection)
+      if @options.daemonize
+        start_daemon
+      else
+        @control.start
+      end
     when :stop
-      Stork::Server::Control.stop(@configuration, @collection)
+      @control.stop
     when :restart
-      Stork::Server::Control.restart(@configuration, @collection)
+      @restart.restart
+    end
+  end
+
+  def start_daemon
+    if Process.respond_to?(:daemon)
+      Process.daemon(true)
+      @control.start
+    else
+      abort 'Process.daemon requires Ruby >= 1.9'
     end
   end
 end
 
 app = StorkCtlApp.new
 app.parse
-app.build
 app.run

data/box/Vagrantfile

@@ -0,0 +1,41 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+VAGRANTFILE_API_VERSION = "2"
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+  config.vm.box = "centos-6.5"
+  config.vm.box_url = "http://opscode-vm-bento.s3.amazonaws.com/vagrant/virtualbox/opscode_centos-6.5_chef-provisionerless.box"
+  config.vm.provision :shell, path: "bootstrap.sh"
+
+  # config.vm.network "forwarded_port", guest: 80, host: 8080
+  config.vm.network "private_network", ip: "10.10.1.10"
+
+  # config.vm.network "public_network"
+  # config.ssh.forward_agent = true
+
+  config.vm.synced_folder File.expand_path(File.dirname(__FILE__) + "/../"), "/mnt/stork"
+
+  config.vm.provider "virtualbox" do |vb|
+    # Don't boot with headless mode
+    vb.memory = 1024
+    # Use VBoxManage to customize the VM. For example to change memory:
+    vb.customize ["modifyvm", :id, "--memory", "1024"]
+    vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
+  end
+  
+  # Enable provisioning with chef solo, specifying a cookbooks path, roles
+  # path, and data_bags path (all relative to this Vagrantfile), and adding
+  # some recipes and/or roles.
+  #
+  # config.vm.provision "chef_solo" do |chef|
+  #   chef.cookbooks_path = "../my-recipes/cookbooks"
+  #   chef.roles_path = "../my-recipes/roles"
+  #   chef.data_bags_path = "../my-recipes/data_bags"
+  #   chef.add_recipe "mysql"
+  #   chef.add_role "web"
+  #
+  #   # You may also specify custom JSON attributes:
+  #   chef.json = { mysql_password: "foo" }
+  # end
+end

data/box/bootstrap.sh

@@ -0,0 +1,317 @@
+#!/bin/bash
+# bootstrap.sh
+# set -x
+###############################################################################
+# Install and start ntp
+###############################################################################
+yum -y install ntp
+ntpdate pool.ntp.org
+/etc/init.d/ntpd start && chkconfig ntpd on
+
+###############################################################################
+# Install tftp and configure it for PXE boot.
+###############################################################################
+yum -y install syslinux-tftpboot tftp-server
+cp /usr/share/syslinux/pxelinux.0 /var/lib/tftpboot/.
+cp /usr/share/syslinux/menu.c32 /var/lib/tftpboot/.
+mkdir /var/lib/tftpboot/pxelinux.cfg
+mkdir /var/lib/tftpboot/distros
+sed -i -re 's/(\s+disable\s+=) yes/\1 no/' /etc/xinetd.d/tftp
+service xinetd start && chkconfig xinetd on
+# Put the default in there.
+cat > /var/lib/tftpboot/pxelinux.cfg/default << 'EOF'
+DEFAULT local
+PROMPT 0
+TIMEOUT 0
+TOTALTIMEOUT 0
+ONTIMEOUT local
+LABEL local
+        LOCALBOOT -1
+EOF
+
+# Download a couple of netboot distros
+MIRROR5="http://mirror.centos.org/centos-5/5.10/os/x86_64/images/pxeboot"
+MIRROR6="http://mirror.centos.org/centos-6/6.5/os/x86_64/images/pxeboot"
+pushd /var/lib/tftpboot/distros
+echo "Downloading a few pxeboot kernels and images"
+curl -s $MIRROR5/initrd.img > centos-5.10-initrd.img
+curl -s $MIRROR5/vmlinuz > centos-5.10-vmlinuz
+curl -s $MIRROR6/initrd.img > centos-6.5-initrd.img
+curl -s $MIRROR6/vmlinuz > centos-6.5-vmlinuz
+popd
+
+###############################################################################
+# Install dhcp. and configure it.
+###############################################################################
+yum -y install dhcp
+cat > /etc/dhcp/dhcpd.conf << 'EOF'
+default-lease-time 600;
+max-lease-time 7200;
+authoritative;
+log-facility local7;
+
+subnet 10.10.1.0 netmask 255.255.255.0 {
+  range 10.10.1.120 10.10.1.200;
+  allow booting;
+  allow bootp;
+  next-server 10.10.1.10;
+  filename "pxelinux.0";
+  option domain-name "vbox.local";
+  option domain-name-servers 10.10.1.10;
+  option routers 10.10.1.10;
+  option broadcast-address 10.10.1.255;
+  host head.vbox.local {
+    option host-name "head";
+    option domain-name "vbox.local";
+    ddns-hostname "head";
+    ddns-domainname "vbox.local";
+    hardware ethernet 02:16:3f:c3:00:02;
+    fixed-address 10.10.1.101;
+  }
+  host node-101.vbox.local {
+    option host-name "node-101";
+    option domain-name "vbox.local";
+    ddns-hostname "node-101";
+    ddns-domainname "vbox.local";
+    hardware ethernet 02:16:3f:c3:01:01;
+    fixed-address 10.10.1.101;
+  }
+  host node-102.vbox.local {
+    option host-name "node-102";
+    option domain-name "vbox.local";
+    ddns-hostname "node-102";
+    ddns-domainname "vbox.local";
+    hardware ethernet 02:16:3f:c3:01:02;
+    fixed-address 10.10.1.102;
+  }
+  host node-103.vbox.local {
+    option host-name "node-103";
+    option domain-name "vbox.local";
+    ddns-hostname "node-103";
+    ddns-domainname "vbox.local";
+    hardware ethernet 02:16:3f:c3:01:03;
+    fixed-address 10.10.1.103;
+  }
+}
+EOF
+/etc/init.d/dhcpd start && chkconfig dhcpd on
+
+###############################################################################
+# Install DNS and configure it and make some network adjustments.
+###############################################################################
+yum -y install bind bind-utils
+
+cat > /etc/named.conf << 'EOF'
+options {
+    listen-on port 53 { 127.0.0.1; 10.10.1.10; };
+    directory "/var/named";
+    version "0.nunya";
+    forwarders { 10.0.2.3; };
+    allow-transfer { none; };
+    allow-query { localhost; };
+    statistics-file "data/named_stats.txt";
+    memstatistics-file "data/named_mem_stats.txt";
+    dnssec-enable no;
+    dnssec-validation no;
+};
+
+logging {
+    channel default_debug {
+        file "data/named.run";
+        severity dynamic;
+    };
+};
+
+zone "." IN {
+    type hint;
+    file "named.ca";
+};
+
+zone "vbox.local" IN {
+    type master;
+    file "vbox.local.zone";
+    allow-update { none; };
+};
+
+zone "1.10.10.in-addr.arpa" IN {
+    type master;
+    file "vbox.local.db";
+    allow-update { none; };
+};
+
+include "/etc/named.rfc1912.zones";
+include "/etc/named.root.key";
+EOF
+
+cat > /var/named/vbox.local.zone << 'EOF'
+$TTL    1D;
+$ORIGIN vbox.local.
+@               IN      SOA     @   root.vbox.local. (
+                        2014062501 ; serial
+                        3h ; refresh
+                        1w ; retry
+                        1w ; expire
+                        3h) ; min
+@           1D  IN  NS      stork
+stork       1D  IN  A       10.10.1.10
+chef        1D  IN  CNAME   stork
+head        1D  IN  A       10.10.1.100
+node-101    1D  IN  A       10.10.1.101
+node-102    1D  IN  A       10.10.1.102
+node-103    1D  IN  A       10.10.1.103
+EOF
+
+cat > /var/named/vbox.local.db << 'EOF'
+$TTL    1D;
+@       IN      SOA     stork.vbox.local.   root.vbox.local. (
+                        2014062501 ; serial
+                        3H ; refresh
+                        15 ; retry
+                        1w ; expire
+                        3h) ; min
+                NS      stork.vbox.local.
+10              PTR     stork.vbox.local.
+100             PTR     head.vbox.local.
+101             PTR     node-101.vbox.local.
+102             PTR     node-102.vbox.local.
+103             PTR     node-103.vbox.local.
+EOF
+
+/etc/init.d/named start && chkconfig named on
+
+###############################################################################
+# Create ssh keys
+###############################################################################
+pushd ~vagrant/.ssh
+ssh-keygen -t rsa -N "" -f id_rsa -N ""
+chown vagrant:vagrant id_rsa*
+popd
+
+###############################################################################
+# Chef setup with chef server and a few basic cookbooks for testing.
+###############################################################################
+pushd /tmp
+wget https://opscode-omnibus-packages.s3.amazonaws.com/el/6/x86_64/chef-server-11.0.12-1.el6.x86_64.rpm
+yum -y localinstall chef-server-11.0.12-1.el6.x86_64.rpm
+chef-server-ctl reconfigure
+wget https://opscode-omnibus-packages.s3.amazonaws.com/el/6/x86_64/chef-11.12.8-2.el6.x86_64.rpm
+yum -y localinstall chef-11.12.8-2.el6.x86_64.rpm
+popd
+
+# Just utilize the admin user that is already created
+pushd /root
+mkdir .chef
+cat > .chef/knife.rb << 'EOF'
+log_level                :info
+log_location             STDOUT
+node_name                'admin'
+client_key               '/etc/chef-server/admin.pem'
+validation_client_name   'chef-validator'
+validation_key           '/etc/chef-server/chef-validator.pem'
+chef_server_url          'https://stork.vbox.local:443'
+syntax_check_cache_path  '/root/.chef/syntax_check_cache'
+EOF
+popd
+
+###############################################################################
+# Set up EPEL and a few tools
+###############################################################################
+pushd /tmp
+wget http://download.fedoraproject.org/pub/epel/6/i386/epel-release-6-8.noarch.rpm
+yum -y localinstall epel-release-6-8.noarch.rpm
+yum -y install git sqlite-devel
+popd
+
+###############################################################################
+# RVM setup and Ruby 2.0.0 installation
+###############################################################################
+curl -sSL https://get.rvm.io | bash -s stable
+source /usr/local/rvm/scripts/rvm
+type rvm | head -1
+rvm install 2.0.0
+
+###############################################################################
+# Install and start stork
+###############################################################################
+pushd /mnt/stork
+rvm use 2.0.0@stork --create
+su vagrant -c 'rake build'
+gem install pkg/*.gem --no-rdoc --no-ri
+popd
+
+# Create the configuration
+mkdir /etc/stork
+pushd /etc/stork
+
+cat > /etc/stork/config.rb << 'EOF'
+# Stork configuration file"
+path                    "/etc/stork"
+bundle_path             "/etc/stork/bundles"
+authorized_keys_file    "/etc/stork/authorized_keys"
+pxe_path                "/var/lib/tftpboot/pxelinux.cfg"
+log_file                "/var/log/stork.log"
+pid_file                "/var/log/stork.pid"
+server                  "stork.vbox.local"
+port                    5000
+bind                    "0.0.0.0"
+timezone                "America/Los_Angeles"
+EOF
+
+# Create the authorized keys file
+cat /home/vagrant/.ssh/id_rsa.pub > /etc/stork/authorized_keys
+# Get the example bundles
+git clone https://github.com/rlyon/stork-bundle.git bundles
+
+cat > /usr/bin/stork_start << 'EOF'
+#!/bin/bash
+source /usr/local/rvm/scripts/rvm
+rvm use 2.0.0@stork
+storkctl start
+EOF
+chmod 755 /usr/bin/stork_start
+
+cat > /usr/bin/stork_update << 'EOF'
+#!/bin/bash
+source /usr/local/rvm/scripts/rvm
+rvm use 2.0.0@stork
+cd /mnt/stork
+rake install
+EOF
+chmod 755 /usr/bin/stork_update
+
+mkdir ~vagrant/.stork
+ln -s /etc/stork/config.rb ~vagrant/.stork/client.rb
+chown -R vagrant:vagrant ~vagrant/.stork
+
+# Make sure that RVM uses the system ruby as default.
+rvm reset
+
+
+
+###############################################################################
+# Networking adjustments
+###############################################################################
+echo "PEERDNS=no" >> /etc/sysconfig/network-scripts/ifcfg-eth0
+cat > /etc/resolv.conf << 'EOF'
+options single-request-reopen
+nameserver 127.0.0.1
+EOF
+
+/etc/init.d/network restart
+
+###############################################################################
+# Turn on NATing so we can use this as the gateway for our isolated test vms
+###############################################################################
+sed -i -re 's/(net.ipv4.ip_forward\s+=) 0/\1 1/' /etc/sysctl.conf
+sysctl -p /etc/sysctl.conf
+/etc/init.d/iptables start
+iptables -A FORWARD -i eth1 -j ACCEPT
+iptables -A FORWARD -o eth1 -j ACCEPT
+iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
+/etc/init.d/iptables save
+
+###############################################################################
+# Since we now have a TFTP, SysLinux, DHCP, DNS, Nating and Stork shared on a
+# mounted directory, we can set up a simple vbox config which will enable us to
+# go through the entire install process on a laptop. 
+###############################################################################

data/box/integration.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+# Make sure that virtualbox is not running dhcp on the host interface
+VBoxManage dhcpserver remove --netname HostInterfaceNetworking-vboxnet0
+
+ID=$(date "+%Y%M%H%m%d%S")
+MAC=$(echo $1 | sed 's/://g' | tr '[:lower:]' '[:upper:]')
+
+VBoxManage createvm --name stork_pxe_$ID --ostype RedHat_64 --register
+
+VBoxManage createhd --filename stork_pxe_$ID.vmdk --format VMDK --size 4000 --variant Standard
+VBoxManage storagectl stork_pxe_$ID --name "IDE Controller" --add ide --controller PIIX4 --hostiocache on --bootable on
+VBoxManage storageattach stork_pxe_$ID --storagectl "IDE Controller" --port 0 --device 0 --type hdd --medium stork_pxe_$ID.vmdk
+
+VBoxManage modifyvm stork_pxe_$ID --nic1 hostonly 
+VBoxManage modifyvm stork_pxe_$ID --cableconnected1 on 
+VBoxManage modifyvm stork_pxe_$ID --hostonlyadapter1 vboxnet0
+VBoxManage modifyvm stork_pxe_$ID --macaddress1 $MAC
+VBoxManage modifyvm stork_pxe_$ID --nictype1 82540EM
+VBoxManage modifyvm stork_pxe_$ID --intnet1 intnet
+
+VBoxManage modifyvm stork_pxe_$ID --boot1 net
+VBoxManage modifyvm stork_pxe_$ID --boot2 disk
+VBoxManage modifyvm stork_pxe_$ID --boot3 none
+VBoxManage modifyvm stork_pxe_$ID --boot4 none
+
+VBoxManage modifyvm stork_pxe_$ID --memory 1024
+
+VBoxManage modifyvm stork_pxe_$ID --acpi on
+VBoxManage modifyvm stork_pxe_$ID --ioapic on
+VBoxManage modifyvm stork_pxe_$ID --rtcuseutc on
+
+VBoxManage startvm stork_pxe_$ID

data/lib/stork/builder.rb

@@ -1,33 +1,36 @@
 module Stork
-  # The Builder object loads all of the configuration files and parses them
+  # The Builder object loads all of the definition files and parses them
   # to build a collection of resources
   #
-  # === Attributes
-  # * +configuration+ - The configuration containing the paths to the resource
-  #                     files.
   class Builder
     attr_reader :collection
-    attr_reader :configuration
-
-    def initialize(configuration)
-      @configuration = configuration
+    # Initialize the builder object.  Create a new collection.
+    #
+    def initialize
       @collection = Stork::Collections.new
     end
 
-    def self.load(configuration)
-      builder = Builder.new(configuration)
+    # Load the resource definition files.  The snippets and templates
+    # are first read in.  Then the rest of the resources will be evalutated
+    # and added to the collections.
+    #
+    # == Returns:
+    # A builder object that contains the the resource collections
+    #
+    def self.load
+      builder = Builder.new
       delegator = BuilderDelegator.new(builder)
 
       # Load in snippets and templates
-      delegator.snippets(configuration.snippets_path)
-      delegator.templates(configuration.templates_path)
+      delegator.snippets(Configuration[:snippets_path])
+      delegator.templates(Configuration[:templates_path])
 
       load_paths = [
-        configuration.distros_path,
-        configuration.chefs_path,
-        configuration.networks_path,
-        configuration.layouts_path,
-        configuration.hosts_path
+        Configuration[:distros_path],
+        Configuration[:chefs_path],
+        Configuration[:networks_path],
+        Configuration[:layouts_path],
+        Configuration[:hosts_path]
       ]
 
       load_paths.each do |path|
@@ -38,52 +41,96 @@ module Stork
       builder
     end
 
-    # Expose a limited number of methods for DSL parsing from files.  Allow
-    # delegation of the host, layout, network, chef, distro, templates and
-    # snippets blocks.
+    # Expose a limited number of resource methods for DSL parsing from 
+    # files.  Allow delegation of the host, layout, network, chef, distro, 
+    # templates and snippets blocks.
+    #
+    # == Parameters
+    # obj::
+    #   The Builder object that is being delegated.
     #
-    # ==== Attributes
-    # * +obj+ - The Builder object that is being delegated.
-    class BuilderDelegator
+    class BuilderDelegator 
       def initialize(obj)
         @delegated = obj
       end
 
+      # Build and add a host to the resource collection.
+      #
+      # == Parameters:
+      # name::
+      #   The fully qualified domain name of the host.
+      #
+      # 
       def host(name, &block)
         @delegated.collection.hosts.add(
           Resource::Host.build(
             name,
-            configuration: @delegated.configuration,
             collection: @delegated.collection,
             &block
           )
         )
       end
 
+
+      # Build and add a layout to the resource collection.
+      #
+      # == Parameters:
+      # name::
+      #   A unique name to identify the layout.
+      #
+      # 
       def layout(name, &block)
         @delegated.collection.layouts.add(
           Resource::Layout.build(name, &block)
         )
       end
 
+      # Build and add a network to the resource collection.
+      #
+      # == Parameters:
+      # name::
+      #   A unique name to identify the network.
+      #
+      # 
       def network(name, &block)
         @delegated.collection.networks.add(
           Resource::Network.build(name, &block)
         )
       end
 
+      # Build and add a chef server to the resource collection.
+      #
+      # == Parameters:
+      # name::
+      #   A unique name to identify the chef server.
+      #
+      #
       def chef(name, &block)
         @delegated.collection.chefs.add(
           Resource::Chef.build(name, &block)
         )
       end
 
+      # Build and add a distro to the resource collection.
+      #
+      # == Parameters:
+      # name::
+      #   A unique name to identify the distro.
+      #
+      #
       def distro(name, &block)
         @delegated.collection.distros.add(
           Resource::Distro.build(name, &block)
         )
       end
 
+      # Read and add a template to the resource collection.
+      #
+      # == Parameters:
+      # name::
+      #   A unique name to identify the template.
+      #
+      #
       def templates(path)
         Dir.glob(path + '/*.erb') do |file|
           @delegated.collection.templates.add(
@@ -92,6 +139,13 @@ module Stork
         end
       end
 
+      # Read and add a snippet to the resource collection.
+      #
+      # == Parameters:
+      # name::
+      #   A unique name to identify the snippet.
+      #
+      #
       def snippets(path)
         Dir.glob(path + '/*.erb') do |file|
           @delegated.collection.snippets.add(

data/lib/stork/client/plugins/host_actions.rb

@@ -0,0 +1,12 @@
+module HostActionsPlugin
+  class HostActions < Stork::Plugin
+    banner "stork host actions (options)"
+
+    def run
+      data = fetch('actions')
+      data['hosts'].each do |host|
+        print(host['name'], host['action'].upcase, pad: 32)
+      end
+    end
+  end
+end

data/lib/stork/client/plugins/host_list.rb

@@ -3,7 +3,7 @@ module HostListPlugin
     banner "stork host list (options)"
 
     def run
-      data = fetch('/hosts')
+      data = fetch('hosts')
       data['hosts'].sort.each do |host|
         puts host
       end

data/lib/stork/client/plugins/host_reload.rb

@@ -0,0 +1,16 @@
+module HostReloadPlugin
+  class HostReload < Stork::Plugin
+    banner "stork host reload (options)"
+
+    def run
+      response = RestClient.get "#{stork}/api/v1/reload"
+      if response.code == 200
+        puts "OK"
+      else
+        data = JSON.parse(response)
+        puts "Error: #{data['message']}"
+        exit 1
+      end
+    end
+  end
+end