deltacloud-core 0.3.0 → 0.4.0

data/support/condor/bash/cached_images.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+source /usr/libexec/condor/cloud_functions
+
+cd $CACHE
+echo "CACHED_IMAGES=\"$(echo * | tr ' ' '\n' | grep -v -e '*' -e .qcow2 | tr '\n' ',')\""
+
+exit 0

data/support/condor/bash/cloud_exit_hook.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+source /usr/libexec/condor/cloud_functions
+
+while read line; do
+   name="${line%% =*}"
+   value="${line#*= }"
+   case $name in
+     VM_XML ) VM_XML="$line" ;;
+   esac
+done
+
+DISK=$(echo $VM_XML | sed "s:.*<source file='\([^']*\)'/>.*:\1:")
+
+rm -f $DISK
+
+exit 0

data/support/condor/bash/cloud_functions

@@ -0,0 +1,175 @@
+source /etc/sysconfig/condor-cloud
+
+ERROR_MSG=
+
+MISSING_DEPS=
+which qemu-img > /dev/null 2>&1
+if [ $? -ne 0 ]; then
+   MISSING_DEPS="$MISSING_DEPS qemu-img"
+   return 1
+fi
+
+#
+# image_exists_global <image>
+#
+# Test to see if the given image is in the global image store. 0 is
+# returned if the image is present, 1 otherwise.
+#
+# This implementation uses the filesystem path $STORAGE as the global
+# store.
+#
+function image_exists_global {
+   IMAGE=$1
+   test -e $STORAGE/$IMAGE
+}
+
+#
+# image_exists_local <image>
+#
+# Test to see if the given image is in the local image store. 0 is
+# returned if the image is present, 1 otherwise.
+#
+# This implementation uses the filesystem path $CACHE as the local
+# store.
+#
+function image_exists_local {
+   IMAGE=$1
+   test -e $CACHE/$IMAGE
+}
+
+#
+# put_image <image>
+#
+# Put the image passed as the first argument, into the global image
+# store.  Return 0 on success and 1 on any failure.
+#
+# This implementation uses the filesystem path $STORAGE as the global
+# store.
+#
+function put_image {
+   IMAGE=$1
+   IMAGE_NAME=$(basename $IMAGE)
+
+   LOCK="$STORAGE/$IMAGE_NAME.lock"
+
+   CAT=cat; which pv > /dev/null 2>&1 && CAT=pv
+
+   while ! ( set -o noclobber; echo "$$" > $LOCK ) 2> /dev/null; do
+      echo "$LOCK held by $(cat $LOCK)" >2
+      sleep $((RANDOM % 29))
+   done
+
+   trap 'rm -f $LOCK; exit $?' INT TERM EXIT
+
+   RC=0
+   if ! image_exists_global $IMAGE_NAME; then
+      trap 'rm -f $LOCK; rm -f $STORAGE/$IMAGE_NAME; exit $?' INT TERM EXIT
+      ERROR_MSG=$($CAT $IMAGE > $STORAGE/$IMAGE_NAME)
+      if [ $? -eq 0 ]; then
+         # Permissions on the image are entirely open
+         chmod a=r $STORAGE/$IMAGE_NAME
+      else
+         RC=1
+      fi
+   fi
+
+   rm -f $LOCK; trap - INT TERM EXIT
+
+   return $RC
+}
+
+#
+# get_image <image>
+#
+# Get an image from the global image store and make a copy locally for
+# use by guest VMs. Return 0 on success and 1 on any failure.
+#
+# This implementation uses the filesystem path $STORAGE as the global
+# store and $CACHE as the local.
+#
+function get_image {
+   IMAGE=$1
+
+   LOCK="$CACHE/$IMAGE.lock"
+
+   if ! image_exists_global $IMAGE; then
+      ERROR_MSG="$STORAGE/$IMAGE not found"
+      return 1
+   fi
+
+   while ! ( set -o noclobber; echo "$$" > $LOCK ) 2> /dev/null; do
+      echo "$LOCK held by $(cat $LOCK)" >2
+      sleep $((RANDOM % 29))
+   done
+
+   trap 'rm -f $LOCK; exit $?' INT TERM EXIT
+
+   if ! image_exists_local $IMAGE; then
+      trap 'rm -f $LOCK; rm -f $CACHE/$IMAGE; exit $?' INT TERM EXIT
+      ERROR_MSG=$(cp $STORAGE/$IMAGE $CACHE/$IMAGE)
+      if [ $? -ne 0 ]; then
+         return 1
+      fi
+
+      # libvirt chowns the qcow2 image to qemu.qemu so that qemu can
+      # read/write it. The base image is not chowned, so we must make sure
+      # it is readable by qemu. If this is not done, a common VMGahpLog
+      # error will be:
+      #   Failed to create libvirt domain: monitor socket did not show up.:
+      #    No such file or directory
+      # Other than readable, no one should ever write to the file, so write
+      # perms are removed.
+      chmod a+r $CACHE/$IMAGE
+   fi
+
+   rm -f $LOCK
+   trap - INT TERM EXIT
+
+   return 0
+}
+
+#
+# make_image <image>
+#
+# Create a qcow2 image based off the given image and place it in the
+# local image store. The qcow2 image is echo'd to stdout. The return
+# value has no meaning.
+#
+# This implementation uses the filesystem path $CACHE as the local
+# store.
+#
+function make_image {
+   BASE_IMAGE=$1
+   LOCATION=$2
+
+   LOCK="$CACHE/$BASE_IMAGE.lock"
+
+   if ! image_exists_local $BASE_IMAGE; then
+      ERROR_MSG="$BASE_IMAGE not found"
+      return 1
+   fi
+
+   #
+   # Use the $LOCK as a barrier in case the image is concurrently
+   # being added to the $CACHE. Granted, that is a case that should
+   # never happen. Unfortunately, the existence of the $LOCK cannot be
+   # used as an assertion condition because a concurrent get_image
+   # will briefly create $LOCK even when the $BASE_IMAGE is already in
+   # the $CACHE.
+   #
+   while ! ( set -o noclobber; echo "$$" > $LOCK ) 2> /dev/null; do
+      echo "$LOCK held by $(cat $LOCK)" >2
+      sleep $((RANDOM % 29))
+   done
+   trap 'rm -f $LOCK; exit $?' INT TERM EXIT
+
+   IMAGE=$LOCATION/$BASE_IMAGE.qcow2
+
+   qemu-img create -f qcow2 -b $CACHE/$BASE_IMAGE $IMAGE > /dev/null 2>&1
+   # XXX: test for success, give return value meaning
+
+   rm -f $LOCK
+
+   echo $IMAGE
+   return 0
+}

data/support/condor/bash/cloud_prepare_hook.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+source /usr/libexec/condor/cloud_functions
+
+while read line; do
+   name="${line%% =*}"
+   value="${line#*= }"
+   case $name in
+     cloud_image ) BASE_IMAGE="$(echo $value | tr -d '\"')" ;;
+     VM_XML ) VM_XML="$line" ;;
+   esac
+done
+
+get_image $BASE_IMAGE
+
+IMAGE=$(make_image $BASE_IMAGE $PWD)
+
+echo $(echo $VM_XML | sed "s:{DISK}:$IMAGE:")
+
+exit 0

data/support/condor/bash/libvirt_cloud_script.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+while read line; do
+   line=$(echo "$line" | tr -d '"')
+   name="${line%% =*}"
+   value="${line#*= }"
+   case $name in
+     VMPARAM_VM_NAME ) NAME="$value" ;;
+     VM_XML ) VM_XML="$value" ;;
+   esac
+done
+
+echo $(echo $VM_XML | sed "s:{NAME}:$NAME:")

data/support/condor/config/50condor_cloud.config

@@ -0,0 +1,37 @@
+DAEMON_LIST = MASTER, SCHEDD, COLLECTOR, NEGOTIATOR, STARTD
+
+GRIDMANAGER_JOB_PROBE_INTERVAL = 30
+
+FILESYSTEM_DOMAIN = condor_cloud
+
+VM_GAHP_DEBUG = D_FULLDEBUG
+
+VM_GAHP_SEND_ALL_CLASSAD = TRUE
+VM_GAHP_LOG = $(LOG)/VMGahpLog
+VM_GAHP_SERVER = $(SBIN)/condor_vm-gahp
+VM_TYPE = kvm
+VM_MEMORY = $(MEMORY)
+VM_NETWORKING = TRUE
+VM_NETWORKING_TYPE = bridge,nat
+VM_NETWORKING_BRIDGE_INTERFACE = vnet0
+
+LIBVIRT_XML_SCRIPT = $(LIBEXEC)/libvirt_cloud_script.sh
+
+
+# Hook must be go=rx
+CLOUD_HOOK_PREPARE_JOB = $(LIBEXEC)/cloud_prepare_hook.sh
+CLOUD_HOOK_JOB_EXIT = $(LIBEXEC)/cloud_exit_hook.sh
+
+# Advertise set of images available on this node
+STARTD_CRON_JOBLIST = CACHED_IMAGES
+STARTD_CRON_CACHED_IMAGES_EXECUTABLE = $(LIBEXEC)/cached_images.sh
+STARTD_CRON_CACHED_IMAGES_PERIOD = 300s
+
+START = true
+
+#SLOT_TYPE_1 = cpus=100%,disk=100%,swap=100%
+#SLOT_TYPE_1_PARTITIONABLE = TRUE
+#NUM_SLOTS = 1
+#NUM_SLOTS_TYPE_1 = 1
+NUM_CPUS = $(DETECTED_CORES) * 4
+MEMORY = $(DETECTED_MEMORY) * 4

data/support/condor/config/50condor_cloud_node.config

@@ -0,0 +1,37 @@
+DAEMON_LIST = MASTER, STARTD
+
+GRIDMANAGER_JOB_PROBE_INTERVAL = 30
+
+FILESYSTEM_DOMAIN = condor_cloud
+
+VM_GAHP_DEBUG = D_FULLDEBUG
+
+VM_GAHP_SEND_ALL_CLASSAD = TRUE
+VM_GAHP_LOG = $(LOG)/VMGahpLog
+VM_GAHP_SERVER = $(SBIN)/condor_vm-gahp
+VM_TYPE = kvm
+VM_MEMORY = $(MEMORY)
+VM_NETWORKING = TRUE
+VM_NETWORKING_TYPE = bridge,nat
+VM_NETWORKING_BRIDGE_INTERFACE = vnet0
+
+LIBVIRT_XML_SCRIPT = $(LIBEXEC)/libvirt_cloud_script.sh
+
+
+# Hook must be go=rx
+CLOUD_HOOK_PREPARE_JOB = $(LIBEXEC)/cloud_prepare_hook.sh
+CLOUD_HOOK_JOB_EXIT = $(LIBEXEC)/cloud_exit_hook.sh
+
+# Advertise set of images available on this node
+STARTD_CRON_JOBLIST = CACHED_IMAGES
+STARTD_CRON_CACHED_IMAGES_EXECUTABLE = $(LIBEXEC)/cached_images.sh
+STARTD_CRON_CACHED_IMAGES_PERIOD = 300s
+
+START = true
+
+#SLOT_TYPE_1 = cpus=100%,disk=100%,swap=100%
+#SLOT_TYPE_1_PARTITIONABLE = TRUE
+#NUM_SLOTS = 1
+#NUM_SLOTS_TYPE_1 = 1
+NUM_CPUS = $(DETECTED_CORES) * 4
+MEMORY = $(DETECTED_MEMORY) * 4

data/support/condor/config/condor-cloud

@@ -0,0 +1,2 @@
+STORAGE=/var/lib/condor-cloud/shared_images
+CACHE=/var/lib/condor-cloud/local_cache

data/support/condor/config/condor_config.local

@@ -0,0 +1,44 @@
+ALLOW_WRITE = *
+ALLOW_ADMINISTRATOR = *
+ALLOW_NEGOTIATOR = *
+ALLOW_NEGOTIATOR_SCHEDD = *
+COLLECTOR_HOST = localhost
+
+DAEMON_LIST = MASTER, SCHEDD, COLLECTOR, NEGOTIATOR, STARTD
+
+GRIDMANAGER_JOB_PROBE_INTERVAL = 30
+
+GRIDMANAGER_DEBUG = D_FULLDEBUG
+NEGOTIATOR_DEBUG = D_FULLDEBUG
+COLLECTOR_DEBUG = D_FULLDEBUG
+VM_GAHP_DEBUG = D_FULLDEBUG
+
+VM_GAHP_SEND_ALL_CLASSAD = TRUE
+VM_GAHP_LOG = $(LOG)/VMGahpLog
+VM_GAHP_SERVER = $(SBIN)/condor_vm-gahp
+VM_TYPE = kvm
+VM_MEMORY = $(MEMORY)
+VM_NETWORKING = TRUE
+VM_NETWORKING_TYPE = bridge,nat
+VM_NETWORKING_BRIDGE_INTERFACE = vnet0
+
+LIBVIRT_XML_SCRIPT = $(LIBEXEC)/libvirt_cloud_script.sh
+
+
+# Hook must be go=rx
+CLOUD_HOOK_PREPARE_JOB = $(LIBEXEC)/cloud_prepare_hook.sh
+CLOUD_HOOK_JOB_EXIT = $(LIBEXEC)/cloud_exit_hook.sh
+
+# Advertise set of images available on this node
+STARTD_CRON_JOBLIST = CACHED_IMAGES
+STARTD_CRON_CACHED_IMAGES_EXECUTABLE = $(LIBEXEC)/cached_images.sh
+STARTD_CRON_CACHED_IMAGES_PERIOD = 300s
+
+START = true
+
+#SLOT_TYPE_1 = cpus=100%,disk=100%,swap=100%
+#SLOT_TYPE_1_PARTITIONABLE = TRUE
+#NUM_SLOTS = 1
+#NUM_SLOTS_TYPE_1 = 1
+#NUM_CPUS = $(DETECTED_CORES) * 4
+MEMORY = $(DETECTED_MEMORY) * 4

data/support/fedora/deltacloud-core

@@ -1,52 +1,74 @@
-#!/bin/bash
+#!/bin/sh
 #
-#
-# deltacloud-core       startup script for deltacloud-core server
-#
-# chkconfig: - 97 03
+# deltacloud-core   Deltacloud API Core
+# chkconfig: 345 90 60
 # description: deltacloud-core is primary server process for the \
-#    Deltacloud Core component.
-#
+#             Deltacloud Core component.
+
+### BEGIN INIT INFO
+# Provides: deltacloud-core
+# Required-Start: 
+# Required-Stop: 
+# Default-Start: 3 5 
+# Default-Stop: 0 1 2 6 
+# Short-Description: Deltacloud Core API deamon
+# Description: Deltacloud Core API provides access to different cloud providers \
+#   using single REST API 
+### END INIT INFO
+
+. /etc/rc.d/init.d/functions
 
 [ -r /etc/sysconfig/deltacloud-core ] && . /etc/sysconfig/deltacloud-core
 
-ENV="${ENV:-production}"
-DRIVER="${DRIVER:-ec2}"
-PORT="${PORT:-3002}"
-LOCKFILE="${LOCKFILE:-/var/lock/subsys/deltacloud-core }"
-LOGFILE="${LOGFILE:-/var/log/deltacloud-core/$DRIVER.log}"
+exec="/usr/bin/deltacloudd"
+prog="deltacloud-core"
 
-[ ! -d /var/log/deltacloud-core ] && mkdir -p /var/log/deltacloud-core
+[ ! -d /var/log/$prog ] && mkdir -p /var/log/$prog
 
-PROG=/usr/bin/deltacloudd
+# You can overide these variables using /etc/sysconfig/deltacloud-core
+#
+# ENV=production|development
+# DRIVER=ec2|mock|gogrid
+#
+#
+API_ENV="${ENV:-production}"
+DRIVER="${DRIVER:-ec2}"
+PORT="${PORT:-3002}"
+LOGFILE="${LOGFILE:-/var/log/$prog/$DRIVER.log}"
+lockfile="${LOCKFILE:-/var/lock/subsys/$prog }"
 
-. /etc/init.d/functions
+[ -r $LOGFILE ] && chown nobody $LOGFILE
+[ -r $lockfile ] && chown nobody $lockfile
 
 start() {
-    echo -n "Starting deltacloud-core: "
+    [ -x $exec ] || exit 5
+
+    echo -n $"Starting $prog: "
+    daemon --user nobody $exec -i $DRIVER -e $API_ENV --port $PORT >> $LOGFILE 2>&1 &
+    retval=$?
 
-    $PROG -i $DRIVER -e $ENV -p $PORT >> $LOGFILE 2>&1 &
-    RETVAL=$?
-    if [ $RETVAL -eq 0 ] && touch $LOCKFILE ; then
+    if [ $retval -eq 0 ] && touch $lockfile ; then
       echo_success
       echo
     else
       echo_failure
       echo
     fi
+    return $retval
 }
 
 stop() {
-    echo -n "Shutting down deltacloud-core: "
-    RETVAL=$?
-    killall deltacloudd
-    if [ $RETVAL -eq 0 ] && rm -f $LOCKFILE ; then
+    echo -n $"Shutting down $prog: "
+    killproc deltacloudd
+    retval=$?
+    if [ $retval -eq 0 ] && rm -f $lockfile ; then
       echo_success
       echo
     else
       echo_failure
       echo
     fi
+    return $retval
 }
 
 case "$1" in
@@ -67,12 +89,12 @@ case "$1" in
       ;;
     status)
       status $PROG
-      RETVAL=$?
+      retval=$?
       ;;
     *)
-      echo "Usage: deltacloud-core {start|stop|restart|status}"
+      echo "Usage: $prog {start|stop|restart|status}"
       exit 1
   ;;
 esac
 
-exit $RETVAL
+exit $?