kameleon-builder 2.1.0 → 2.1.1

data/lib/kameleon/utils.rb

@@ -1,7 +1,8 @@
 module Kameleon
   module Utils
 
-    def self.resolve_vars(raw, yaml_path, initial_variables)
+    def self.resolve_vars(raw, yaml_path, initial_variables, kwargs = {})
+      strict = kwargs.fetch('strict', true)
       raw.to_s.gsub(/\$\$\{[a-zA-Z0-9\-_]+\}|\$\$[a-zA-Z0-9\-_]+/) do |var|
         # remove the dollars
         if var.include? "{"
@@ -13,7 +14,9 @@ module Kameleon
         if initial_variables.has_key? strip_var
           value = initial_variables[strip_var]
         else
-          fail RecipeError, "#{yaml_path}: variable #{var} not found in local or global"
+          if strict
+            fail RecipeError, "#{yaml_path}: variable #{var} not found in local or global" \
+          end
         end
         return $` + resolve_vars(value.to_s + $', yaml_path, initial_variables)
       end

data/templates/debian7-g5k.yaml

@@ -29,8 +29,8 @@ global:
     - $$distrib
 
   # Frontend options
-  g5k_user: $$my_gk5_username  # CHANGE ME
-  g5k_site: $$my_gk5_site  # CHANGE ME
+  g5k_user: $$my_g5k_username  # CHANGE ME
+  g5k_site: $$my_g5k_site  # CHANGE ME
 
   # Kenv
   kenv: "wheezy-x64-base"

data/templates/debian7-kameleon.yaml

@@ -0,0 +1,43 @@
+#==============================================================================
+# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
+#==============================================================================
+#
+# DESCRIPTION: Debian 7 appliance with kameleon.
+#
+#==============================================================================
+
+---
+extend: debian7-desktop
+
+global:
+  # You can see the base template `debian7-desktop.yaml` to know the
+    # variables that you can override
+    arch: amd64
+
+bootstrap:
+  - @base
+
+setup:
+  - @base
+  - autologin
+  - install_software:
+    - packages: parted qemu-system debootstrap qemu-utils
+  - install_kameleon:
+    - deb_install:
+      - exec_in: wget http://kameleon.imag.fr/pkg/kameleon_2.1.0+20140612204940-1_amd64.deb -O kameleon_2.1.0_amd64.deb
+      - exec_in: dpkg -i kameleon_2.1.0_amd64.deb
+      - exec_in: rm -f kameleon_2.1.0_amd64.deb
+  - prepare_export:
+    - nullify_freespace:
+      - exec_in: rm -rf /tmp/* || true
+      - exec_in: rm -rf /var/tmp/* || true
+      - exec_in: |
+          echo "Nullify freespace..."
+          dd if=/dev/zero of=/bigemptyfile bs=1M 2>&1 >/dev/null || true
+          rm -f /bigemptyfile
+
+export:
+  - save_appliance:
+    - save_as_qcow2:
+      - exec_out: qemu-img convert -c -O qcow2 $$image_disk $KAMELEON_WORKDIR/$$kameleon_recipe_name.qcow2
+      - exec_out: echo "Saved qcow2 appliance to $KAMELEON_WORKDIR/$$kameleon_recipe_name.qcow2"

data/templates/debian7.yaml

@@ -96,7 +96,7 @@ setup:
     - dist_upgrade: true
   - install_software:
     - packages: >
-        debian-keyring ntp zip unzip rsync sudo less vim bash-completion
+        debian-keyring ntp rsync sudo less vim bash-completion
   - configure_kernel:
     - arch: $$arch
   # Configuration

data/templates/docker-debian7.yaml

@@ -0,0 +1,107 @@
+#==============================================================================
+# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
+#==============================================================================
+#
+# DESCRIPTION: Build a debian image for docker with docker
+#
+#==============================================================================
+
+---
+# Loads some helpful aliases
+aliases: defaults.yaml
+# Enables qcow2 checkpoint
+# checkpoint: docker.yaml
+#== Global variables use by Kameleon engine and the steps
+global:
+  ## User varibales : used by the recipe
+  user_name: kameleon
+  user_password: $$user_name
+
+  # Distribution
+  distrib: debian
+  release: wheezy
+  arch: amd64
+
+  ## Docker options
+  docker_dns: 208.67.222.222
+  docker_image: $$kameleon_recipe_name
+  docker_hostname: $$kameleon_recipe_name
+
+  # rootfs options
+  rootfs_download_path: /var/cache/kameleon/$$distrib/$$release/$$arch/rootfs
+
+  ## System variables. Required by kameleon engine
+  # Include specific steps
+  include_steps:
+    - $$distrib/$$release
+    - $$distrib
+
+  # Apt options
+  apt_repository: http://ftp.debian.org/debian/
+  apt_enable_contrib: true
+  apt_enable_nonfree: true
+  apt_install_recommends: false
+
+  # Shell session from where we launch exec_out commands. There is often a
+  # local bash session, but it can be a remote shell on other machines or on
+  # any shell. (eg. bash, chroot, fakechroot, ssh, tmux, lxc...)
+  out_context:
+    cmd: bash
+    workdir: $$kameleon_cwd
+
+  # Shell session that allows us to connect to the building machine in order to
+  # configure it and setup additional programs
+  ssh_config_file: $$kameleon_cwd/ssh_config
+  in_context:
+    cmd: LC_ALL=POSIX ssh -F $$ssh_config_file $$kameleon_recipe_name -t /bin/bash
+    workdir: /root/kameleon_workdir
+
+#== Bootstrap the new system and create the 'in_context'
+bootstrap:
+  - debootstrap:
+    - include_pkg: >
+        ifupdown locales libui-dialog-perl dialog isc-dhcp-client netbase
+        net-tools iproute acpid openssh-server pciutils extlinux
+        linux-image-$$arch
+    - release: $$release
+    - arch: $$arch
+    - repository: $$apt_repository
+    - enable_cache: true
+  - prepare_docker
+  - start_docker
+
+
+#== Install and configuration steps
+# WARNING: this part should be independante from the out context (whenever
+# possible...)
+setup:
+  # Install
+  - configure_apt:
+    - repository: $$apt_repository
+    - enable_contrib_repo: $$apt_enable_contrib
+    - enable_nonfree_repo: $$apt_enable_nonfree
+    - install_recommends: $$apt_install_recommends
+  - upgrade_system:
+    - dist_upgrade: true
+  - install_software:
+    - packages: >
+        debian-keyring ntp rsync sudo less vim bash-completion
+  # Configuration
+  - configure_system:
+    - locales: POSIX C en_US fr_FR de_DE
+    - lang: en_US.UTF-8
+    - timezone: UTC
+  - configure_keyboard:
+    - layout: "us,fr,de"
+  - create_group:
+    - name: admin
+  - create_user:
+    - name: $$user_name
+    - groups: sudo admin
+    - password: $$user_password
+
+#== Export the generated appliance in the format of your choice
+export:
+  - save_docker_appliance:
+    - commit:
+      - exec_out: docker commit $(cat MAIN_CONTAINER_ID) -t $$kameleon_recipe_name:latest

data/templates/steps/bootstrap/initialize_disk_chroot.yaml

@@ -8,7 +8,7 @@
   - exec_out: mkdir -p $$kameleon_cwd/checkpoints
   - exec_out: |
       if [ ! -e "$$image_disk" ] ; then
-        qemu-img create -f qcow2 -o preallocation=metadata $$kameleon_cwd/checkpoints/base.qcow2 $$image_size
+        qemu-img create -f qcow2 $$kameleon_cwd/checkpoints/base.qcow2 $$image_size
         # keep a link to the last checkpoint disk
         ln -sf $$kameleon_cwd/checkpoints/base.qcow2 $$image_disk
       fi

data/templates/steps/bootstrap/initialize_disk_qemu.yaml

@@ -7,7 +7,7 @@
   - check_cmd_out: qemu-img
   - exec_out: |
       rm -f $$image_disk
-      qemu-img create -f qcow2 -o preallocation=metadata $$image_disk $$image_size
+      qemu-img create -f qcow2 $$image_disk $$image_size
 
 - load_nbd_module:
   - on_checkpoint: redo

data/templates/steps/bootstrap/prepare_docker.yaml

@@ -1,13 +1,4 @@
-
-- check_docker:
-  - on_bootstrap_init:
-    - exec_out: DOCKER=${DOCKER:-docker}
-    - rescue:
-      - exec_out: command -V $DOCKER
-      - breakpoint: >
-          error: Docker executable no found. Make sure Docker
-          is installed and/or use the DOCKER variable to set Docker
-          executable.
+- insecure_ssh_key: $$kameleon_cwd/insecure_ssh_key
 
 - clean_containers:
   - on_checkpoint: redo
@@ -19,31 +10,42 @@
     - exec_out: cat CONTAINERS_TO_CLEAN | xargs -I {} docker rm {}
     - exec_out: rm -f CONTAINERS_TO_CLEAN
 
-- import_rootfs:
-  - exec_out: |
-      docker images | grep -q $$image \
-      || (echo "Importing $$image to docker..." && cat "$$rootfs_archive"\
-          | docker import - $$image \
-          | xargs -I {} docker tag {} $$image:base)
-  - exec_out: docker tag $$image:base $$image:latest
 
-- configure_sshd:
-  - on_checkpoint: redo
-  - exec_out: echo -e  'y\n' | ssh-keygen -q -t rsa -f $$insecure_ssh_key -N ''
+- configure_ssh_access:
+  - exec_out: echo -e  'y\n' | ssh-keygen -q -t dsa -f $$insecure_ssh_key -N ''
   - exec_out: cat $$insecure_ssh_key
+  - exec_out: chroot $$rootfs_download_path mkdir -p /root/.ssh
   - exec_out: |
-      CID="kameleon_container_$[($RANDOM % ($[50000 - 50] + 1)) + 50]"
-      cat $${insecure_ssh_key}.pub | \
-        docker run -i -a stdin --dns $$dns --name $CID $$image:latest \
-          /bin/bash -c "mkdir -p /root/.ssh ; \
-                        tee -a /root/.ssh/authorized_keys"
-  - on_export_init:
-    - exec_in: rm -rf /root/.ssh
-  - exec_out: echo "$CID" >> CONTAINERS_TO_CLEAN
-  - exec_out: |
-      docker images | grep $$image \
-                    | grep sshd \
-                    | awk '{print $3}' \
-                    | xargs -I {} docker rmi {}
-  - exec_out: docker commit $CID $$image:sshd > /dev/null
-  - exec_out: docker tag $$image:sshd $$image:latest
+        cat $${insecure_ssh_key}.pub > $$rootfs_download_path/root/.ssh/authorized_keys
+        cat $${insecure_ssh_key}.pub > $$rootfs_download_path/root/.ssh/kameleon_insecure_public_key
+        chmod 600 $$rootfs_download_path/root/.ssh/authorized_keys
+        chmod 755 $$rootfs_download_path/root/.ssh
+  - on_clean:
+    - exec_out: rm -rf $$rootfs_download_path/root/.ssh
+
+- import_to_docker:
+  - check_cmd_out: docker
+  - exec_out: echo "Importing $$docker_image to docker..."
+  - exec_out: tar -C $$rootfs_download_path -c . | $DOCKER import - $$docker_image:base
+
+- create_ssh_config:
+  - write_out:
+    - $$ssh_config_file
+    - |
+      Host $$kameleon_recipe_name
+      HostName 127.0.0.1
+      User root
+      IdentityFile $$insecure_ssh_key
+      UserKnownHostsFile /dev/null
+      StrictHostKeyChecking no
+      PasswordAuthentication no
+      IdentitiesOnly yes
+      LogLevel FATAL
+      ForwardAgent yes
+      ControlPath /tmp/$${kameleon_short_uuid}%r@%h:%p
+      ControlMaster auto
+      ControlPersist yes
+      Compression yes
+      Protocol 2
+  - on_export_clean:
+    - exec_out: rm -f /tmp/$${kameleon_short_uuid}*

data/templates/steps/bootstrap/prepare_qemu.yaml

@@ -40,7 +40,7 @@
       IdentitiesOnly yes
       LogLevel FATAL
       ForwardAgent yes
-      ControlPath $(pwd)/master-%l-%r@%h:%p
+      ControlPath /tmp/$${kameleon_short_uuid}%r@%h:%p
       ControlMaster auto
       ControlPersist yes
       Compression yes

data/templates/steps/bootstrap/start_docker.yaml

@@ -1,12 +1,16 @@
 - start_sshd:
   - on_checkpoint: redo
   - exec_out: |
-      CID=$(docker run -d -p 127.0.0.1::22 -i --dns $$dns -h $$hostname --privileged "$$image:latest" \
-            /bin/bash -c "service ssh restart ; tail -f /dev/null")
+      CID=$(docker run -d -p 127.0.0.1::22 -i \
+              --dns $$docker_dns \
+              -h $$docker_hostname \
+              --privileged "$$docker_image:base" \
+              /bin/bash -c "service ssh restart ; tail -f /dev/null")
   - exec_out: echo "$CID" >> CONTAINERS_TO_CLEAN
   - exec_out: echo $CID > MAIN_CONTAINER_ID
   - on_export_clean:
     - exec_out: rm -f MAIN_CONTAINER_ID
   - exec_out: echo $(docker port $CID 22) | cut -d':' -f2 > MAIN_CONTAINER_PORT
+  - exec_out: echo "Port $(cat MAIN_CONTAINER_PORT)" >> $$ssh_config_file
   - on_export_clean:
     - exec_out: rm -f MAIN_CONTAINER_PORT

data/templates/steps/bootstrap/start_qemu.yaml

@@ -47,7 +47,7 @@
     - exec_in: |
         sleep 2 && shutdown -h now &
     - exec_out: |
-        while nc -w 0 -z localhost $$qemu_monitor_port
+        while nc -w 1 -z localhost $$qemu_monitor_port
         do
           sleep 1
           echo -n "."
@@ -58,7 +58,7 @@
   - on_checkpoint: redo
   - on_export_clean:
     - exec_out: |
-        if nc -w 0 -z localhost $$qemu_monitor_port 2>/dev/null
+        if nc -w 1 -z localhost $$qemu_monitor_port 2>/dev/null
         then
           echo "Shutting down qemu virtual machine..."
           echo "system_reset" | nc localhost $$qemu_monitor_port 1>/dev/null 2>&1

data/templates/steps/checkpoints/qemu.yaml

@@ -1,6 +1,6 @@
 create: |
     if [ -f "$$kameleon_cwd/checkpoint_enabled" ] ; then
-      if nc -w 0 -z localhost $$qemu_monitor_port 2>/dev/null
+      if nc -w 1 -z localhost $$qemu_monitor_port 2>/dev/null
       then
         echo "savevm @microstep_id" | nc localhost $$qemu_monitor_port
       fi

data/templates/steps/export/save_appliance_from_g5k.yaml

@@ -5,7 +5,7 @@
   - exec_in: tgz-g5k $$filename.tar.gz
   - exec_out: |
       rsync -avz -e "ssh -F $$out_cwd/ssh_config" \
-        $$kameleon_recipe_name:/root/$$filename.tar.gz \
+        $$kameleon_recipe_name:$$in_cwd/$$filename.tar.gz \
         $$out_cwd/$$filename.tar.gz
   - exec_out: echo "Saved tar.gz appliance to $$out_cwd/$$filename.tar.gz"
   - exec_in: rm -f $$in_cwd/$$filename.tar.gz

data/version.txt

@@ -1 +1 @@
-2.1.0
+2.1.1

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kameleon-builder
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-12 00:00:00.000000000 Z
+date: 2014-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: childprocess
@@ -85,7 +85,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 3.0.4
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -93,7 +93,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 3.0.4
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -182,13 +182,15 @@ email:
 - bruno.bzeznik@imag.fr
 executables:
 - kameleon
-extensions: []
+extensions:
+- ext/mkrf_conf.rb
 extra_rdoc_files: []
 files:
 - .editorconfig
 - .env
 - .gitignore
 - AUTHORS
+- CHANGELOG.rst
 - COPYING
 - Gemfile
 - README.rst
@@ -233,6 +235,9 @@ files:
 - docs/source/commands.rst
 - docs/source/conf.py
 - docs/source/context.rst
+- docs/source/debian7.yaml
+- docs/source/debian_customized.yaml
+- docs/source/debian_customized_g5k.yaml
 - docs/source/faq.rst
 - docs/source/getting_started.rst
 - docs/source/grid5000_tutorial.rst
@@ -240,8 +245,10 @@ files:
 - docs/source/installation.rst
 - docs/source/persistent_cache.rst
 - docs/source/recipe.rst
+- docs/source/tau_install_g5k.yaml
 - docs/source/use_cases.rst
 - docs/source/workspace.rst
+- ext/mkrf_conf.rb
 - kameleon-builder.gemspec
 - lib/kameleon.rb
 - lib/kameleon/cli.rb
@@ -280,10 +287,11 @@ files:
 - templates/archlinux.yaml
 - templates/debian-testing.yaml
 - templates/debian7-desktop.yaml
-- templates/debian7-docker.yaml
 - templates/debian7-g5k.yaml
+- templates/debian7-kameleon.yaml
 - templates/debian7-oar-dev.yaml
 - templates/debian7.yaml
+- templates/docker-debian7.yaml
 - templates/extend.erb
 - templates/fedora-rawhide.yaml
 - templates/fedora20-desktop.yaml
@@ -380,7 +388,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
-      version: 1.9.3
+      version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: