kameleon-builder 2.0.0.dev

data/docs/source/context.rst

@@ -0,0 +1,42 @@
+-------
+Context
+-------
+
+To understand how Kameleon work you have to get the *context* notion. A context
+is an execution environnement with his variables (like $PATH, $TERM,...), his
+tools (debootstrap, yum, ...) and all his specifics (filesystem, local/remote,
+...). When you build an appliance you deal with 3 contexts:
+
+- The *local* context which is the Kameleon execution environnement
+- The *out* context where you will bootstrap the appliance
+- The *in* context which is inside the newly created appliance
+
+These context are setup using the two globals variables: ``out_context``
+and ``in_context``. They both takes 3 arguments:
+
+cmd
+    The command to initialize the context
+workdir (optional)
+    The working directory to tell to Kameleon where to execute the command
+exec_prefix (optional)
+    The command to execute before every Kameleon command in this context
+
+For example, you are building an appliance on your laptop and you run Kameleon
+in a bash shell with this configuration:
+
+.. code-block:: yaml
+
+    out_context:
+        cmd: bash
+        workdir: $$kameleon_cwd
+
+    in_context:
+        cmd: chroot $$rootfs bash
+        workdir: /
+
+
+Your *local* context is this shell where you launch Kameleon on your laptop,
+the *out* is a child bash of this context, and the *in* is inside the new
+environnement accessed by the chroot. As you can see the local and the out
+context are often very similar but sometimes it could be useful for the out
+context to be elsewhere (in a VM for example).

data/docs/source/faq.rst

@@ -0,0 +1,3 @@
+---
+FAQ
+---

data/docs/source/getting_started.rst

@@ -0,0 +1,3 @@
+---------------
+Getting Started
+---------------

data/docs/source/index.rst

@@ -0,0 +1,38 @@
+.. Kameleon documentation master file, created by
+   sphinx-quickstart on Thu Feb 13 19:01:14 2014.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to Kameleon's documentation!
+====================================
+
+Kameleon should be seen as a simple but powerful tool to generate customized
+appliances. With Kameleon, you make your recipe that describes how to create
+step by step your own distribution. At start Kameleon is used to create custom
+kvm, LXC, VirtualBox, iso images, ..., but as it is designed to be very generic
+you can probably do a lot more than that.
+
+
+.. toctree::
+    :maxdepth: 2
+
+    why.rst
+    installation.rst
+    getting_started.rst
+    cli.rst
+    recipe.rst
+    context.rst
+    commands.rst
+    workspace.rst
+    checkpoint.rst
+    aliases.rst
+    faq.rst
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+

data/docs/source/installation.rst

@@ -0,0 +1,3 @@
+------------
+Installation
+------------

data/docs/source/recipe.rst

@@ -0,0 +1,256 @@
+------
+Recipe
+------
+
+Kameleon compute YAML files, named  *recipes*, that describes how you will
+build your appliance. These files are stored in the root of your `Workspace`_.
+A recipe is a hierarchical structure of `Sections`_, `Steps`_, `Microsteps`_ and
+`Commands`_. Here is an overview of this structure:
+
+.. code-block:: yaml
+
+    recipe
+    └── section
+        └── step
+            └── microstep
+                └──command
+
+The recipe also contains set of `Global variables`_ declaration and some
+imports like `Aliases`_ and `Checkpoint`_.
+
+This is an example of a recipe:
+
+.. code-block:: yaml
+
+    ---
+    # Loads some helpful aliases
+    aliases: defaults.yaml
+    # Enables qcow2 checkpoint
+    checkpoint: qcow2.yaml
+    #== Global variables use by Kameleon engine and the steps
+    global:
+      ## User varibales : used by the recipe
+      cachedir: /var/cache/kameleon
+      rootfs: $$kameleon_cwd/rootfs
+      user_name: kameleon
+      arch: amd64
+    
+      nbd_device: /dev/nbd10
+      container: "$${kameleon_recipe_name}_temp.qcow2"
+    
+      distrib: debian
+      release: wheezy
+    
+      ## System variables. Required by kameleon engine
+      # Include specific steps
+      include_steps: [$$distrib/$$release, $$distrib]
+    
+      # Shell session from where we launch exec_out commands. There is often a
+      # local bash session, but it can be a remote shell on other machines or on
+      # any shell. (eg. bash, chroot, fakechroot, ssh, tmux, lxc...)
+      out_context:
+        cmd: bash
+        workdir: $$kameleon_cwd
+    
+      # Shell session that allows us to connect to the building machine in order to
+      # configure it and setup additional programs
+      default_env: "USER=root HOME=/root LC_ALL=POSIX"
+      in_context:
+        cmd: $$default_env chroot $$rootfs bash
+        workdir: /
+    
+    #== Bootstrap the new system and create the 'in_context'
+    bootstrap:
+      - debootstrap:
+        - release: $$release
+        - arch: $$arch
+        - repository: http://ftp.fr.debian.org/debian/
+        - rootfs_archive: $$cachedir/$$distrib/$$release/$$arch/debootstrap.tar.gz
+      - prepare_appliance_with_nbd:
+        - mountdir: $$rootfs
+        - image_size: 2G
+        - filesystem_type: ext4
+        - rootfs_archive: $$cachedir/$$distrib/$$release/$$arch/debootstrap.tar.gz
+      - start_chroot:
+        - rootfs: $$rootfs
+    
+    
+    #== Install and configuration steps
+    # WARNING: this part should be independante from the build context (whenever
+    # possible...)
+    setup:
+      # Install
+      - software_install:
+        - packages: >
+            debian-keyring ntp zip unzip rsync sudo less vim bash-completion
+      - kernel_install:
+        - arch: $$arch
+      # Configuration
+      - system_config:
+        - locales: fr_FR en_US
+        - lang: fr_FR.UTF-8
+        - timezone: Europe/Paris
+      - keyboard_config:
+        - layout: "fr,us"
+      - network_config:
+        - hostname: kameleon-$$distrib
+      - create_user:
+        - name: $$user_name
+        - group: admin
+        - password: $$user_name
+        
+    #== Export the generated appliance in the format of your choice
+    export:
+      - save_appliance_from_nbd:
+        - filename: "$${kameleon_recipe_name}"
+        - save_as_qcow2
+        # - save_as_tgz
+        # - save_as_raw
+        # - save_as_vmdk
+        # - save_as_vdi
+
+Section
+-------
+
+Each section is a list of `Steps`_. Currently, there is 3 sections:
+
+bootstrap
+    This section contains the bootstrap of the new system and create the *in*
+    context (see `Contexts`_).
+
+setup
+    This one is dedicated to the install and configuration steps.
+
+export
+    Export the generated appliance in the format of your choice.
+
+
+Step
+----
+
+Each *step* contains a list of microsteps that contains a list of Commands_
+written in one YAML file.  To be found by Kameleon this file must be named by
+with the step name plus the YAML extension ``.yaml``. For example the
+``software_install.yaml`` step file looks like this:
+
+.. code-block:: yaml
+
+    # Software Install
+    - add_contribs_source:
+      - exec_in: perl -pi -e "s/main$/main contrib non-free/" /etc/apt/sources.list
+    - update_repositories:
+      - exec_in: apt-get -y --force-yes update
+    - upgrade_system:
+      - exec_in: apt-get -y --force-yes dist-upgrade
+    - clean:
+      - on_export_init:
+        - exec_in: apt-get -y --force-yes autoclean
+        - exec_in: apt-get -y --force-yes clean
+        - exec_in: apt-get -y --force-yes autoremove
+    # default packages
+    - packages: "ntp sudo"
+    - extra_packages:
+      - exec_in: apt-get -y --force-yes install $$packages
+
+
+A step will be called like a function in the recipe. You should provide a set
+of local variables if needed by the step or to override default variables (see
+Variables_). Optionally, you can select only some microsteps to execute. Here
+is an example of step call:
+
+.. code-block:: yaml
+
+    - software_install:
+        - update_repositories
+        - add_contribs_source
+        - clean
+        - extra_packages
+        - packages: "debian-keyring ntp zip unzip rsync sudo"
+
+Steps path
+~~~~~~~~~~
+
+The steps are YAML formated files stored in the ``recipe/steps`` directory of
+the Workspace_. To enable a better recipe reuse and ease of write the steps
+are stored by default in specific folders depending on the sections.
+
+Kameleon is looking for the steps files using the ``include_steps`` list value,
+if it is set in the recipe (NOT mandatory). These includes are often the
+distribution steps. For example if you are building an ubuntu based
+distribution you can use:
+
+.. code-block:: yaml
+
+    include_steps:
+        - ubuntu
+        - debian/wheezy
+        - debian
+
+It also search uppermost within the current section folder. For the previous
+example, in the bootstrap section, the search paths are scanned in this
+order:
+
+.. code-block:: yaml
+
+    steps/bootstrap/ubuntu
+    steps/ubuntu
+    steps/bootstrap/debian/wheezy
+    steps/debian/wheezy
+    steps/bootstrap/debian
+    steps/debian
+    steps/bootstrap/
+    steps/
+
+
+Variables
+---------
+
+Kameleon is using preprocessed variables. You can define it with the YAML
+key/value syntax ``my_var: my_value``.To access these variables you have to use
+the two dollars (``$$``) prefix.  Like in a Shell you can also use
+``$${var_name}`` to include your variables in string like this
+``my-$${variable_name}-templated``. It's also possible to use nested variables
+like:
+
+.. code-block:: yaml
+
+    my_var: foo
+    my_nested_var: $${my_var}-bar
+
+Be careful, in YAML you cannot mix dictionary and list on the same level.
+That's why, in the global dictionary, you can define your variables like in the
+example above but, in the recipe or the steps, you must prefix your variable
+with a ``-`` like this ``- my_var: foo``.
+
+
+Global variables
+~~~~~~~~~~~~~~~~~
+
+Global variables are defined in the ``global`` dictionary of the recipe.
+Kameleon use some global variable to enable the appliance build. See Context_
+and `Steps path`_ for more details
+
+
+Step local variables
+~~~~~~~~~~~~~~~~~~~~
+
+In the recipe, you can provide some variables when you call a step. This
+variable override the global and the default variables.
+
+
+Step default variables
+~~~~~~~~~~~~~~~~~~~~~~
+
+In the step file, you can define some default variables for your microsteps. Be
+careful, to avoid some mistakes, these variables can be override by the step
+local variables but not by the global ones. If this is the behavior you
+expected just add a step local variable that take the global variable value
+like this:
+
+.. code-block:: yaml
+
+    global:
+        foo: bar
+    setup:
+        - my_step:
+            - foo: $$foo

data/docs/source/why.rst

@@ -0,0 +1,3 @@
+--------------
+Why Kameleon ?
+--------------

data/docs/source/workspace.rst

@@ -0,0 +1,11 @@
+---------
+Workspace
+---------
+
+The workspaces are the folders containing your Kameleon recipes and builds. When
+you use ``kameleon new`` a workspace is created if it does not exists. A
+workspace may contains several recipes.
+
+*Be careful*: All the *steps are shared between recipes within a workspace*. So
+if you do NOT want to share steps between different recipes you MUST use
+different workspace.

data/kameleon-builder.gemspec

@@ -0,0 +1,37 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'kameleon/version'
+
+Gem::Specification.new do |s|
+  s.name          = "kameleon-builder"
+  s.version       = Kameleon::VERSION
+  s.authors       = ["Salem Harrache",
+                     "Michael Mercier",
+                     "Cristan Ruiz",
+                     "Bruno Bzeznik"]
+  s.email         = ["salem.harrache@inria.fr",
+                     "michael.mercier@inria.fr",
+                     "cristian.ruiz@imag.fr",
+                     "bruno.bzeznik@imag.fr"]
+  s.description   = %q{The mindful appliance builder}
+  s.summary       = %q{Kameleon is a tool to build virtual machines from scratch}
+  s.homepage      = "http://kameleon.readthedocs.org/"
+  s.license       = "GPL-2"
+
+  s.files         = `git ls-files`.split($/)
+  s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(tests|s|features)/})
+  s.require_paths = ["lib"]
+
+  s.add_dependency 'childprocess', '~> 0.3'
+  s.add_dependency 'thor', '~> 0.15'
+  s.add_dependency 'table_print', '~> 1.5'
+  s.add_dependency 'log4r-color', '~> 1.2'
+
+  s.add_development_dependency 'pry', '~> 0.9'
+  s.add_development_dependency 'pry-debugger', '~> 0.2'
+  s.add_development_dependency 'rake', '~> 10.1'
+  s.add_development_dependency 'minitest', '~> 4.7'
+  s.add_development_dependency 'coveralls', '~> 0.7'
+end

data/lib/kameleon.rb

@@ -0,0 +1,75 @@
+require 'securerandom'
+require 'yaml'
+require 'fileutils'
+require 'optparse'
+require 'erb'
+require 'ostruct'
+require 'tempfile'
+require 'pp'
+require 'thor'
+require 'childprocess'
+require 'log4r-color'
+require 'log4r-color/configurator'
+require 'pathname'
+require 'table_print'
+
+module Kameleon
+  # to force yaml to dump ASCII-8Bit strings as strings
+  YAML::ENGINE.yamler='syck'
+
+  # add a PROGRESS and NOTICE level
+  Log4r::Configurator.custom_levels(:DEBUG, :INFO, :NOTICE,
+                                    :PROGRESS, :WARN, :ERROR,
+                                    :FATAL)
+
+  class << self
+    attr_writer :logger, :env, :source_root, :templates_path, :templates_names,
+                :templates_files
+
+    # The source root is the path to the root directory of the kameleon gem.
+    def source_root
+      @source_root ||= Pathname.new(File.expand_path('../../', __FILE__))
+    end
+
+    def templates_path
+      @templates_path ||= Pathname.new(File.join(source_root, 'templates'))
+    end
+
+    def templates_files
+      if @templates.nil?
+        files = Dir.foreach(templates_path).map do |f|
+          Pathname.new(File.join(templates_path, f)) if f.include?(".yaml")
+        end
+        @templates = files.compact
+      end
+      @templates
+    end
+
+    def templates_names
+      if @templates_names.nil?
+        names = templates_files.map do |f|
+          f.basename(f.extname).to_s
+        end
+        @templates_names = names
+      end
+      @templates_names
+    end
+
+    def logger
+      @logger ||= Log4r::Logger.new("kameleon::[global]")
+    end
+
+    def env
+      @env ||= Environment.new
+    end
+
+  end
+end
+
+# Load the things which must be loaded before anything else
+require 'kameleon/utils'
+require 'kameleon/error'
+require 'kameleon/cli'
+require 'kameleon/environment'
+require 'kameleon/version'
+require 'kameleon/logger'