chef-tlc-workflow 0.1.0

data/lib/chef-tlc-workflow.rb

@@ -0,0 +1,5 @@
+require "chef-tlc-workflow/version"
+
+module ChefTLCWorkflow
+  # Your code goes here...
+end

data/lib/chef-tlc-workflow/helpers.rb

@@ -0,0 +1,126 @@
+module ChefTLCWorkflow
+
+  module Helpers
+
+    #
+    # reads the direct dependencies defined in `metadata.rb` and 
+    # returns a map of dependency => version, e.g.:
+    #
+    #   { 'foo' => '1.0.0', 'bar' => '0.1.0' }
+    #
+    def self.read_metadata_deps
+      require 'chef/cookbook/metadata'
+      metadata = ::Chef::Cookbook::Metadata.new
+      metadata.from_file "metadata.rb"
+      metadata.dependencies
+    end
+
+    #
+    # reads the direct dependencies defined in `Cheffile` and 
+    # returns a map of dependency => version, e.g.:
+    #
+    #   { 'foo' => '1.0.0', 'bar' => '0.1.0' }
+    #
+    def self.read_cheffile_deps
+      require 'librarian/chef/environment'
+      env = ::Librarian::Chef::Environment.new
+      deps = env.spec.dependencies
+      Hash[deps.map { |dep| [dep.name, dep.requirement.to_s] }]
+    end
+
+    #
+    # resolves the dependencies as specified in `Cheffile` and 
+    # returns a map of dependency => version (including the 
+    # transitive ones), e.g.:
+    #
+    #   { 'foo' => '1.0.0', 'bar' => '0.1.0', 'baz_which_depends_on_bar' => '1.5.0' }
+    #
+    def self.read_and_resolve_cheffile_deps
+      require 'librarian/chef/environment'
+      env = ::Librarian::Chef::Environment.new
+      deps = env.resolver.resolve(env.spec).manifests
+      Hash[deps.map { |dep| [dep.name, dep.version.to_s] }]
+    end
+
+    #
+    # returns the direct dependencies defined in `metadata.rb` as an 
+    # array of triples:
+    # 
+    #   [[<cookbook_name>, <cookbook_version>, <location>], ...]
+    #
+    # where `<location>` is `nil`, unless the `locations_yml`
+    # parameter is given and the specified file contains a location 
+    # mapping for `<cookbook_name>`
+    # 
+    # Example usage in `Cheffile`:
+    #   
+    #   require 'chef-tlc-workflow/helpers'
+    #   
+    #   ChefTLCWorkflow::Helpers::from_metadata.each do |cb_name, cb_version, location|
+    #     cookbook cb_name, cb_version, location
+    #   end
+    # 
+    def self.from_metadata(locations_yml = nil)
+      read_metadata_deps.to_a.map do |cb_name, cb_version|
+        if locations_yml
+          inferred_location = resolve_location_from_file(locations_yml, cb_name, cb_version)
+        else
+          inferred_location = nil
+        end
+        [cb_name, cb_version, inferred_location]
+      end
+    end
+
+    #
+    # reads in the YAML file containing the application cookbooks and
+    # returns either all application cookbooks defined there or a specific
+    # ones if the `name` and/or `version` parameters are given to filter
+    # the selection.
+    #
+    # Example .yml file:
+    #
+    #   - name: "sample-app"
+    #     version: "0.1.0"
+    #     git: "https://github.com/tknerr/sample-app-tlc.git"
+    #     ref: "v0.1.0"
+    #   - name: "sample-app"
+    #     version: "0.2.0"
+    #     path: "../sample-app"
+    #
+    #
+    def self.read_app_cookbooks(yml_file, name = nil, version = nil)
+      # TODO: validate format
+      app_cookbooks = YAML.load_file yml_file
+      app_cookbooks.select! { |ac| ac['name'] == name } if name
+      app_cookbooks.select! { |ac| ac['version'] == version } if version
+      app_cookbooks
+    end
+
+    #
+    # parse input string "<app-cookbook-name>@<version>" into
+    # a `[name, version]` pair by splitting at the '@' character
+    #
+    def self.parse_name_and_version(string)
+      if string == nil
+        [nil, nil]
+      else
+        string.split '@'
+      end
+    end
+
+    private
+
+      def self.resolve_location_from_file(locations_yml, cb_name, cb_version)
+        cookbook_index = YAML.load_file(locations_yml)
+        if cookbook_index[cb_name]
+          default_opts = Hash.new
+          default_opts[:git] = cookbook_index[cb_name][:git] if cookbook_index[cb_name][:git]
+          default_opts[:ref] = cookbook_index[cb_name][:ref] if cookbook_index[cb_name][:ref]
+          version_specific_opts = cookbook_index[cb_name][cb_version] || {}
+          return default_opts.merge version_specific_opts
+        else
+          return nil
+        end
+      end
+  end
+end

data/lib/chef-tlc-workflow/version.rb

@@ -0,0 +1,3 @@
+module ChefTLCWorkflow
+  VERSION = "0.1.0"
+end

data/lib/chef-workflow/tasks/tlc.rb

@@ -0,0 +1,4 @@
+require 'chef-workflow-tasklib'
+
+chef_workflow_task 'tlc/deps'
+chef_workflow_task 'tlc/test'

data/lib/chef-workflow/tasks/tlc/deps.rb

@@ -0,0 +1,100 @@
+
+namespace :tlc do
+  namespace :deps do
+
+    require 'chef-tlc-workflow/helpers'
+
+    #
+    # resolve dependencies using librarian
+    #
+    task :resolve do
+      require 'fileutils'
+      FileUtils.rm_rf('Cheffile.lock')
+      sh "librarian-chef clean"
+      sh "librarian-chef install"
+    end
+
+    #
+    # check if dependencies in metadata.rb and Cheffile are consistent
+    #
+    task :check do
+      errors = []
+      metadata_deps = ChefTLCWorkflow::Helpers::read_metadata_deps
+      resolved_cheffile_deps = ChefTLCWorkflow::Helpers::read_and_resolve_cheffile_deps
+      resolved_cheffile_deps.each do | dep, version |
+        if metadata_deps.has_key?(dep)
+          metadata_ver = ::Gem::Requirement.new(metadata_deps[dep])
+          cheffile_ver = ::Gem::Requirement.new(version)
+          if metadata_ver != cheffile_ver
+            errors << "dependency version for '#{dep}' is inconsistent: '#{metadata_ver}' vs '#{cheffile_ver}'!"
+          end
+        else
+          errors << "dependency '#{dep}' is missing in metadata.rb!"
+        end
+      end
+      puts errors.empty? ? "everything OK" : errors
+    end
+
+    #
+    # resolve an application cookbook from `app_cookbooks.yml` with all its dependencies
+    #
+    task :resolve_app_cookbook, [:app_cookbook] do |t, args|
+      name, version = ChefTLCWorkflow::Helpers::parse_name_and_version(args[:app_cookbook])
+      app_cookbooks = ChefTLCWorkflow::Helpers::read_app_cookbooks("app_cookbooks.yml", name, version)
+      app_cookbooks.each do |app_cookbook|
+        resolve_app_cookbook(app_cookbook)
+      end
+    end
+
+    #
+    # resolve an application cookbook with all it's dependenices 
+    # whilst honoring the application cookbook's Cheffile:
+    #
+    # 1. clone (:git,:ref) or copy (:path) the app cookbook to './tmp'
+    # 2. resolve dependencies (inlcuding app cookbook itself) as defined in
+    #    the app cookbook's Cheffile to './cookbooks/<app-cookbook-name>-<version>'
+    #
+    #
+    def self.resolve_app_cookbook(app_cookbook)
+      name = app_cookbook['name']
+      version = app_cookbook['version']
+      git_loc = app_cookbook['git']
+      git_ref = app_cookbook['ref']
+      path_loc = app_cookbook['path']
+      has_git_loc = git_loc != nil
+      has_path_loc = path_loc != nil
+
+      fail "must specify either `git` or `path` location for #{name}" if !has_path_loc && !has_git_loc
+      fail "must not specify both `git` and `path` location for #{name}" if has_path_loc && has_git_loc
+
+      target_dir = "cookbooks/#{name}-#{version}"
+      tmp_dir = "tmp/tlc/#{name}-#{version}"
+
+      # clone / copy to temp dir
+      FileUtils.rm_rf tmp_dir
+      FileUtils.mkdir_p tmp_dir
+      if has_git_loc
+        sh "git clone -b #{git_ref || 'master'} #{git_loc} #{tmp_dir}"
+      elsif has_path_loc
+        FileUtils.cp_r cookbook_files_to_copy(path_loc), tmp_dir
+      end
+
+      # resolve deps from tmp_dir into target_dir
+      fail "No Cheffile found in '#{tmp_dir}'" unless File.exist? "#{tmp_dir}/Cheffile"
+      FileUtils.rm_rf target_dir
+      FileUtils.mkdir_p target_dir
+      sh "cd #{tmp_dir} && librarian-chef install --path #{File.absolute_path(target_dir)}"
+
+      # copy application cookbook itself if it was not reference in Cheffile using `:path => '.'`
+      app_cookbook_in_targetdir = "#{target_dir}/#{name}"
+      unless File.exist? app_cookbook_in_targetdir
+        FileUtils.mkdir_p app_cookbook_in_targetdir
+        FileUtils.cp_r Dir.glob("#{tmp_dir}/*"), app_cookbook_in_targetdir
+      end
+    end
+
+    def self.cookbook_files_to_copy(path)
+      Dir.glob("#{path}/*").reject {|p| p.end_with? '/tmp' }
+    end
+  end
+end

data/lib/chef-workflow/tasks/tlc/test.rb

@@ -0,0 +1,15 @@
+
+namespace :tlc do
+  namespace :test do
+
+  	#
+    # destroy the default Vagrant VM, resolve dependencies and converge the default Vagrant VM
+    #
+    task :converge do
+      sh "vagrant destroy -f"
+      Rake::Task["tlc:deps:resolve"].invoke
+      sh "vagrant up"
+    end
+  end
+end
+

data/test/Gemfile

@@ -0,0 +1,7 @@
+source :rubygems
+
+source 'https://gems.gemfury.com/hUe8s8nSyzxs7JMMSZV8/' # vagrant-1.0.5.1
+source 'https://gems.gemfury.com/psBbdHx94zqZrvxpiVmm/' # librarian-0.0.26.2
+
+gem "chef-tlc-workflow", "0.1.0"
+  #:path => "#{File.dirname(__FILE__)}/../../chef-tlc-workflow"

data/test/ec2-bootstrap/.chef/bootstrap/omnibus-chef.sh

@@ -0,0 +1,34 @@
+#!/bin/bash -ex
+
+if which curl 2>/dev/null; then
+    HTTP_GET_CMD="curl -L"
+else
+    HTTP_GET_CMD="wget -qO-"
+fi
+
+# read/parse user-data
+USERDATA=`$HTTP_GET_CMD http://169.254.169.254/latest/user-data`
+CHEF_VERSION=`echo $USERDATA | tr -s ',' '\n' | grep "chef_version" | cut -d'=' -f2`
+FQDN=`echo $USERDATA | tr -s ',' '\n' | grep "fqdn" | cut -d'=' -f2`
+
+# install Chef
+if [ -z "$CHEF_VERSION" ]; then
+	$HTTP_GET_CMD https://www.opscode.com/chef/install.sh | sudo bash -s
+else
+	$HTTP_GET_CMD https://www.opscode.com/chef/install.sh | sudo bash -s -- -v $CHEF_VERSION
+fi
+
+# set proper hostname
+if [ "$FQDN" ]; then
+	HOSTNAME=`echo $FQDN | sed -r 's/\..*//'`
+	if [ "`grep "127.0.1.1" /etc/hosts`" ]; then
+		sudo sed -r -i "s/^(127[.]0[.]1[.]1[[:space:]]+).*$/\\1$FQDN $HOSTNAME/" /etc/hosts
+	else
+		sudo sed -r -i "s/^(127[.]0[.]0[.]1[[:space:]]+localhost[[:space:]]*)$/\\1\n127.0.1.1 $FQDN $HOSTNAME/" /etc/hosts
+	fi
+	sudo sed -i "s/.*$/$HOSTNAME/" /etc/hostname
+	sudo hostname -F /etc/hostname
+fi
+
+# XXX: ensure that file_cache_path configured in solo.rb exists (Mccloud does not create it)
+sudo mkdir -p /var/chef-solo 

data/test/ec2-bootstrap/.gitignore

@@ -0,0 +1,3 @@
+cookbooks/
+tmp/
+Cheffile.lock

data/test/ec2-bootstrap/Mccloudfile

@@ -0,0 +1,60 @@
+Mccloud::Config.run do |config|
+
+  # identity / namespace
+  config.mccloud.prefix="mccloud"
+  config.mccloud.environment="tlc"
+  config.mccloud.identity=ENV['USERNAME']
+
+  # define AWS cloud provider for EU-West region
+  config.provider.define "aws-eu-west" do |provider_config|
+    provider_config.provider.flavor = :aws
+    provider_config.provider.options = { }
+    provider_config.provider.region = "eu-west-1"
+    provider_config.provider.check_keypairs = true
+    provider_config.provider.check_security_groups = true
+    provider_config.provider.namespace = "mccloud-tlc-#{ENV['USERNAME']}"
+  end
+
+  # ***********************************************
+  # VM Definitions
+  # ***********************************************
+  
+  config.vm.define "sample-app" do |config|
+
+    # official Ubuntu 12.04 AMI
+    config.vm.ami = "ami-524e4726"
+    config.vm.provider= "aws-eu-west"
+    config.vm.flavor = "m1.small"
+    config.vm.zone = "eu-west-1c"
+    config.vm.user = "ubuntu"
+
+    # NOTE: the security groups must exist (e.g. create it beforehand via AWS console)
+    config.vm.security_groups = [ "mccloud", "http" ]
+    
+    # see http://fog.io/1.1.2/rdoc/Fog/Compute/AWS/Servers.html
+    # and https://github.com/fog/fog/blob/v1.1.2/lib/fog/aws/requests/compute/run_instances.rb
+    config.vm.create_options = {
+      :user_data => "chef_version=10.18.2-1,fqdn=sample-app.example.com"
+    }
+
+    # NOTE: the keypair (for logging in to the VM) must exist (e.g. create it beforehand via AWS console)
+    config.vm.key_name = "mccloud-key-tlc"
+    config.vm.private_key_path = "#{ENV['HOME']}/.ssh/mccloud_rsa"
+    config.vm.public_key_path = "#{ENV['HOME']}/.ssh/mccloud_rsa.pub"
+  
+    # bootstrap template (runs at first-boot only)
+    config.vm.bootstrap = ".chef/bootstrap/omnibus-chef.sh"
+
+    # provisioning sript (runs on each mccloud up or provision)
+    config.vm.provision :chef_solo do |chef|
+      chef.cookbooks_path = [ "cookbooks/sample-app-0.1.0" ]
+      chef.log_level = "info"
+      chef.add_recipe "sample-app"
+      chef.json.merge!({
+        :sample_app => {
+          :words_of_wisdom => "YEAH! I did it for the Lulz!"
+        }
+      })
+    end
+  end
+end

data/test/ec2-bootstrap/Rakefile

@@ -0,0 +1,57 @@
+require 'bundler/setup'
+require 'fileutils'
+require 'chef-workflow/tasks/tlc/deps'
+
+desc "resolve application cookbook with all its dependencies"
+task :resolve_deps, [:app_cookbook] do |t, args|
+  Rake::Task["tlc:deps:resolve_app_cookbook"].invoke(args[:app_cookbook])
+end
+
+desc "bring up the mccloud \"VM\" as configured in the Mccloudfile"
+task :up, [:vm_name] do |t, args|
+  vm_name = get_required_args(args, :vm_name)
+  sh "mccloud up #{vm_name}"
+end
+
+desc "provision the mccloud \"VM\" with the provisioners as defined in the Mccloudfile"
+task :provision, [:vm_name] do |t, args|
+  vm_name = get_required_args(args, :vm_name)
+  sh "mccloud provision #{vm_name}"
+end
+
+desc "destroy the mccloud \"VM\" with the given name"
+task :destroy, [:vm_name] do |t, args|
+  vm_name = get_required_args(args, :vm_name)
+  sh "mccloud destroy #{vm_name}"
+end
+
+desc "ssh into the mccloud \"VM\" with the given name"
+task :ssh, [:vm_name] do |t, args|
+  vm_name = get_required_args(args, :vm_name)
+  sh "mccloud ssh #{vm_name}"
+end
+
+desc "show status of all mccloud \"VMs\" defined in the Mccloudfile"
+task :status do
+  sh "mccloud status"
+end
+
+desc "returns the ip address of the mccloud \"VM\" with the given name"
+task :get_ip, [:vm_name] do |t, args|
+  vm_name = get_required_args(args, :vm_name)
+  # TODO: filter out IP of given vm rather than printing status for all
+  sh "mccloud status"
+end
+
+
+#
+# helper methods below
+#
+def get_required_args(args, *param_keys)
+  param_values = Array.new
+  param_keys.each do |param_key|
+    fail "parameter #{param_key.to_sym} is required" unless args[param_key.to_sym]
+    param_values << args[param_key.to_sym]
+  end
+  param_values.size == 1 ? param_values[0] : param_values
+end

data/test/ec2-bootstrap/app_cookbooks.yml

@@ -0,0 +1,12 @@
+
+- name: "sample-app"
+  version: "0.1.0"
+  path: "../sample-app"
+  # git: "https://github.com/tknerr/sample-app-tlc.git"
+  # ref: "v0.1.0"
+
+- name: "sample-app"
+  version: "0.2.0"
+  path: "../sample-app"
+  # git: "https://github.com/tknerr/sample-app-tlc.git"
+  # ref: "v0.2.0"