wildcloud-keeper 0.0.1

data/README.md

@@ -0,0 +1,7 @@
+# Wildcloud - Keeper
+
+Responsible for managing applications on it's node - mounting, starting, managing resources.
+
+# License
+
+Project is licensed under the terms of the Apache 2 License.

data/bin/wildcloud-keeper

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+#
+# Copyright 2011 Marek Jelen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+$: << File.expand_path('../../lib', __FILE__)
+
+require 'rubygems'
+
+require 'eventmachine'
+
+require 'wildcloud/keeper/runtime'
+
+EventMachine.run do
+  Wildcloud::Keeper::Runtime.new
+end

data/lib/wildcloud/keeper.rb

@@ -0,0 +1,13 @@
+# Copyright 2011 Marek Jelen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

data/lib/wildcloud/keeper/configuration.rb

@@ -0,0 +1,32 @@
+# Copyright 2011 Marek Jelen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'yaml'
+require 'wildcloud/keeper/logger'
+
+module Wildcloud
+  module Keeper
+
+    def self.configuration
+      return @configuration if @configuration
+      file = '/etc/wildcloud/keeper.yml'
+      unless File.exists?(file)
+        file = './keeper.yml'
+      end
+      Keeper.logger.info('Configuration', "Loading from file #{file}")
+      @configuration = YAML.load_file(file)
+    end
+
+  end
+end

data/lib/wildcloud/keeper/deployers/aufs.rb

@@ -0,0 +1,92 @@
+# Copyright 2011 Marek Jelen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fileutils'
+
+require 'wildcloud/keeper/configuration'
+require 'wildcloud/keeper/logger'
+
+module Wildcloud
+  module Keeper
+    module Deployers
+      class Aufs
+
+        def deploy(options = {})
+          options[:base_image] ||= 'base'
+
+          @id = options[:id]
+          @appid = options[:appid]
+
+          @temp_file    = File.join(config['paths']['tmp'],     "#{@id}.tar.gz")
+          @image_path   = File.join(config['paths']['images'],  "#{@id}")
+          @target_path  = File.join(config['paths']['mounts'],  "#{@id}")
+          @temp_path    = File.join(config['paths']['temp'],    "#{@id}")
+
+          FileUtils.mkdir_p(@image_path)
+          FileUtils.mkdir_p(@target_path)
+          FileUtils.mkdir_p(@temp_path)
+
+          @branches = "br=#{@image_path}=rw:#{File.join(config['paths']['images'], 'base')}=ro"
+          options[:root_path] = @image_path
+
+          unless options[:build]
+            run("curl -v -o #{@temp_file} -H 'X-Appid: #{config['storage']['id']}' #{config['storage']['url']}/#{@appid}.tar.gz")
+            run("tar -xf #{@temp_file} -C #{@image_path}")
+            @branches = "br=#{@temp_path}=rw:#{@image_path}=ro:#{File.join(config['paths']['images'], options[:base_image])}=ro"
+            options[:root_path] = @temp_path
+          end
+
+          run("mount -t aufs -o #{@branches} none #{@target_path}")
+        end
+
+        def undeploy(options)
+
+          @id ||= options[:id]
+          @appid ||= options[:appid]
+
+          @temp_file    ||= File.join(config['paths']['tmp'],     "#{@id}.tar.gz")
+          @image_path   ||= File.join(config['paths']['images'],  "#{@id}")
+          @target_path  ||= File.join(config['paths']['mounts'],  "#{@id}")
+          @temp_path    ||= File.join(config['paths']['temp'],    "#{@id}")
+
+          run("umount #{@target_path}")
+
+          if options[:persistent]
+            run("tar -zcf #{@temp_file} -C #{@image_path} .")
+            run("curl -v -X PUT -H 'X-Appid: #{config['storage']['id']}' -T #{@temp_file} #{config['storage']['url']}/#{@appid}.tar.gz")
+          end
+
+          FileUtils.rm_rf(@image_path)
+          FileUtils.rm_rf(@target_path)
+          FileUtils.rm_rf(@temp_path)
+
+          FileUtils.rm(@temp_file)
+        end
+
+        private
+
+        def config
+          Keeper.configuration
+        end
+
+        def run(command)
+          Keeper.logger.debug('Aufs') { command }
+          stdout = `#{command}`
+          Keeper.logger.debug('Aufs') { stdout }
+        end
+
+      end
+    end
+  end
+end

data/lib/wildcloud/keeper/isolators/lxc.rb

@@ -0,0 +1,93 @@
+# Copyright 2011 Marek Jelen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fileutils'
+require 'erb'
+
+require 'wildcloud/keeper/configuration'
+require 'wildcloud/keeper/logger'
+
+module Wildcloud
+  module Keeper
+    module Isolators
+      class Lxc
+
+        def start(options)
+
+          @id = options[:id]
+          @target_path = File.join(config['paths']['mounts'], "#{@id}")
+
+          @config_file = File.join(config['paths']['config'], "#{@id}.config")
+          @fstab_file = File.join(config['paths']['config'], "#{@id}.fstab")
+
+          vm_config = ERB.new(File.read(File.expand_path('../../templates/app.config', __FILE__))).result(binding)
+          vm_fstab = ERB.new(File.read(File.expand_path('../../templates/app.fstab', __FILE__))).result(binding)
+
+          FileUtils.mkdir_p(config['paths']['config'])
+
+          File.open(@config_file, 'w') { |file| file.write(vm_config) }
+          File.open(@fstab_file, 'w') { |file| file.write(vm_fstab) }
+
+          run("lxc-create -f #{@config_file} -n #{@id}")
+          run("lxc-start -d -n #{@id}")
+
+        end
+
+        def stop(options)
+
+          @id ||= options[:id]
+          @config_file ||= File.join(config['paths']['config'], "#{@id}.config")
+          @fstab_file ||= File.join(config['paths']['config'], "#{@id}.fstab")
+
+
+          if options[:build]
+            build_status_file = File.join(config['paths']['images'], "#{@id}", 'var', 'build.done')
+            until File.exists?(build_status_file)
+              sleep(5)
+            end
+          end
+
+          run("lxc-stop -n #{@id}")
+          run("lxc-destroy -n #{@id}")
+
+          FileUtils.rm(@config_file)
+          FileUtils.rm(@fstab_file)
+        end
+
+        def resources(options)
+
+          @id ||= options[:id]
+
+          run("lxc-cgroup -n #{@id} memory.limit_in_bytes \"#{options[:memory]}\"") if options[:memory]
+          run("lxc-cgroup -n #{@id} memory.memsw.limit_in_bytes \"#{options[:swap]}\"") if options[:swap]
+          run("lxc-cgroup -n #{@id} cpuset.cpus \"#{options[:cpus]}\"") if options[:cpus]
+          run("lxc-cgroup -n #{@id} cpu.shares \"#{options[:cpu_share]}\"") if options[:cpu_share]
+        end
+
+        private
+
+        def config
+          Keeper.configuration
+        end
+
+        def run(command)
+          Keeper.logger.debug('LXC') { command }
+          stdout = `#{command}`
+          Keeper.logger.debug('LXC') { stdout }
+        end
+
+      end
+    end
+  end
+end

data/lib/wildcloud/keeper/logger.rb

@@ -0,0 +1,41 @@
+# Copyright 2011 Marek Jelen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'wildcloud/logger'
+require 'wildcloud/logger/middleware/console'
+require 'wildcloud/logger/middleware/amqp'
+require 'wildcloud/logger/middleware/json'
+
+require 'json'
+
+module Wildcloud
+  module Keeper
+
+    def self.logger
+      unless @logger
+        @logger = Wildcloud::Logger::Logger.new
+        @logger.application = 'wildcloud.keeper'
+        @logger.add(Wildcloud::Logger::Middleware::Console)
+      end
+      @logger
+    end
+
+    def self.add_amqp_logger(amqp)
+      @logger.add(Wildcloud::Logger::Middleware::Json)
+      @topic = AMQP::Channel.new(amqp).topic('wildcloud.logger')
+      @logger.add(Wildcloud::Logger::Middleware::Amqp, :exchange => @topic, :routing_key => 'wildcloud.keeper')
+    end
+
+  end
+end

data/lib/wildcloud/keeper/runtime.rb

@@ -0,0 +1,196 @@
+# Copyright 2011 Marek Jelen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fileutils'
+require 'thread'
+require 'yaml'
+
+require 'wildcloud/keeper/logger'
+require 'wildcloud/keeper/configuration'
+
+require 'wildcloud/keeper/isolators/lxc'
+require 'wildcloud/keeper/deployers/aufs'
+
+require 'wildcloud/keeper/transport/amqp'
+
+module Wildcloud
+  module Keeper
+    class Runtime
+
+      def initialize
+        @repository = {}
+
+        Keeper.logger.info('Runtime') { 'Starting transport' }
+        @transport = Transport::Amqp.new
+
+        Keeper.logger.info('Runtime') { 'Starting thread-pool' }
+        @queue = Queue.new
+        @thread_pool = []
+
+        Keeper.configuration['node']['workers'].times do |i|
+          Keeper.logger.debug('Runtime', "Starting thread ##{i}")
+          Thread.new(i) do |id|
+            Thread.current.abort_on_exception = false
+            loop do
+              Keeper.logger.debug('Runtime') { "Thread ##{id} waiting for task" }
+              begin
+                @queue.pop.call
+              rescue Exception => exception
+                Keeper.logger.fatal('Runtime') { "Exception in thread #{id}: #{exception.message}" }
+                Keeper.logger.debug('Runtime') { exception }
+                Keeper.logger.debug('Runtime') { exception.backtrace }
+              else
+                Keeper.logger.debug('Runtime') { "Thread ##{id} handled task successfully" }
+              end
+            end
+          end
+        end
+
+        handler = self.method(:handle)
+        @transport.start(&handler)
+        @transport.send({:type => :sshkey, :node => Keeper.configuration['node']['name'], :key => File.read(File.expand_path('~/.ssh/id_rsa.pub')).strip}, :master)
+        heartbeat
+      end
+
+      def heartbeat
+        @transport.send({:type => 'heartbeat', :node => Keeper.configuration['node']['name']}, :master)
+        EM.add_timer(5, method(:heartbeat))
+      end
+
+      def handle(message)
+        Keeper.logger.debug('Runtime') { "Message received #{message.inspect}" }
+        method = "handle_#{message['type']}".to_sym
+        scope = self
+        @queue << proc do
+          scope.send(method, message)
+        end
+      rescue Exception => exception
+        Keeper.logger.fatal('Runtime') { "Exception in runtime: #{exception.message}" }
+        Keeper.logger.fatal(exception)
+      end
+
+      def handle_build(message)
+        handle_deploy(message)
+        handle_undeploy(message)
+      end
+
+      def instance(id, message)
+        unless @repository[id]
+          @repository[id] = {:isolator => Isolators::Lxc.new, :deployer => Deployers::Aufs.new}
+          @repository[id][:options] = {
+              :id => "instance_#{message['id']}",
+              :appid => message['appid'],
+              :base_image => message['image'],
+              :persistent => message['persistent'],
+              :ip_address => message['ip_address'],
+              :memory => message['memory'],
+              :swap => message['swap'],
+              :cpus => message['cpus'],
+              :cpu_share => message['cpu_share']
+          }
+          if message['type'] == 'build'
+            @repository[id][:options].merge!({
+                :id => "build_#{message['id']}",
+                :persistent => true,
+                :build => true,
+                :repository => message['repository'],
+                :revision => message['revision']
+            })
+          end
+        end
+        @repository[id]
+      end
+
+      def get_instance_name(message)
+        message['type'] == 'build' ? "build_#{message['id']}" : "instance_#{message['id']}"
+      end
+
+      def get_instance(message)
+        instance = instance(get_instance_name(message), message)
+        options = instance[:options]
+        [instance, options]
+      end
+
+      def handle_deploy(message)
+        instance, options = get_instance(message)
+        instance[:deployer].deploy(options)
+        write_system_configuration(options)
+        instance[:isolator].start(options)
+        unless options[:build]
+          @transport.send({:type => :deployed, :node => Keeper.configuration['node']['name'], :id => message['id']}, :master)
+        end
+      end
+
+      def handle_undeploy(message)
+        instance, options = get_instance(message)
+        instance[:isolator].stop(options)
+        clean_system_configuration(options)
+        if options[:build]
+          build_log = File.read(File.join(options[:root_path], 'var', 'build.log'))
+        end
+        instance[:deployer].undeploy(options)
+        if options[:build]
+          @transport.send({:type => :build_log, :node => Keeper.configuration['node']['name'], :id => message['id'], :content => build_log}, :master)
+        else
+          @transport.send({:type => :undeployed, :node => Keeper.configuration['node']['name'], :id => message['id']}, :master)
+        end
+      end
+
+      def handle_resources(message)
+        instance_name = get_instance_name(message)
+        instance = @repository[instance_name]
+        unless instance
+          return
+        end
+        options = instance[:options]
+        options[:memory] = message['memory']
+        options[:swap] = message['swap']
+        options[:cpus] = message['cpus']
+        options[:cpu_share] = message['cpu_share']
+        instance[:isolator].resources(options)
+      end
+
+      def write_system_configuration(options)
+        root = options[:root_path]
+
+        interfaces = ERB.new(File.read(File.expand_path('../templates/interfaces', __FILE__))).result(binding)
+        interfaces_path = File.join(root, 'etc', 'network', 'interfaces')
+        FileUtils.mkdir_p(File.dirname(interfaces_path))
+        File.open(interfaces_path, 'w') { |file| file.write(interfaces) }
+
+        if options[:build]
+          FileUtils.mkdir_p(File.join(root, 'root', '.ssh'))
+          FileUtils.cp(File.expand_path('~/.ssh/id_rsa'), File.join(root, 'root', '.ssh', 'id_rsa'))
+          FileUtils.cp(File.expand_path('~/.ssh/id_rsa.pub'), File.join(root, 'root', '.ssh', 'id_rsa.pub'))
+
+          File.open(File.join(root, 'root', 'build.yml'), 'w') { |file| file.write(YAML::dump(options)) }
+        end
+      end
+
+      def clean_system_configuration(options)
+        root = options[:root_path]
+
+        if options[:build]
+          FileUtils.rm_rf(File.join(root, 'root', '.ssh'))
+        end
+      end
+
+      def handle_quit
+        Keeper.logger.fatal('Runtime') { "Shutdown requested." }
+        EventMachine.stop_event_loop
+      end
+
+    end
+  end
+end