marabunta 0.1.0

data/LICENSE

@@ -0,0 +1,22 @@
+== Marabunta
+
+Copyright (c) 2010 David Calavera
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,36 @@
+Marabunta
+---------
+
+Large scale cloud nodes distribution and deployment using Bittorrent, Capistrano and Libvirt.
+
+How it works
+------------
+
+Marabunta relies on Murder, http://github.com/lg/murder, to distribute
+the virtual disks to the cloud nodes and uses the Libvirt api to deploy
+and start the images once they are distributed.
+
+Currently Marabunta just supports deployments on Kvm but it can support as many hypervisors
+as Libvirt supports.
+
+It uses the libvirt jar library rather than ruby-libvirt because it's
+more up to date so the tasks must be run with JRuby.
+
+Configuration
+-------------
+
+The most simple way to use Marabunta is setting the deployment strategy
+to `:marabunta`. It also requires to set the name of the `:hypervisor`.
+
+There are some deployment configuration templates into the directory `examples`.
+
+Marabunta can also be used without the deployment strategy. Follow the
+steps that Murder requires for the manual usage and then run the
+marabunta deploy task:
+
+  $ jruby -S cap marabunta:deploy tag="MURDER_TAG_NAME"
+
+Copyright
+---------
+
+Copyright (c) 2010 David Calavera<calavera@apache.org>. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,133 @@
+require 'rubygems'
+require 'rake'
+require 'date'
+
+#############################################################################
+#
+# Helper functions
+#
+#############################################################################
+
+def name
+  @name ||= Dir['*.gemspec'].first.split('.').first
+end
+
+def version
+  line = File.read("lib/#{name}.rb")[/^\s*VERSION\s*=\s*.*/]
+  line.match(/.*VERSION\s*=\s*['"](.*)['"]/)[1]
+end
+
+def date
+  Date.today.to_s
+end
+
+def rubyforge_project
+  name
+end
+
+def gemspec_file
+  "#{name}.gemspec"
+end
+
+def gem_file
+  "#{name}-#{version}.gem"
+end
+
+def replace_header(head, header_name)
+  head.sub!(/(\.#{header_name}\s*= ').*'/) { "#{$1}#{send(header_name)}'"}
+end
+
+#############################################################################
+#
+# Standard tasks
+#
+#############################################################################
+
+task :default => :spec
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_opts = ['--options', 'spec/spec.opts']
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "#{name} #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+desc "Open an irb session preloaded with this library"
+task :console do
+  sh "irb -rubygems -r ./lib/#{name}.rb"
+end
+
+#############################################################################
+#
+# Custom tasks (add your own tasks here)
+#
+#############################################################################
+
+
+
+#############################################################################
+#
+# Packaging tasks
+#
+#############################################################################
+
+task :release => :build do
+  unless `git branch` =~ /^\* master$/
+    puts "You must be on the master branch to release!"
+    exit!
+  end
+  sh "git commit --allow-empty -a -m 'Release #{version}'"
+  sh "git tag v#{version}"
+  sh "git push origin master"
+  sh "git push --tags"
+  sh "gem push pkg/#{name}-#{version}.gem"
+end
+
+task :build => :gemspec do
+  sh "mkdir -p pkg"
+  sh "gem build #{gemspec_file}"
+  sh "mv #{gem_file} pkg"
+end
+
+task :gemspec => :validate do
+  # read spec file and split out manifest section
+  spec = File.read(gemspec_file)
+  head, manifest, tail = spec.split("  # = MANIFEST =\n")
+
+  # replace name version and date
+  replace_header(head, :name)
+  replace_header(head, :version)
+  replace_header(head, :date)
+  #comment this out if your rubyforge_project has a different name
+  replace_header(head, :rubyforge_project)
+
+  # determine file list from git ls-files
+  files = `git ls-files`.
+    split("\n").
+    sort.
+    reject { |file| file =~ /^\./ }.
+    reject { |file| file =~ /^(rdoc|pkg)/ }.
+    map { |file| "    #{file}" }.
+    join("\n")
+
+  # piece file back together and write
+  manifest = "  s.files = %w[\n#{files}\n  ]\n"
+  spec = [head, manifest, tail].join("  # = MANIFEST =\n")
+  File.open(gemspec_file, 'w') { |io| io.write(spec) }
+  puts "Updated #{gemspec_file}"
+end
+
+task :validate do
+  unless Dir['VERSION*'].empty?
+    puts "A `VERSION` file at root level violates Gem best practices."
+    exit!
+  end
+end

data/examples/kvm-deploy.rb

@@ -0,0 +1,28 @@
+require 'marabunta'
+
+set :user, 'marabunta'
+
+set :scm, :none
+set :repository, '/tmp/disks'
+
+#
+# MURDER CONFIGURATION
+#
+set :remote_murder_path, '/var/lib/murder'
+after 'deploy:setup', 'murder:distribute_files'
+
+before 'murder:start_seeding', 'murder:start_tracker'
+after 'murder:stop_seeding', 'murder:stop_tracker'
+
+role :peers, 'localhost', '10.60.1.76'
+role :seeder, 'localhost'
+role :tracker, 'localhost'
+
+set :default_destination_path, '/var/lib/virt'
+set :default_seeder_files_path, '/tmp/disks'
+
+#
+# MARABUNTA CONFIGURATION
+#
+set :deploy_via, :marabunta
+set :hypervisor, 'Kvm'

data/lib/capistrano/recipes/deploy/strategy/marabunta.rb

@@ -0,0 +1,18 @@
+require 'murder'
+require 'capistrano/recipes/deploy/strategy/murder'
+
+module Capistrano
+  module Recipes
+    module Deploy
+      module Strategy
+        class Marabunta < Murder
+          def upload(filename, remote_filename)
+            super(filename, remote_filename)
+
+            marabunta.deploy
+          end
+        end
+      end
+    end
+  end
+end

data/lib/marabunta.rb

@@ -0,0 +1,26 @@
+module Marabunta
+  VERSION = '0.1.0'
+end
+
+require File.expand_path('../marabunta/marabunta', __FILE__)
+
+Capistrano::Configuration.instance(:must_exist).load do
+
+  # currently just 'Kvm' is supported
+  set :hypervisor, ''
+
+  namespace :marabunta do
+    desc 'Deploy disks into the target machines'
+    task :deploy, :role => :seeder do
+      require_tag
+
+      # TODO: find a better way to know the disks to deploy
+      disks = capture("ls #{default_seeder_files_path}").chomp
+
+      destination_path = File.join(default_destination_path, tag)
+      disks = disks.split("\s+").map {|file| File.join(destination_path, file) }
+
+      Marabunta.deploy(roles[:peers].to_ary, hypervisor,  disks)
+    end
+  end
+end

data/lib/marabunta/kvm.rb

@@ -0,0 +1,57 @@
+module Marabunta
+  module Hypervisor
+    require 'erb'
+
+    import org.libvirt.Connect
+    import java.util.UUID
+
+    class Kvm
+      attr_reader :address
+
+      TCP = "qemu+tcp://%s/system?no_tty"
+
+      def self.connect(address)
+        Kvm.new(address).connect
+      end
+
+      def initialize(ip_address, protocol = TCP)
+        @address = protocol % ip_address
+      end
+
+      def connect
+        @connection = Connect.new(@address, false)
+        self
+      end
+
+      def disconnect
+        @connection.close
+      end
+
+      def deploy(disks_path)
+        disks_path.each do |disk|
+          domain = KvmDomain[disk]
+
+          @connection.domainDefineXML(domain).create
+        end
+      end
+    end
+
+    class KvmDomain
+      def self.[](disk)
+        KvmDomain.new(disk).create
+      end
+
+      def initialize(disk)
+        @uuid = UUID.randomUUID.to_s
+        @name = File.basename(disk)
+        @volume_path = disk
+        # TODO: allow more configuration options
+      end
+
+      def create
+        template = ERB.new(File.read(File.expand_path('../kvm_domain.erb', __FILE__)))
+        template.result(binding)
+      end
+    end
+  end
+end

data/lib/marabunta/kvm_domain.erb

@@ -0,0 +1,36 @@
+<domain type="kvm">
+    <name><%= @name %></name>
+    <uuid><%= @uuid %></uuid>
+    <memory>524288</memory>
+    <currentMemory>524288</currentMemory>
+    <vcpu>1</vcpu>
+    <os>
+        <type>hvm</type>
+        <boot dev="hd"/>
+        <loader>/usr/bin/qemu-kvm</loader>
+    </os>
+    <features>
+        <acpi/>
+        <apic/>
+        <pae/>
+    </features>"
+    <clock offset="utc"/>
+    <on_poweroff>destroy</on_poweroff>
+    <on_reboot>restart</on_reboot>
+    <on_crash>destroy</on_crash>
+    <devices>
+        <emulator>/usr/bin/qemu-kvm</emulator>
+        <input type="mouse" bus="ps2"/>
+        <!--graphics type='vnc' port='???' listen='???'/-->
+        <serial type="pty">
+            <target port="0"/>
+        </serial>
+        <console type="pty">
+            <target port="0"/>
+        </console>
+        <disk device="disk" type="file">
+            <target dev="hda" bus="ide"/>
+            <source file="<%= @volume_path %>"/>
+        </disk>
+    </devices>
+</domain>

data/lib/marabunta/marabunta.rb

@@ -0,0 +1,25 @@
+module Marabunta
+  require 'java'
+
+  __JARS__ = File.expand_path('../../../jars', __FILE__)
+  require File.expand_path('jna-3.0.9.jar', __JARS__)
+  require File.expand_path('libvirt-0.4.6.jar', __JARS__)
+  require File.expand_path('../kvm', __FILE__)
+
+
+  def self.deploy(machines, hypervisor, disks_path)
+    unless Marabunta::Hypervisor.const_defined?(hypervisor)
+      raise "Unsupported hypervisor: #{hypervisor}"
+    end
+
+    hypervisor = Marabunta::Hypervisor.const_get(hypervisor)
+
+    machines.each do |machine|
+      connection = hypervisor.connect(machine)
+
+      connection.deploy(disks_path)
+
+      connection.disconnect
+    end
+  end
+end

data/marabunta.gemspec

@@ -0,0 +1,83 @@
+## This is the rakegem gemspec template. Make sure you read and understand
+## all of the comments. Some sections require modification, and others can
+## be deleted if you don't need them. Once you understand the contents of
+## this file, feel free to delete any comments that begin with two hash marks.
+## You can find comprehensive Gem::Specification documentation, at
+## http://docs.rubygems.org/read/chapter/20
+Gem::Specification.new do |s|
+  s.specification_version = 2 if s.respond_to? :specification_version=
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.rubygems_version = '1.3.5'
+
+  ## Leave these as is they will be modified for you by the rake gemspec task.
+  ## If your rubyforge_project name is different, then edit it and comment out
+  ## the sub! line in the Rakefile
+  s.name              = 'marabunta'
+  s.version           = '0.1.0'
+  s.date              = '2010-09-24'
+  s.rubyforge_project = 'marabunta'
+
+  ## Make sure your summary is short. The description may be as long
+  ## as you like.
+  s.summary     = "Cloud nodes deployment using BitTorrent, Capistrano and Libvirt"
+  s.description = "Large scale distribution and deployment of virtual disks using BitTorrent, Capistrano and Libvirt"
+
+  ## List the primary authors. If there are a bunch of authors, it's probably
+  ## better to set the email to an email list or something. If you don't have
+  ## a custom homepage, consider using your GitHub URL or the like.
+  s.authors  = ["David Calavera"]
+  s.email    = 'calavera@apache.org'
+  s.homepage = 'http://github.com/calavera/marabunta'
+
+  ## This gets added to the $LOAD_PATH so that 'lib/NAME.rb' can be required as
+  ## require 'NAME.rb' or'/lib/NAME/file.rb' can be as require 'NAME/file.rb'
+  s.require_paths = %w[lib]
+
+  ## If your gem includes any executables, list them here.
+  #s.executables = ["name"]
+  #s.default_executable = 'name'
+
+  ## Specify any RDoc options here. You'll want to add your README and
+  ## LICENSE files to the extra_rdoc_files list.
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.extra_rdoc_files = %w[README.md LICENSE]
+
+  ## List your runtime dependencies here. Runtime dependencies are those
+  ## that are needed for an end user to actually USE your code.
+  s.add_dependency('capistrano')
+  s.add_dependency('murder')
+
+  ## List your development dependencies here. Development dependencies are
+  ## those that are only needed during development
+  s.add_development_dependency('rspec')
+  s.add_development_dependency('mocha')
+
+
+  ## Leave this section as-is. It will be automatically generated from the
+  ## contents of your Git repository via the gemspec task. DO NOT REMOVE
+  ## THE MANIFEST COMMENTS, they are used as delimiters by the task.
+  # = MANIFEST =
+  s.files = %w[
+    LICENSE
+    README.md
+    Rakefile
+    examples/kvm-deploy.rb
+    jars/jna-3.0.9.jar
+    jars/libvirt-0.4.6.jar
+    lib/capistrano/recipes/deploy/strategy/marabunta.rb
+    lib/marabunta.rb
+    lib/marabunta/kvm.rb
+    lib/marabunta/kvm_domain.erb
+    lib/marabunta/marabunta.rb
+    marabunta.gemspec
+    spec/marabunta/kvm_spec.rb
+    spec/marabunta/marabunta_spec.rb
+    spec/spec.opts
+    spec/spec_helper.rb
+  ]
+  # = MANIFEST =
+
+  ## Test files will be grabbed from the file list. Make sure the path glob
+  ## matches what you actually use.
+  s.test_files = s.files.select { |path| path =~ /^spec\/.*_spec\.rb/ }
+end

data/spec/marabunta/kvm_spec.rb

@@ -0,0 +1,61 @@
+require File.expand_path('../../spec_helper',  __FILE__)
+
+describe Marabunta::Hypervisor::Kvm do
+  before(:each) do
+  end
+
+  it "creates a new instance for each connection" do
+    org.libvirt.Connect.expects(:new).twice.returns(mock)
+
+    conn1 = Marabunta::Hypervisor::Kvm.connect('localhost')
+    conn2 = Marabunta::Hypervisor::Kvm.connect('localhost')
+
+    conn1.should_not == conn2
+  end
+
+  it "uses tcp connection by default" do
+    org.libvirt.Connect.expects(:new).returns(mock)
+    conn = Marabunta::Hypervisor::Kvm.connect('localhost')
+
+    conn.address.should == (Marabunta::Hypervisor::Kvm::TCP % 'localhost')
+  end
+
+  it "creates a new domain for each disk" do
+    connection_mock = mock
+    domain_mock = mock
+
+    domain_mock.expects(:create).twice
+    connection_mock.expects(:domainDefineXML).twice.
+      returns(domain_mock)
+
+    org.libvirt.Connect.expects(:new).returns(connection_mock)
+
+    conn = Marabunta::Hypervisor::Kvm.connect('localhost')
+    conn.deploy(['disk1', 'disk2'])
+  end
+
+  context 'domain' do
+    it "generates an uuid for each disk" do
+      domain1 = Marabunta::Hypervisor::KvmDomain.new('disk1').create
+      domain2 = Marabunta::Hypervisor::KvmDomain.new('disk2').create
+
+      xml_value(domain1, 'uuid').should_not == xml_value(domain2, 'uuid')
+    end
+
+    it "uses the basename of the disk as domain name" do
+      domain = Marabunta::Hypervisor::KvmDomain.new('/tmp/disk').create
+
+      xml_value(domain, 'name').should == 'disk'
+    end
+
+    it "uses the full disk path as volume path" do
+      domain = Marabunta::Hypervisor::KvmDomain.new('/tmp/disk').create
+
+      domain[%r{<source file="([^"]+)"}, 1].should == '/tmp/disk'
+    end
+
+    def xml_value(xml, node)
+      xml[%r{<(#{node})>(.+)</\1>}, 2]
+    end
+  end
+end

data/spec/marabunta/marabunta_spec.rb

@@ -0,0 +1,27 @@
+require File.expand_path('../../spec_helper',  __FILE__)
+
+describe Marabunta do
+  it "does nothing if the hypervisor is not supported" do
+    lambda {
+      Marabunta.deploy([], 'Xen', '/tmp')
+    }.should raise_error
+  end
+
+  it "loads the hypervisor if it's supported" do
+    lambda {
+      Marabunta.deploy([], 'Kvm', '/tmp')
+    }.should_not raise_error
+  end
+
+  it "connects with each machine's hypervisor" do
+    kvm = mock
+    kvm.expects(:deploy)
+    kvm.expects(:disconnect)
+
+    Marabunta::Hypervisor::Kvm.expects(:connect).
+      with('localhost').
+      returns(kvm)
+
+    Marabunta.deploy(['localhost'], 'Kvm', '/tmp')
+  end
+end

data/spec/spec.opts

@@ -0,0 +1,2 @@
+--colour
+--format specdoc

data/spec/spec_helper.rb

@@ -0,0 +1,16 @@
+require 'rubygems'
+begin
+  require 'spec'
+rescue LoadError
+  gem 'rspec'
+  require 'spec'
+end
+
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+
+require 'marabunta/marabunta'
+require 'mocha'
+
+Spec::Runner.configure do |config|
+  config.mock_with :mocha
+end

metadata

@@ -0,0 +1,132 @@
+--- !ruby/object:Gem::Specification 
+name: marabunta
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- David Calavera
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-09-24 00:00:00 +02:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: capistrano
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: murder
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: mocha
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id004
+description: Large scale distribution and deployment of virtual disks using BitTorrent, Capistrano and Libvirt
+email: calavera@apache.org
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.md
+- LICENSE
+files: 
+- LICENSE
+- README.md
+- Rakefile
+- examples/kvm-deploy.rb
+- jars/jna-3.0.9.jar
+- jars/libvirt-0.4.6.jar
+- lib/capistrano/recipes/deploy/strategy/marabunta.rb
+- lib/marabunta.rb
+- lib/marabunta/kvm.rb
+- lib/marabunta/kvm_domain.erb
+- lib/marabunta/marabunta.rb
+- marabunta.gemspec
+- spec/marabunta/kvm_spec.rb
+- spec/marabunta/marabunta_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/calavera/marabunta
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: marabunta
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 2
+summary: Cloud nodes deployment using BitTorrent, Capistrano and Libvirt
+test_files: 
+- spec/marabunta/kvm_spec.rb
+- spec/marabunta/marabunta_spec.rb