PackerFiles 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ed4d8bf7590cae12aeecfbea9800a7a74de39e43
+  data.tar.gz: 9189448e12badff0acb1fb2a8300f5d5fdd77d17
+SHA512:
+  metadata.gz: 4117c5b57a67a5f9e1ec59072dc79038bbb0edfa45d4462eee2acd5a303217fb6bf1c6d739edccd1428c0336dc21ac90581b46a6e3d7ee61e28aca7eede62547
+  data.tar.gz: a4f3c5f76406dc249f1a0c7bdd7c4e392f4ee554d2c3d0951b2139cecebfecb145e0091fc82d1217ac7d58972266dae73ffab58a213069e85f176f5b0a29fe0e

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in PackerFiles.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Anand Venkatanarayanan
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/PackerFiles.gemspec

@@ -0,0 +1,36 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'PackerFiles/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "PackerFiles"
+  spec.version       = PackerFiles::VERSION
+  spec.authors       = ["Anand Venkatanarayanan"]
+  spec.email         = ["external_mails_anandv-github@yahoo.co.in"]
+  spec.description   = %q{Generates Packer files based on a simple spec. file for certain configurations.}
+  spec.summary       = %q{Generates Packer files based on a simple spec. file for certain configurations.}
+  spec.homepage      = "https://github.com/anandvenkatanarayanan/PackerFiles/"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib", 
+                        File.join("lib", spec.name), 
+                        File.join("lib", spec.name, "OS")
+                       ]
+
+  # Development dependencies
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec", "~> 2.6"
+  
+  # Run Time dependencies
+  spec.add_runtime_dependency 'json'
+  spec.add_runtime_dependency 'erubis'
+  spec.add_runtime_dependency 'thor'
+  spec.add_runtime_dependency 'nokogiri'
+
+end
+

data/README.md

@@ -0,0 +1,32 @@
+# PackerFiles
+
+A Ruby library that can generate various files required for packer to work in a consistent manner without having to manually edit those files. It is mostly used for creating full fledged VM Images that run on top of a hypervisor and not for cloud environments like Amazon or Digital ocean.
+
+An approximate specification for Packer File generator is in PackerFile.rb file at the root directory
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'PackerFiles'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install PackerFiles
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,21 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+# Create Rake Task for running all the test cases
+RSpec::Core::RakeTask.new('test/all') do |test|
+end
+
+# Create Rake Task for running all the Core Test cases
+RSpec::Core::RakeTask.new('test/core') do |test|
+  test.pattern = 'spec/core/*_spec.rb'
+end
+
+# Create Rake Task for running all the OS Test cases
+RSpec::Core::RakeTask.new('test/os') do |test|
+  test.pattern = 'spec/os/**/**_spec.rb'
+end
+
+# Create Rake Task for running all the Hypervisor Test cases
+RSpec::Core::RakeTask.new('test/virtual') do |test|
+  test.pattern = 'spec/virtual/**/**_spec.rb'
+end

data/bin/pkf

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+# Offload all calls to CLI
+require 'CLI/Root'
+PackerFiles::CLI::Root.start(ARGV)

data/lib/PackerFiles/CLI/OS.rb

@@ -0,0 +1,44 @@
+# Requires
+require 'thor'
+require 'OS/Finder'
+require 'OS/DocGenerator'
+
+# Implement the CLI sub-commands for OS.
+module PackerFiles
+ module CLI
+   class OS < Thor
+
+      # List all supported Operating systems.
+      desc "list", "list all supported operating systems"
+      def list
+	Finder.new.class_names.each do |cl|
+	   name = cl.name.gsub('PackerFiles::','')
+	   if cl.versions.empty?
+	      versions = 'Any'
+	   else
+	      versions = cl.versions.to_s
+	   end
+	   puts "#{name} => #{versions}"
+	end
+      end
+
+      # Generate the documentation needed for a given class name
+      desc "init", "Create a Template file for the specified OS"
+      option :name,  :required => true,  :desc => 'OS Name', :type => :string
+      option :force, :desc    => 'Overwrite existing File, if any', :type => :boolean,
+                     :default => false
+      option :file, :desc => 'Output file name', :default => 'PackerFile',
+                    :type => :string 
+      def init
+         klass = Finder.new.class_name(options[:name])
+	 if (File.exist?(options[:file]) && !options[:force])
+            $stderr.puts "Cannot overwrite #{options[:file]} w/o --force"
+	    return 1
+         end
+         DocGenerator.new(klass, options[:file]).Generate
+      end 
+
+
+   end
+ end
+end

data/lib/PackerFiles/CLI/Root.rb

@@ -0,0 +1,41 @@
+# Requires
+require 'thor'
+require_relative 'OS'
+
+# The ROOT of all the CLI commands. Other Sub commands are
+# declared as they go.
+module PackerFiles
+  module CLI
+     class Root < Thor
+
+        # define sub-command for OS
+        desc "os SUBCOMMAND ...ARGS", "Commands on Supported OS"
+        subcommand "os", OS
+
+        # define the default task as generate
+        default_task :generate
+
+        # Default command that generates various artificats for Packer to
+        # work correctly.
+        desc "generate", "Generate Artificats for packer to work correctly"
+        option :defaults, :desc    => 'Use Defaults for missing values', 
+                          :type    => :boolean, 
+                          :default => false
+        option :file,     :desc => 'OS Spec. File', :default => 'PackerFile',
+                          :type => :string 
+        def generate
+	   if !File.exist?options[:file]
+	      puts "File not found #{options[:file]}"
+	      exit(1)
+           end
+	   begin
+              load(options[:file])
+           rescue PackerFiles::Core::Error => e
+              puts "Error: #{e.message}"
+              exit(1)
+           end
+        end
+
+     end
+  end
+end

data/lib/PackerFiles/Core/Base.rb

@@ -0,0 +1,23 @@
+
+# Requires
+require 'PackerFiles'
+
+# Top level Module for this gem.
+module PackerFiles
+   
+   # Contains the Core classes/methods and everthing else.
+   module Core
+
+      # The Base class which defines interfaces that all PackerFile
+      # classes must adhere to
+      class Base
+   
+         # Do nothing. 
+         def normalize; end
+   
+         # Return the name of the documentation file, for this class.
+         def self.doc_file; end
+      end
+
+   end # Core
+end # PackerFiles

data/lib/PackerFiles/Core/CDImage.rb

@@ -0,0 +1,185 @@
+# Requires
+require 'open-uri'
+require 'uri'
+require 'Core/Base'
+require 'Core/Exceptions'
+require 'Utils/FastestURL'
+
+module PackerFiles
+
+   module Core
+
+      # Base class for CDImage implmentation
+      class CDImageImpl
+
+	 # Return back the ISO name (It can be a regex pattern)
+	 def get_iso_name_pattern
+	    '.*'
+	 end
+
+	 # Return back the Checksum File Pattern (It can be a regex pattern)
+	 def get_check_sum_pattern
+	    'MD5SUM'
+	 end
+    
+	 # Return back the Checksum File Pattern (It can be a regex pattern)
+	 def get_check_sum_type
+	    'md5'
+	 end
+
+	 # Return the list of index URLs.
+	 def get_index_urls
+	    []
+	 end
+      end
+
+      # Used for figuring out the URLs and Checksum values for the installer images
+      # of debian.
+      class CDImage < Base
+      
+	# Accessors that can be set.
+	attr_accessor :arch
+	attr_accessor :release
+	attr_accessor :iso_name_pattern
+	attr_accessor :check_sum_pattern
+	attr_accessor :check_sum_type
+       
+	# Accessors that can be derived if not set 
+	attr_accessor :index_urls
+	attr_accessor :iso_url
+	attr_accessor :check_sum
+
+	# Implementation class for CD Image.
+	attr_writer   :impl
+
+	# Documentation for this class
+	def self.doc_file
+	   PackerFiles.DirPath('Core/example/CDImage.txt').first
+	end
+
+	# Constructor that yields self for accessor settings.
+	def initialize
+	   yield self if block_given?
+	end
+
+	# Overriden normalize
+	def normalize
+	  
+	   # Do some basic checks.
+	   raise NilException.new(self, 'arch')    if @arch.nil?
+	   raise NilException.new(self, 'release') if @release.nil?
+
+	   # Get everything else.
+	   set_iso_url_and_checksum
+	end
+
+	# Compute the ISO URL and Check SUM for the CD Image, but only if it is
+	# not defined already.
+	private
+	def set_iso_url_and_checksum
+	   if @iso_url.nil? || @check_sum.nil?
+	      set_iso_name_checksum_pattern
+	      index_url, valid   = fastest_url
+	      if (valid)
+		(@iso_url, ck_url) = get_iso_checksum_url(index_url)
+		@check_sum         = get_check_sum(ck_url)
+	      else
+		@iso_url        = URI::join(index_url, @iso_name_pattern).to_s
+		@check_sum_type = 'none'
+		@check_sum      = ''
+	      end
+	   end
+	   raise NilException.new(self, 'check_sum_type')    if @check_sum_type.nil?
+	end
+
+	# Compute the ISO name of the CD Image from other variables, but only
+	# if it is not defined already.
+	def set_iso_name_checksum_pattern
+	   if @iso_name_pattern.nil? && !@impl.nil?
+	      name              = @impl.get_iso_name_pattern 
+	      @iso_name_pattern = name.gsub('@release@', @release).gsub('@arch@', @arch)
+	   end
+	   if @check_sum_pattern.nil? && !@impl.nil?
+	      name      = @impl.get_check_sum_pattern
+	      @check_sum_pattern = name.gsub('@release@', @release).gsub('@arch@', @arch)
+	   end
+	   if @check_sum_type.nil? && !@impl.nil?
+	      @check_sum_type    = @impl.get_check_sum_type
+	   end
+	   
+	   # Raise more errors if implementation does not fill values.
+	   raise NilException.new(self, 'iso_name_pattern')  if @iso_name_pattern.nil?
+	   raise NilException.new(self, 'check_sum_pattern') if @check_sum_pattern.nil?
+	   raise NilException.new(self, 'check_sum_type')    if @check_sum_type.nil?
+	end
+
+	# Return the MD5 Checksum for the ISO URL
+	def get_check_sum(ck_url)
+	  name = URI::parse(@iso_url).path.split('/').last
+	  begin
+	     open(ck_url) {|f|
+		f.each_line do |line|
+		   if match = line.match(name)
+		      return line.split(' ').first
+		   end
+		end
+	     }
+	  rescue OpenURI::HTTPError => e
+	     raise URLException.new(ck_url, e)
+	  end
+	end
+
+	# Return the full URL for the CD Image.
+	private
+	def get_iso_checksum_url(index_url)
+	  iso_regex = "href=\"(#{@iso_name_pattern})\""
+	  ck_regex  = "href=\"(#{@check_sum_pattern})\""
+	  iso_url   = ''
+	  ck_url    = ''	
+	  open(index_url) {|f|
+	     f.each_line do |line|
+	       if match = line.match(iso_regex)
+		  iso_url = URI::join(index_url, match[1]).to_s
+	       end
+	       if match = line.match(ck_regex)
+		  ck_url = URI::join(index_url, match[1]).to_s
+	       end
+	     end
+	  }
+	  raise "Cannot find file matching #{@iso_name_pattern}"  if iso_url.empty?
+	  raise "Cannot find file matching #{@check_sum_pattern}" if ck_url.empty?
+	  return iso_url, ck_url
+	end
+
+	# Get the Fastest Index URL from the list of curated index URLs
+	def fastest_url
+
+	   # Slow URL = 5 seconds latency with a trial = 1
+	   f    = Utils::FastestURL.new(5.0, 1)
+	   urls = curated_index_urls
+	   raise EmptyArrayException.new(self, 'index_urls') if urls.empty?
+	   all = f.best_urls(urls)
+	   if all.empty?
+	      return urls.first, false
+	   else
+	      return all.flatten.first, true
+	   end
+	end
+
+	# Curate the URL list by replacing '@release@' with @release and '@arch'
+	# with @arch.
+	def curated_index_urls
+	   @index_urls ||= @impl.index_urls if !@impl.nil?
+	   raise NilException.new(self, 'index_urls')        if @index_urls.nil?
+	   raise EmptyArrayException.new(self, 'index_urls') if @index_urls.empty?
+	   @index_urls.map! {|url|
+	      url.gsub('@release@', @release).gsub('@arch@', @arch)
+	   }
+	end
+
+      end 
+
+      
+   end # Core
+
+end # PackerFiles

data/lib/PackerFiles/Core/Disk.rb

@@ -0,0 +1,135 @@
+
+# Requires
+require 'Utils/Size'
+require_relative 'Base'
+require_relative 'Partition'
+require_relative 'VolumeGroup'
+require_relative 'Exceptions'
+
+module PackerFiles
+
+ module Core
+
+    # Define the Disk class that can handle disks specified in a Packerfile.
+    # Just the barebones attributes that are required for this class are specified
+    # here. The conversion of these attributes into a OS build specific file is 
+    # done by derived classes in the OS specific directories.
+    class Disk < Base
+
+      # Include MiB as a method inside Disk
+      include Utils::Size
+
+      # Specify attributes
+      attr_accessor :name  
+      attr_accessor :size
+      attr_accessor :partitions
+      attr_accessor :vg
+
+      # Documentation for this class 
+      def self.doc_file
+	 PackerFiles.DirPath('Core/example/Disk.txt').first
+      end
+
+      # Constructor to just specify accessor varibales
+      def initialize
+	@partitions = []
+	yield self if block_given?
+      end
+
+      # Tells if the disk is a raw disk that contains no partitions
+      # and Volume group
+      def is_raw?
+	 return true if @partitions.empty? && @vg.nil?
+	 false
+      end
+
+      # Tells if the disk has only custom partition modifiers and no 
+      # volume group.
+      def only_partitions?
+	return true if !@partitions.empty? && @vg.nil?
+	false      
+      end
+    
+      # Tells if there is just a custom VG specification on a
+      # disk w/o any partitions.
+      def only_vg?
+	return true if @partitions.empty? && !@vg.nil?
+	false
+      end
+
+      # Tells if there is both a custom VG specification on a 
+      # disk and partitions.
+      def vg_with_partitions?
+	return true if !@partitions.empty? && !@vg.nil?
+      end
+
+      # Adds a volume group to this disk by doing an instance-eval
+      def VolumeGroup(&block)
+	 if block_given?
+	   vg = VolumeGroup.new
+	   vg.instance_eval &block
+	   @vg = vg
+	 end
+      end
+
+      # Adds a Partition on this disk by doing an instance-eval
+      def Partition(&block)
+	 if block_given?
+	   part = Partition.new
+	   part.instance_eval &block
+	   if (part.name.nil?)
+	      part.name = @partitions.size
+	   end
+	   @partitions.push(part)
+	 end
+      end
+
+      # Normalize the various values into something useful. Usually 
+      # this means adding UTF-8 into the default string. if desired
+      def normalize
+	 raise NilException.new(self, 'name')       if @name.nil?
+	 raise NilException.new(self, 'size')       if @size.nil?
+	 raise NilException.new(self, 'partitions') if @partitions.nil?
+
+	 # Convert given size into MiB
+	 @size = MiB(@size)
+
+	 # Re-create the object by converting the given size into MiB
+	 @partitions.each {|p|
+	    p.size = MiB(p.size)
+	    p.normalize
+	 }
+
+	 # Add all partition sizes and if they exceed disk size, cry...
+	 part_size = @partitions.inject(0) {|val, p| val + p.size}
+	 raise SizeException.new(@partitions.first, self) if (@size < part_size)
+
+	 # Normalize the embedded VG, if it is present
+	 return if @vg.nil?
+	 @vg.normalize
+     
+	 # Calculate sum of all LVs
+	 lv_size = @vg.lvs.inject(0) { |val, lv| val + lv.size }
+
+	 # Calculate the size of the Physical Volumes. It is either
+	 # the whole disk or the list of partitions that match the
+	 # disk names
+	 if @partitions.nil? || @partitions.empty?
+	   total_size = @size
+	 else
+	   total_size = @vg.disk_names.inject(0) {|val,name|
+	      match = @partitions.select {|part| (part.name.to_s == name.to_s) }
+	      raise "Disk name #{name} doesn't match partition names" if match.empty?
+	      raise "Multiple matches for #{name} in Partitions" if match.size > 1
+	      val + match.first.size
+	   }
+	 end
+	 raise SizeException.new(@vg.lvs.first, @vg) if lv_size > total_size
+      end
+
+    end # Disk
+
+ end # Core
+
+end # PackerFiles
+

data/lib/PackerFiles/Core/Exceptions.rb

@@ -0,0 +1,65 @@
+# Requires
+
+
+# Exceptions generated by the PackerFile generator.
+module PackerFiles
+
+   module Core
+
+      # The top level class for all exceptions shown by this gem.
+      class Error   < Exception
+	 attr_accessor :message
+      end
+    
+      # The Nil exception is generated when a particular class member is nil. 
+      class NilException < Error
+
+	 # Constructor. 
+	 def initialize(obj, member_name)
+	    @super
+	    class_name   = obj.class.name.gsub(/.*::/, '')
+	    self.message = "#{class_name}.#{member_name} is not set"
+	 end
+      end
+
+      # The EmptyArray Exception is generated when a particular class member array
+      # is empty.
+      class EmptyArrayException < Error
+
+	 # Constructor. 
+	 def initialize(obj, member_name)
+	    @super
+	    class_name   = obj.class.name.gsub(/.*::/, '')
+	    self.message = "#{class_name}.#{member_name} is empty" 
+	 end
+      end
+
+      # The Size Exception is generated when child object sizes are greater than
+      # that of Parent objects.
+      class SizeException < Error
+
+	 # Constructor. 
+	 def initialize(parent_obj, child_obj)
+	    @super
+	    parent  = parent_obj.class.name.gsub(/.*::/, '')
+	    child   = child_obj.class.name.gsub(/.*::/, '')
+	    self.message = "Sum of #{child} sizes are greater than #{parent} size"
+	 end
+      end
+
+     # The URL Exception is generated if a URL cannot be opened and it's contents
+     # cannot be read.
+     class URLException < Error
+
+	# Constructor
+	def initialize(url, inner_obj)
+	   @super
+	   self.message = "URL #{url} could not be opened! HTTP Error: #{inner_obj.to_s}"
+	end
+
+     end
+
+   end  # Core
+
+end # PackerFiles
+