vagrant-compose 0.2.4 → 0.7.0

data/lib/vagrant/compose/declarative/cluster.rb

@@ -0,0 +1,139 @@
+require 'open4'
+require_relative "../node"
+
+module VagrantPlugins
+  module Compose
+    module Declarative
+
+      class Cluster
+
+        # The name of the cluster
+        attr_reader   :name
+
+        # The default vagrant base box to be used for creating vagrant machines in this cluster.
+        # This setting can be changed at group/node level
+        attr_accessor :box
+
+        # The network domain to wich the cluster belongs (used for computing nodes fqdn)
+        attr_accessor :domain
+
+        # A variable that reflects if vagrant is tageting a single machine, like f.i. when executing vagrant up machine-name
+        attr_reader   :multimachine_filter
+
+        # The root path for ansible playbook; it is used as a base path for computing ansible_group_vars and ansible_host_vars
+        # It defaults to current directory/provisioning
+        attr_accessor :ansible_playbook_path
+
+      	# Costruttore di una istanza di cluster.
+        def initialize()
+        	@multimachine_filter = ""
+        end
+
+        # Implements cluster creation from a playbook file
+	      def from (file)
+	        # calls vagrant-playbook utility for executing the playbook file.
+	        playbook = YAML.load(pycompose (file))
+
+	        # extract cluster attributes
+	        @name   								= playbook.keys[0]
+	        @box    								= playbook[@name]['box']
+	        @domain 								= playbook[@name]['domain']
+	        @ansible_playbook_path  = playbook[@name]['ansible_playbook_path']
+
+          # extract nodes
+          nodes = []
+          playbook[@name]['nodes'].each do |node|
+
+          	boxname = node.keys[0]
+
+            box             = node[boxname]['box']
+            hostname        = node[boxname]['hostname']
+            aliases         = node[boxname]['hostname']
+            fqdn            = node[boxname]['fqdn']
+            ip              = node[boxname]['ip']
+            cpus            = node[boxname]['cpus']
+            memory          = node[boxname]['memory']
+            ansible_groups  = node[boxname]['ansible_groups']
+            attributes      = node[boxname]['attributes']
+            index           = node[boxname]['index']
+            group_index     = node[boxname]['group_index']
+
+            nodes << VagrantPlugins::Compose::Node.new(box, boxname, hostname, fqdn, aliases, ip, cpus, memory, ansible_groups, attributes, index, group_index)
+          end
+
+          # extract ansible inventory, ansible_group_vars, ansible_host_vars
+          ansible_groups = {}
+          if playbook[@name].key?("ansible")
+          	ansible = playbook[@name]['ansible']
+
+          	# extract ansible inventory
+          	ansible_groups = ansible['inventory']
+
+          	# cleanup ansible_group_vars files
+	          # TODO: make safe
+	          ansible_group_vars_path = File.join(@ansible_playbook_path, 'group_vars')
+
+	          FileUtils.mkdir_p(ansible_group_vars_path) unless File.exists?(ansible_group_vars_path)
+	          Dir.foreach(ansible_group_vars_path) {|f| fn = File.join(ansible_group_vars_path, f); File.delete(fn) if f.end_with?(".yml")}
+
+	          #generazione ansible_group_vars file (NB. 1 group = 1 gruppo host ansible)
+	          ansible['group_vars'].each do |group, vars|
+	            # crea il file (se sono state generate delle variabili)
+	            unless vars.empty?
+	              # TODO: make safe
+	              File.open(File.join(ansible_group_vars_path,"#{group}.yml") , 'w+') do |file|
+	                file.puts YAML::dump(vars)
+	              end
+	            end
+	          end
+
+	          # cleanup ansible_host_vars files (NB. 1 nodo = 1 host)
+	          # TODO: make safe
+	          ansible_host_vars_path = File.join(@ansible_playbook_path, 'host_vars')
+
+	          FileUtils.mkdir_p(ansible_host_vars_path) unless File.exists?(ansible_host_vars_path)
+	          Dir.foreach(ansible_host_vars_path) {|f| fn = File.join(ansible_host_vars_path, f); File.delete(fn) if f.end_with?(".yml")}
+
+	          #generazione ansible_host_vars file 
+	          ansible['host_vars'].each do |host, vars|
+	            # crea il file (se sono state generate delle variabili)
+	            unless vars.empty?
+	              # TODO: make safe
+	              File.open(File.join(ansible_host_vars_path,"#{host}.yml") , 'w+') do |file|
+	                file.puts YAML::dump(vars)
+	              end
+	            end
+	          end
+          end
+
+	        return nodes, ansible_groups 
+	      end
+
+
+	      # Executes pycompose command
+	      def pycompose (file)
+	        p_err = ""
+	        p_out = ""
+
+	        begin
+	          p_status = Open4::popen4("vagrant-playbook -f #{file}") do |pid, stdin, stdout, stderr|
+	              p_err = stderr.read.strip
+	              p_out = stdout.read.strip
+	          end
+	        rescue Errno::ENOENT
+	          raise VagrantPlugins::Compose::Errors::PyComposeMissing
+	        rescue Exception => e
+	          raise VagrantPlugins::Compose::Errors::PyComposeError, :message => e.message
+	        end
+
+	        if p_status.exitstatus != 0
+	          raise VagrantPlugins::Compose::Errors::PyComposeError, :message => p_err
+	        end
+
+	        return p_out
+	      end
+
+      end
+    end
+  end
+end

data/lib/vagrant/compose/errors.rb

@@ -30,9 +30,13 @@ module VagrantPlugins
         error_key(:host_var_expression_error)
       end
 
+      class PyComposeMissing < VagrantComposeError
+        error_key(:pycompose_missing)
+      end
+
       class PyComposeError < VagrantComposeError
         error_key(:pycompose_error)
       end
     end
   end
-end
+end

data/lib/vagrant/compose/programmatic/cluster.rb

@@ -0,0 +1,227 @@
+require_relative "node_group"
+
+module VagrantPlugins
+  module Compose
+    module Programmatic
+
+      # This class defines a cluster, thas is a set of group of nodes, where nodes in each group has similar characteristics.
+      # Basically, a cluster is a data structure that can be used as a recipe for setting up and provisioning a
+      # vagrant cluster composed by several machines with different roles.
+      class Cluster
+
+        # The name of the cluster
+        attr_reader   :name
+
+        # The default vagrant base box to be used for creating vagrant machines in this cluster.
+        # This setting can be changed at group/node level
+        attr_accessor :box
+
+        # The network domain to wich the cluster belongs (used for computing nodes fqdn)
+        attr_accessor :domain
+
+        # The root path for ansible playbook; it is used as a base path for computing ansible_group_vars and ansible_host_vars
+        # It defaults to current directory/provisioning
+        attr_accessor :ansible_playbook_path
+
+        # A dictionary, allowing to setup ansible_group_vars generators foreach node_group
+        attr_reader :ansible_group_vars
+
+        # A dictionary, allowing to setup ansible_host_vars generators foreach node_group
+        attr_reader :ansible_host_vars
+
+        # A dictionary, allowing to setup context vars to be uses is value_generators when composing nodes
+        attr_reader :ansible_context_vars
+
+        # Costruttore di una istanza di cluster.
+        def initialize(name)
+          @group_index             = 0
+          @node_groups           = {}
+          @ansible_context_vars  = {}
+          @ansible_group_vars    = {}
+          @ansible_host_vars     = {}
+          @multimachine_filter   = ""
+          @ansible_playbook_path = File.join(Dir.pwd, 'provisioning')
+
+          @name        = name
+          @box         = 'ubuntu/trusty64'
+          @domain      = 'vagrant'
+        end
+
+        # Metodo per la creazione di un gruppo di nodi; in fase di creazione, il blocco inizializza
+        # i valori/le expressioni da utilizzarsi nella valorizzazione degli attributi dei nodi in fase di compose.
+        #
+        # Oltre alla creazione dei nodi, il metodo prevede anche l'esecuzione di un blocco di codice per
+        # la configurazione del gruppo di nodi stesso.
+        def nodes(instances, name, &block)
+          raise RuntimeError, "Nodes #{name} already exists in this cluster." unless not @node_groups.has_key?(name)
+
+          @node_groups[name] = NodeGroup.new(@group_index, instances, name)
+          @node_groups[name].box        = @box
+          @node_groups[name].boxname    = lambda { |group_index, group_name, node_index| return "#{group_name}#{node_index + 1}" }
+          @node_groups[name].hostname   = lambda { |group_index, group_name, node_index| return "#{group_name}#{node_index + 1}" }
+          @node_groups[name].aliases    = []
+          @node_groups[name].ip         = lambda { |group_index, group_name, node_index| return "172.31.#{group_index}.#{100 + node_index + 1}" }
+          @node_groups[name].cpus       = 1
+          @node_groups[name].memory     = 256
+          @node_groups[name].ansible_groups = []
+          @node_groups[name].attributes = {}
+
+          @group_index += 1
+
+          block.call(@node_groups[name]) if block_given?
+        end
+
+        # Prepara il provisioning del cluster
+        def compose
+
+          ## Fase1: Creazione dei nodi
+
+          # sviluppa i vari gruppi di nodi, creando i singoli nodi
+          nodes = []
+
+          @node_groups.each do |key, group|
+            group.compose(@name, @domain, nodes.size) do |node|
+              nodes << node
+            end
+          end
+
+          # sviluppa i gruppi abbinando a ciascono i nodi creati
+          # NB. tiene in considerazione anche l'eventualità che un gruppo possa essere composto da nodi appartenenti a diversi node_groups
+          ansible_groups= {}
+          nodes.each do |node|
+            node.ansible_groups.each do |ansible_group|
+              ansible_groups[ansible_group] = [] unless ansible_groups.has_key? (ansible_group)
+              ansible_groups[ansible_group] << node
+            end
+          end
+
+          ## Fase2: Configurazione provisioning del cluster via Ansible
+          # Ogni nodo diventerà una vm su cui sarà fatto il provisioning, ovvero un host nell'inventory di ansible
+          # Ad ogni gruppo corrispondono nodi con caratteristiche simili
+
+          # genearazione inventory file per ansible, aka ansible_groups in Vagrant (NB. 1 group = 1 gruppo ansible)
+          ansible_groups_provision = {}
+          ansible_groups.each do |ansible_group, ansible_group_nodes|
+            ansible_groups_provision[ansible_group] = []
+            ansible_group_nodes.each do |node|
+              ansible_groups_provision[ansible_group] << {"boxname" => node.boxname , "hostname" => node.hostname}
+            end
+          end
+
+          # Oltre alla creazione del file di inventory per ansible, contenente gruppi e host, è supportata:
+          # - la creazione di file ansible_group_vars, ovvero di file preposti a contenere una serie di variabili - specifico di ogni gruppo di host  -
+          #   per condizionare il provisioning ansible sulla base delle caratteristiche del cluster specifico
+          # - la creazione di file ansible_host_vars, ovvero di file preposti a contenere una serie di variabili - specifico di ogni host -
+          #   per condizionare il provisioning ansible sulla base delle caratteristiche del cluster specifico
+
+          # La generazione delle variabili utilizza una serie di VariableProvisioner, uno o più d'uno per ogni gruppo di hosts, configurati durante la
+          # definizione del cluster.
+
+          context = {}
+
+          #genearazione context (NB. 1 group = 1 gruppo host ansible)
+          ansible_groups.each do |ansible_group, ansible_group_nodes|
+            # genero le variabili per il group
+            provisioners = @ansible_context_vars[ansible_group]
+            unless provisioners.nil?
+
+              # se necessario, normalizzo provisioner in array provisioners
+              provisioners = [ provisioners ] if not provisioners.respond_to?('each')
+              # per tutti i provisioners abbinati al ruolo
+              provisioners.each do |provisioner|
+                begin
+                  vars = provisioner.call(context, ansible_group_nodes)
+
+                  #TODO: gestire conflitto (n>=2 gruppi che generano la stessa variabile - con valori diversi)
+                  context = context.merge(vars)
+                rescue Exception => e
+                  raise VagrantPlugins::Compose::Errors::ContextVarExpressionError, :message => e.message, :ansible_group => ansible_group
+                end
+              end
+            end
+          end
+
+          # cleanup ansible_group_vars files
+          # TODO: make safe
+          ansible_group_vars_path = File.join(@ansible_playbook_path, 'group_vars')
+
+          FileUtils.mkdir_p(ansible_group_vars_path) unless File.exists?(ansible_group_vars_path)
+          Dir.foreach(ansible_group_vars_path) {|f| fn = File.join(ansible_group_vars_path, f); File.delete(fn) if f.end_with?(".yml")}
+
+          #generazione ansible_group_vars file (NB. 1 group = 1 gruppo host ansible)
+          ansible_groups.each do |ansible_group, ansible_group_nodes|
+            ansible_group_vars = {}
+            # genero le variabili per il group
+            provisioners = @ansible_group_vars[ansible_group]
+            unless provisioners.nil?
+              # se necessario, normalizzo provisioner in array provisioners
+              provisioners = [ provisioners ] if not provisioners.respond_to?('each')
+              # per tutti i provisioners abbinati al ruolo
+              provisioners.each do |provisioner|
+                begin
+                  vars = provisioner.call(context, ansible_group_nodes)
+
+                  #TODO: gestire conflitto (n>=2 gruppi che generano la stessa variabile - con valori diversi)
+                  ansible_group_vars = ansible_group_vars.merge(vars)
+                rescue Exception => e
+                  raise VagrantPlugins::Compose::Errors::GroupVarExpressionError, :message => e.message, :ansible_group => ansible_group
+                end
+              end
+            end
+
+            # crea il file (se sono state generate delle variabili)
+            unless ansible_group_vars.empty?
+              # TODO: make safe
+              File.open(File.join(ansible_group_vars_path,"#{ansible_group}.yml") , 'w+') do |file|
+                file.puts YAML::dump(ansible_group_vars)
+              end
+            end
+          end
+
+          # cleanup ansible_host_vars files (NB. 1 nodo = 1 host)
+          # TODO: make safe
+          ansible_host_vars_path = File.join(@ansible_playbook_path, 'host_vars')
+
+          FileUtils.mkdir_p(ansible_host_vars_path) unless File.exists?(ansible_host_vars_path)
+          Dir.foreach(ansible_host_vars_path) {|f| fn = File.join(ansible_host_vars_path, f); File.delete(fn) if f.end_with?(".yml")}
+
+          #generazione ansible_host_vars file
+          nodes.each do |node|
+            # genero le variabili per il nodo; il nodo, può essere abbinato a diversi gruppi
+            ansible_host_vars = {}
+            node.ansible_groups.each do |ansible_group|
+              # genero le variabili per il gruppo
+              provisioners = @ansible_host_vars[ansible_group]
+              unless provisioners.nil?
+                # se necessario, normalizzo provisioner in array provisioners
+                provisioners = [ provisioners ] if not provisioners.respond_to?('each')
+                # per tutti i provisioners abbinati al gruppo
+                provisioners.each do |provisioner|
+                  begin
+                    vars = provisioner.call(context, node)
+
+                    #TODO: gestire conflitto (n>=2 gruppi che generano la stessa variabile - con valori diversi)
+                    ansible_host_vars = ansible_host_vars.merge(vars)
+                  rescue Exception => e
+                    raise VagrantPlugins::Compose::Errors::HostVarExpressionError, :message => e.message, :host => node.hostname, :ansible_group => ansible_group
+                  end
+                end
+              end
+            end
+
+            # crea il file (se sono state generate delle variabili)
+            unless ansible_host_vars.empty?
+              # TODO: make safe
+              File.open(File.join(ansible_host_vars_path,"#{node.hostname}.yml") , 'w+') do |file|
+                file.puts YAML::dump(ansible_host_vars)
+              end
+            end
+          end
+
+          return nodes, ansible_groups_provision
+        end
+      end
+
+    end
+  end
+end

data/lib/vagrant/compose/programmatic/node_group.rb

@@ -0,0 +1,104 @@
+require_relative "../node"
+
+module VagrantPlugins
+  module Compose
+    module Programmatic
+
+      # This class defines a group of nodes, representig a set of vagrant machines with similar characteristics.
+      # Nodes will be composed by NodeGroup.compose method, according with the configuration of values/value_generator
+      # of the group of node itself.
+      class NodeGroup
+
+        # A number identifying the group of nodes withing the cluster.
+        attr_reader :index
+
+        # The name of the group of nodes
+        attr_reader :name
+
+        # The number of nodes/instances to be created in the group of nodes.
+        attr_reader :instances
+
+        # The value/value generator to be used for assigning to each node in this group a vagrant base box to be used for creating vagrant machines implementing nodes in this group.
+        attr_accessor :box
+
+        # The value/value generator to be used for assigning to each node in this group a box name a.k.a. the name for the machine in VirtualBox/VMware console.
+        attr_accessor :boxname
+
+        # The value/value generator to be used for assigning to each node in this group a unique hostname
+        attr_accessor :hostname
+
+        # The value/value generator to be used for assigning to each node in this group a unique list of aliases a.k.a. alternative host names
+        attr_accessor :aliases
+
+        # The value/value generator to be used for assigning to each node in this groupa unique ip
+        attr_accessor :ip
+
+        # The value/value generator to be used for assigning to each node in this group cpus
+        attr_accessor :cpus
+
+        # The value/value generator to be used for assigning to each node in this group memory
+        attr_accessor :memory
+
+        # The value/value generator to be used for assigning each node in this group to a list of ansible groups
+        attr_accessor :ansible_groups
+
+        # The value/value generator to be used for assigning a dictionary with custom attributes - Hash(String, obj) - to each node in this group.
+        attr_accessor :attributes
+
+        def initialize(index, instances, name)
+          @index  = index
+          @name = name
+          @instances = instances
+        end
+
+        # Composes the group of nodes, by creating the required number of nodes
+        # in accordance with values/value_generators.
+        # Additionally, some "embedded" trasformation will be applied to attributes (boxname, hostname) and
+        # some "autogenerated" node properties will be computed (fqdn).
+        def compose(cluster_name, cluster_domain, cluster_offset)
+          node_index = 0
+          while node_index < @instances
+            box            = generate(:box, @box, node_index)
+            boxname        = maybe_prefix(cluster_name,
+                                          "#{generate(:boxname, @boxname, node_index)}")
+            hostname       = maybe_prefix(cluster_name,
+                                          "#{generate(:hostname, @hostname, node_index)}")
+            aliases        = generate(:aliases, @aliases, node_index).join(',')
+            fqdn           = cluster_domain.empty? ? "#{hostname}" : "#{hostname}.#{cluster_domain}"
+            ip             = generate(:ip, @ip, node_index)
+            cpus           = generate(:cpus, @cpus, node_index)
+            memory         = generate(:memory, @memory, node_index)
+            ansible_groups = generate(:ansible_groups, @ansible_groups, node_index)
+            attributes     = generate(:attributes, @attributes, node_index)
+            yield VagrantPlugins::Compose::Node.new(box, boxname, hostname, fqdn, aliases, ip, cpus, memory, ansible_groups, attributes, cluster_offset + node_index, node_index)
+
+            node_index += 1
+          end
+        end
+
+        # utility function for concatenating cluster name (if present) to boxname/hostname
+        def maybe_prefix(cluster_name, name)
+          if cluster_name && cluster_name.length > 0
+            "#{cluster_name}-" + name
+          else
+            name
+          end
+        end
+
+        # utility function for resolving value/value generators
+        def generate(var, generator, node_index)
+          unless generator.respond_to? :call
+            return generator
+          else
+            begin
+              return generator.call(@index, @name, node_index)
+            rescue Exception => e
+              raise VagrantPlugins::Compose::Errors::AttributeExpressionError, :message => e.message, :attribute => var, :node_index => node_index, :node_group_name => name
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/vagrant/compose/version.rb

@@ -1,5 +1,5 @@
 module Vagrant
   module Compose
-    VERSION = "0.2.4"
+    VERSION = "0.7.0"
   end
 end

data/vagrant-compose.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |spec|
   spec.license       = "MIT"
   spec.authors       = ["Fabrizio Pandini"]
   spec.email         = ["fabrizio.pandini@unicredit.eu"]
-  spec.homepage      = "https://github.com/fabriziopandini/vagrant-compose/"  
+  spec.homepage      = "https://github.com/fabriziopandini/vagrant-compose/"
   spec.summary       = %q{A Vagrant plugin that helps building complex multi-machine scenarios.}
   spec.description   = %q{A Vagrant plugin that helps building complex multi-machine scenarios.}
 
@@ -17,6 +17,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 10.4.2"
   spec.add_development_dependency "rspec", "~> 3.4.0"
   spec.add_development_dependency "rspec-its", "~> 1.2.0"
+  spec.add_dependency "open4", "~> 1.3.4"
 
   # The following block of code determines the files that should be included
   # in the gem. It does this by reading all the files in the directory where
@@ -48,7 +49,7 @@ Gem::Specification.new do |spec|
         File.fnmatch(ignore, File.basename(file), File::FNM_PATHNAME)
     end
   end
- 
+
   spec.files         = unignored_files
   spec.executables   = unignored_files.map { |f| f[/^bin\/(.*)/, 1] }.compact
   spec.require_path  = 'lib'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vagrant-compose
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.7.0
 platform: ruby
 authors:
 - Fabrizio Pandini
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-26 00:00:00.000000000 Z
+date: 2016-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -52,6 +52,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: open4
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.4
 description: A Vagrant plugin that helps building complex multi-machine scenarios.
 email:
 - fabrizio.pandini@unicredit.eu
@@ -64,14 +78,17 @@ files:
 - bin/console
 - bin/setup
 - CHANGELOG.md
+- doc/declarative.md
+- doc/programmatic.md
 - Gemfile
 - lib/locales/en.yml
 - lib/vagrant/compose/config.rb
+- lib/vagrant/compose/declarative/cluster.rb
 - lib/vagrant/compose/errors.rb
+- lib/vagrant/compose/node.rb
 - lib/vagrant/compose/plugin.rb
-- lib/vagrant/compose/util/cluster.rb
-- lib/vagrant/compose/util/node.rb
-- lib/vagrant/compose/util/node_group.rb
+- lib/vagrant/compose/programmatic/cluster.rb
+- lib/vagrant/compose/programmatic/node_group.rb
 - lib/vagrant/compose/version.rb
 - lib/vagrant/compose.rb
 - LICENCE