knife-stackbuilder 0.5.2

data/lib/stackbuilder/stack/node_task.rb

@@ -0,0 +1,424 @@
+# Copyright (c) 2014 Mevan Samaratunga
+
+include StackBuilder::Common::Helpers
+
+module StackBuilder::Stack
+
+    class NodeTask
+        
+        attr_reader :name
+
+        attr_accessor :scale
+        attr_accessor :prev_scale
+        attr_accessor :sync
+
+        attr_accessor :deleted
+        
+        attr_reader :counter
+        attr_reader :parent_nodes
+        attr_reader :child_nodes
+
+        attr_reader :resource_sync
+        attr_reader :manager
+
+        SYNC_NONE  = 0  # All node instances processed asynchronously
+        SYNC_FIRST = 1  # First node instance is processed synchronously and the rest asynchronously
+        SYNC_ALL   = 2  # All node instances are processed synchronously
+        
+        def initialize(manager, nodes, node_config, id)
+            
+            @logger = StackBuilder::Common::Config.logger
+
+            @manager = manager
+
+            @id = id
+            @nodes = nodes
+            @parent_nodes = [ ]
+            @child_nodes = [ ]
+            @counter = 0
+            
+            @name = node_config['node']
+            @attributes = (node_config.has_key?('attributes') ? node_config['attributes'] : { })
+
+            case node_config['sync']
+                when "first"
+                    @sync = SYNC_FIRST
+                when "all"
+                    @sync = SYNC_ALL
+                else
+                    @sync = SYNC_NONE
+            end
+
+            if node_config.has_key?('targets')
+
+                @logger.warn("Ignoring 'scale' attribute for '#{@name}' as that node has targets.") \
+                    if node_config.has_key?("scale")
+
+                @scale = 0
+            else
+                current_scale = manager.get_scale
+                if current_scale==0
+                    @scale = (node_config.has_key?("scale") ? node_config["scale"] : 1)
+                else
+                    @scale = current_scale
+                end
+
+                raise ArgumentError, "The scale for node \"#{@name}\" must be greater than 0." if @scale < 1
+            end
+            @prev_scale = @scale
+
+            @targets = [ ]
+
+            @node_mutex = Mutex.new
+            @resource_sync = [ ]
+
+            @deleted = false
+        end
+
+        def add_dependency(node_name, is_target = false)
+
+            node = @nodes[node_name]
+
+            @targets << node if is_target
+
+            node.parent_nodes << self unless node.parent_nodes.include?(self)
+            self.child_nodes << node unless self.child_nodes.include?(node)
+        end
+
+        def process_attribute_dependencies
+
+            @attributes.each_value do |v|
+
+                if v =~ /^nodes\[.*\]$/
+
+                    lookup_keys = v.split(/[\[\]]/).reject { |l| l == "nodes" || l.empty? }
+                    add_dependency(lookup_keys.shift)
+                end
+            end
+        end
+
+        def init_dependency_count(count = nil)
+
+            if count.nil?
+                @counter = child_nodes.size
+            else
+                @counter += count
+            end
+
+            @counter
+        end
+        
+        def dec_dependency_count
+            @node_mutex.synchronize {
+                @counter -= 1
+                return @counter
+            }
+        end
+
+        def prepare
+
+            threads = [ ]
+
+            if @targets.empty?
+
+                # You need to prepare nodes only if this node task
+                # is the target. i.e. no referenced targets
+
+                current_scale = @manager.get_scale
+
+                @resource_sync.size.step(current_scale - 1) do |i|
+                    @resource_sync[i] ||= StackBuilder::Common::Semaphore.new
+                    @resource_sync[i].signal
+                end
+
+                if current_scale>@scale
+
+                    @logger.debug("Scaling #{self} from #{current_scale} down to #{@scale}")
+
+                    # Scale Down
+
+                    delete_events = Set.new([ "stop", "uninstall" ])
+                    @scale.step(current_scale - 1).to_a.each do |i|
+
+                        resource_sync = @resource_sync[i]
+                        resource_sync.wait
+
+                        threads << Thread.new {
+
+                            begin
+                                @logger.debug("Deleting #{self} #{i}.")
+                                $stdout.printf("Deleting node resource '%s[%d]'.\n",
+                                    @name, i) unless @logger.debug?
+
+                                @manager.process(i, delete_events, parse_attributes(@attributes, i))
+                                @manager.delete(i)
+
+                            rescue Exception => msg
+
+                                puts("Fatal Error: #{msg}")
+                                @logger.debug(msg.backtrace.join("\n\t"))
+
+                                raise StackBuilder::Common::StackDeleteError,
+                                    "Deleting node resource '#{name}[{i}]' " +
+                                    "terminated with an error: #{msg}"
+                            ensure
+                                resource_sync.signal
+                            end
+                        }
+                    end
+
+                    (current_scale - 1).downto(@scale) do |i|
+                        @resource_sync.delete_at(i)
+                    end
+                end
+
+                if @scale>current_scale && !@deleted
+
+                    @logger.debug("Scaling #{self} from #{current_scale} up to #{@scale}")
+
+                    # Scale up
+
+                    current_scale.step(@scale - 1) do |i|
+
+                        sync = StackBuilder::Common::Semaphore.new
+                        @resource_sync[i] = sync
+
+                        threads << Thread.new {
+
+                            begin
+                                @logger.debug("Creating #{self} #{i}.")
+                                $stdout.printf( "Creating node resource '%s[%d]'.\n",
+                                    @name, i) unless @logger.debug?
+
+                                @manager.create(i)
+
+                            rescue Exception => msg
+
+                                puts("Fatal Error: #{msg}")
+                                @logger.debug(msg.backtrace.join("\n\t"))
+
+                                raise StackBuilder::Common::StackCreateError,
+                                    "Creating node resource '#{name}[#{i}]' " +
+                                    "terminated with an error: #{msg}"
+                            ensure
+                                @resource_sync[i].signal
+                            end
+                        }
+                    end
+                end
+
+                @prev_scale = current_scale
+                @manager.set_scale(@scale)
+            end
+
+            threads
+        end
+
+        def orchestrate(events)
+
+            threads = [ ]
+
+            scale = (@deleted ? @manager.get_scale : @scale)
+            if scale > 0
+
+                if @sync == "first"
+                    @manager.process(scale, events, self.parse_attributes(@attributes, 0))
+                    scale -= 1
+                end
+
+                if @sync == "all"
+                    scale.times do |i|
+                        @manager.process(i, events, self.parse_attributes(@attributes, i))
+                    end
+                else
+                    scale.times do |i|
+                        spawn_processing(i, events, threads)
+                    end
+                end
+
+            elsif !@targets.empty?
+
+                @targets.each do |t|
+                    t.scale.times do |i|
+                        spawn_processing(i, events, threads, t)
+                    end
+                end
+            end
+
+            threads.each { |t| t.join }
+
+            executable_parents = [ ]
+            parent_nodes.each do |p|
+                executable_parents << p if p.dec_dependency_count == 0
+            end
+            executable_parents
+        end
+
+        def to_s
+            p = "Parent_Nodes[#{@parent_nodes.collect { |n| "#{n.name}:#{n.counter}" }.join(", ")}]"
+            c = "Child_Nodes[#{@child_nodes.collect { |n| n.name }.join(", ")}]"
+            
+            "(#{@name}, #{p}, #{c}, " +
+                "Sync[#{@sync==SYNC_NONE ? "async" : @sync==SYNC_FIRST ? "first" : "alls"}], " +
+                "Scale[#{manager.get_scale}])"
+        end
+
+        private
+
+        def spawn_processing(i, events, threads, target = nil)
+
+            if target.nil?
+                target_manager = nil
+                prev_scale = @prev_scale
+            else
+                target_manager = target.manager
+                prev_scale = target.prev_scale
+            end
+
+            if target_manager.nil?
+                $stdout.printf( "Processing node '%s[%d]'.\n", @name, i)
+            else
+                $stdout.printf( "Processing target node '%s[%d]' from %s.\n", target_manager.name, i, @name)
+            end
+
+            # If no scale up occurs then run only the given events.
+            orchestrate_events = events.clone
+            # If new VMs have been added to the cluster to scale up then add default events for the new VM.
+            orchestrate_events = orchestrate_events.merge([ "create", "install", "configure" ]) if i >= prev_scale
+
+            @logger.debug("Events for node '#{name}' instance #{i} build: " +
+                "#{orchestrate_events.collect { |e| e } .join(", ")}") if @logger.debug?
+
+            if @sync==SYNC_ALL || (i==0 && @sync==SYNC_FIRST)
+                @manager.process(i, orchestrate_events, parse_attributes(@attributes, i), target_manager)
+            else
+                @resource_sync[i].wait if target.nil?
+                threads << Thread.new {
+
+                    begin
+                        @manager.process(i, orchestrate_events, parse_attributes(@attributes, i), target_manager)
+
+                    rescue Exception => msg
+
+                        puts("Fatal Error: #{msg}")
+                        @logger.debug(msg.backtrace.join("\n\t"))
+
+                        raise StackBuilder::Common::StackOrchestrateError,
+                            "Processing node resource '#{name}[#{i}]' " +
+                            "terminated with an error: #{msg}"
+                    ensure
+                        @resource_sync[i].signal if target.nil?
+                    end
+                }
+            end
+        end
+
+        def parse_attributes(attributes, index)
+
+            results = { }
+            attributes.each_pair do |k, v|
+
+                @logger.debug("Evaluating #{k} = #{v}")
+
+                if v.is_a?(Hash)
+                    results[k] = parse_attributes(v, index)
+
+                elsif v.is_a?(Array)
+
+                    results[k] = [ ]
+                    v.each do |aval|
+                        results[k] << parse_attributes( { "#" => aval }, index)["#"]
+                    end
+
+                elsif v =~ /^nodes\[.*\](.size)?$/
+
+                    lookup_keys = v.split(/[\[\]]/).reject { |l| l == "nodes" || l.empty? }
+
+                    l = lookup_keys.shift
+                    node = @nodes[l]
+                    unless node.nil?
+
+                        node_attributes = node.manager.node_attributes
+                        unless node_attributes.nil? || node_attributes.empty?
+
+                            indexes = [ ]
+                            values = [ ]
+
+                            l = lookup_keys.shift
+                            case l
+                                when ".size"
+                                    values << node.scale
+                                when "*"
+                                    indexes = (0..node.scale-1).to_a
+                                when /\d+/
+                                    indexes << l.to_i
+                                else
+                                    indexes << 0
+                            end
+
+                            indexes.each do |i|
+                                v = node_attributes[i]
+                                lookup_keys.each do |j|
+                                    v = v[j]
+                                    break if v.nil?
+                                end
+                                values << v
+                            end
+
+                            results[k] = (l == "*" ? values : values[0])
+                        end
+                    end
+
+                elsif v.is_a?(String)
+                    v = v.split(/(\#){|}/)
+
+                    if v.size == 1
+                        results[k] = v[0]
+                    else
+                        result = ""
+
+                        is_var = false
+                        v.each do |s|
+
+                            if is_var
+                                is_var = false
+                                sstr = (s == "index" ? index.to_s : parse_attributes( { "#" => s }, index)["#"])
+                                result += sstr unless sstr.nil?
+                                next
+                            end
+
+                            if s == "#"
+                                is_var = true
+                                next
+                            end
+
+                            result += s
+                        end
+
+                        if result.start_with?('<<!')
+
+                            s = result[3, result.length-3]
+                            @logger.debug("Evaluating the result of: #{s}")
+                            results[k] = eval(s)
+                        else
+                            results[k] = result
+                        end
+                    end
+                else
+                    results[k] = v
+                end
+
+                @logger.debug("Evaluated #{k} = #{results[k]}")
+            end
+
+            results
+
+        rescue Exception => msg
+
+            @logger.debug("Fatal Error: #{msg} : #{msg.backtrace.join("\n\t")}")
+            raise msg
+        end
+
+    end
+    
+end

data/lib/stackbuilder/stack/stack.rb

@@ -0,0 +1,224 @@
+# Copyright (c) 2014 Mevan Samaratunga
+
+include StackBuilder::Common::Helpers
+
+module StackBuilder::Stack
+
+    class Stack
+        
+        attr_reader :id
+        attr_reader :name
+        attr_reader :nodes
+        
+        def initialize(provider, stack_file, id = nil, overrides = nil)
+
+            StackBuilder::Common::Config.set_silent
+            @logger = StackBuilder::Common::Config.logger
+
+            raise InvalidArgs, "Node provider is not derived from
+                StackBuilder::Stack::NodeProvider." unless provider.is_a?(NodeProvider)
+
+            @provider = provider
+            env_vars = provider.get_env_vars
+
+            stack = StackBuilder::Common.load_yaml(stack_file, env_vars)
+            @logger.debug("Initializing stack definition:\n #{stack.to_yaml}")
+
+            overrides = JSON.load(File.new(overrides, 'r')) unless overrides.nil? || !overrides.end_with?('.json')
+            merge_maps(stack, overrides) unless overrides.nil?
+
+            if id.nil?
+                @id = SecureRandom.uuid.gsub(/-/, '')
+                @provider.set_stack(stack, @id)
+            else
+                @id = id
+                @provider.set_stack(stack, @id)
+            end
+
+            @name = stack["name"]
+            @nodes = { }
+
+            if stack.has_key?("stack") && stack["stack"].is_a?(Array)
+                
+                stack["stack"].each do |n|
+                    
+                    raise ArgumentError, "Node does not have a 'node' attribute " +
+                        "that identifies it: #{n}" unless n.has_key?("node")
+                    
+                    node_id = n["node"]
+                    raise ArgumentError, "Node identified by \"#{node_id}\" " +
+                        "already exists." if @nodes.has_key? (node_id)
+
+                    n["attributes"] = { } if n["attributes"].nil?
+                    merge_maps(n["attributes"], stack["attributes"]) unless stack["attributes"].nil?
+
+                    node_manager = @provider.get_node_manager(n)
+                    raise InvalidArgs, "Node task is of an invalid type. It is not derived " +
+                        "from StackBuilder::Stack::Node." unless node_manager.is_a?(NodeManager)
+
+                    @nodes[node_id] = NodeTask.new(node_manager, @nodes, n, id)
+                end
+
+                # Associate dependencies
+                stack["stack"].each do |n|
+                    
+                    node_task = @nodes[n["node"]]
+                    
+                    if n.has_key?("depends_on") && n["depends_on"].is_a?(Array)
+                        
+                        n["depends_on"].each do |d|
+                            
+                            raise ArgumentError, "Dependency node \"#{d}\" " +
+                                "is not defined." unless @nodes.has_key?(d)
+                            
+                            node_task.add_dependency(d)
+                        end
+                    end
+
+                    if n.has_key?("targets") && n["targets"].is_a?(Array)
+
+                        n["targets"].each do |d|
+
+                            raise ArgumentError, "Target node \"#{d}\" " +
+                                "is not defined." unless @nodes.has_key?(d)
+
+                            node_task.add_dependency(d, true)
+                        end
+                    end
+
+                    node_task.process_attribute_dependencies
+                end
+                
+            else
+                raise ArgumentError, "System needs to have at least one node defined."
+            end
+        end
+
+        def orchestrate(events = nil, name = nil, scale = nil)
+
+            events = Set.new([ "configure" ]) if events.nil?
+
+            unless name.nil?
+                node = @nodes[name]
+                raise StackBuilder::Common::StackBuilderError, "Invalid node name \"#{name}'\"." if node.nil?
+
+                unless scale.nil?
+                    raise ArgumentError, "The scale for node \"#{@name}\" must be greater than 0." if scale < 1
+                    node.scale = scale
+                end
+            end
+
+            prep_threads = [ ]
+            execution_list = [ ]
+
+            if name.nil?
+
+                @nodes.each_value do |n|
+                    execution_list << n if n.init_dependency_count == 0
+                    prep_threads += n.prepare
+                end
+
+                task_count = @nodes.size
+            else
+                # Only process nodes that depend on 'name' and their dependencies
+
+                def add_parent_task(node, prep_threads, nodes_visited)
+
+                    prep_threads += node.prepare
+                    nodes_visited << node.name
+
+                    node.init_dependency_count(1)
+
+                    node.parent_nodes.each do |n|
+                        add_parent_task(n, prep_threads, nodes_visited)
+                    end
+                end
+
+                node = @nodes[name]
+                nodes_visited = Set.new([ node.name ])
+
+                execution_list << node
+                prep_threads += node.prepare
+
+                node.parent_nodes.each do |n|
+                    add_parent_task(n, prep_threads, nodes_visited)
+                end
+
+                task_count = nodes_visited.size
+            end
+
+            execution_count = 0
+            terminate = false
+            
+            while !terminate && !execution_list.empty? do
+
+                mutex = Mutex.new
+                new_execution_list = [ ]
+                
+                exec_threads = [ ]
+                execution_list.each do |n|
+                    exec_threads << Thread.new {
+                        begin
+                            executable_parents = n.orchestrate(events)
+                            
+                            mutex.synchronize {
+                                new_execution_list |= executable_parents
+                                execution_count += 1
+                            }                    
+                        rescue Exception => msg
+                            @logger.error("Processing node '#{n}' terminated with an exception: #{msg}")
+                            @logger.info(msg.backtrace.join("\n\t"))
+                            terminate = true
+                        end
+                    }
+                end
+                exec_threads.each { |t| t.join }
+
+                execution_list = new_execution_list
+            end
+            
+            prep_threads.each { |t| t.join }
+
+            @nodes.each_value do |n|
+                n.prev_scale = n.scale
+            end
+
+            raise StackBuilder::Common::StackBuilderError, "Processing of stack nodes " +
+                "did not complete because of errors." if execution_count < task_count
+        end
+
+        def scale(name, scale)
+            self.orchestrate(nil, name, scale)
+        end
+
+        def destroy
+
+            @nodes.values.each { |n| n.deleted = true }
+            
+            begin
+                destroy_events = Set.new([ "stop", "uninstall" ])
+                self.orchestrate(destroy_events)
+            rescue Exception => msg
+                @logger.warn("An error was encountered attempting to do an orderly tear down of the system: #{msg}")
+                @logger.info("All remaining nodes will be destroyed forcefully.")
+            end
+            
+            threads = [ ]
+            
+            @nodes.values.each do |n|
+                (n.manager.get_scale - 1).downto(0) do |i|
+                    threads << Thread.new {
+        
+                        @logger.debug("Deleted #{n} #{i}.")
+                        $stdout.printf("Deleting node \"%s\" #%d.\n", n.name, i) unless @logger.debug?
+                        
+                        n.manager.delete(i)
+                    }
+                end
+            end
+            
+            threads.each { |t| t.join }
+        end
+    end
+
+end

data/lib/stackbuilder/version.rb

@@ -0,0 +1,8 @@
+    # Copyright (c) 2014 Mevan Samaratunga
+
+module Knife
+    module StackBuilder
+        VERSION = "0.5.2"
+        MAJOR, MINOR, TINY = VERSION.split('.')
+    end
+end