stackmate 0.0.4 → 0.1.0

data/lib/stackmate/participants/common.rb

@@ -1,91 +1,240 @@
+require 'json'
+require 'yaml'
 require 'stackmate/logging'
-require 'stackmate/aws_attribs'
 require 'stackmate/intrinsic_functions'
+require 'stackmate/resolver'
+require 'stackmate/stackpi'
+require 'net/http'
+require 'uri'
+require 'time'
 
 module StackMate
 
 
-class WaitConditionHandle < Ruote::Participant
-  include Logging
+  # class ParamHandle < Ruote::Participant
+  #   include Logging
+  #   def on_workitem
+  #     logger.debug "Setting resolved parameter #{participant_name}"
+  #     workitem['ResolvedNames'][participant_name] = workitem['']
+  #   end
+  # end
+
+  class WaitConditionHandle < Ruote::Participant
+    include Logging
+
+    def create
+      logger.debug "Entering #{participant_name} "
+      workitem[participant_name] = {}
+      presigned_url = 'http://localhost:4567/waitcondition/' + workitem.fei.wfid + '/' + participant_name
+      workitem.fields['ResolvedNames'][participant_name] = presigned_url
+      logger.info "Your pre-signed URL is: #{presigned_url} "
+      logger.info "Try: \ncurl -X PUT --data 'foo' #{presigned_url}"
+      WaitCondition.create_handle(participant_name, presigned_url)
+      workitem[participant_name][:physical_id] = presigned_url
+    end
 
-  def on_workitem
-    logger.debug "Entering #{participant_name} "
-    workitem[participant_name] = {}
-    presigned_url = 'http://localhost:4567/waitcondition/' + workitem.fei.wfid + '/' + participant_name
-    workitem.fields['ResolvedNames'][participant_name] = presigned_url
-    logger.info "Your pre-signed URL is: #{presigned_url} "
-    logger.info "Try: \ncurl -X PUT --data 'foo' #{presigned_url}"
-    WaitCondition.create_handle(participant_name, presigned_url)
-    workitem[participant_name][:physical_id] = presigned_url
+    def delete
+      #p workitem
+      logger.info "In delete #{participant_name}"
+      return nil if !workitem[participant_name]
+      physical_id = workitem[participant_name][:physical_id]
+      if physical_id
+        workitem[participant_name] = {}
+        WaitCondition.delete_handle(participant_name)
+      end
+    end
 
-    reply
-  end
-end
-
-class WaitCondition < Ruote::Participant
-  include Logging
-  @@handles = {}
-  @@conditions = []
-
-  def on_workitem
-    logger.debug "Entering #{workitem.participant_name} "
-    workitem[participant_name] = {}
-    @@conditions << self
-    stackname = workitem.fields['ResolvedNames']['AWS::StackName']
-    workitem[participant_name][:physical_id] =  stackname + '-' + 'WaitCondition'
+    def on_workitem
+      if workitem['params']['operation'] == 'create'
+        create
+      else
+        #rollback / delete
+        delete
+      end
+      reply
+    end
   end
 
-  def self.create_handle(handle_name, handle)
+  class WaitCondition < Ruote::Participant
+    include Logging
+    @@handles = {}
+    @@conditions = []
+
+    def create
+      logger.debug "Entering #{workitem.participant_name} "
+      workitem[participant_name] = {}
+      @@conditions << self
+      stackname = workitem.fields['ResolvedNames']['AWS::StackName']
+      workitem[participant_name][:physical_id] =  stackname + '-' + 'WaitCondition'
+    end
+
+    def delete
+      logger.info "In delete #{participant_name}"
+      #no-op
+    end
+
+    def on_workitem
+      if workitem['params']['operation'] == 'create'
+        create
+      else
+        #rollback / delete
+        delete
+        reply
+      end
+    end
+
+    def self.create_handle(handle_name, handle)
       @@handles[handle_name] = handle
-  end
+    end
+
+    def self.delete_handle(handle_name)
+      @@handles.delete(handle_name)
+    end
 
-  def set_handle(handle_name)
+    def set_handle(handle_name)
       reply(workitem) if @@handles[handle_name]
-  end
+    end
 
-  def self.get_conditions()
+    def self.get_conditions()
       @@conditions
+    end
+  end
+
+  class Output < Ruote::Participant
+    include Logging
+    include Intrinsic
+
+    def on_workitem
+      #p workitem.fields.keys
+      logger.debug "Entering #{workitem.participant_name} "
+      outputs = workitem.fields['Outputs']
+      logger.debug "In StackMate::Output.on_workitem #{outputs.inspect}"
+      outputs.each do |key, val|
+        v = val['Value']
+        constructed_value = intrinsic(v, workitem)
+        val['Value'] = constructed_value
+        logger.debug "Output: key = #{key}, value = #{constructed_value} descr = #{val['Description']}"
+      end
+
+      logger.debug "Output Done"
+      reply
+    end
   end
-end
-
-class Output < Ruote::Participant
-  include Logging
-  include Intrinsic
-
-  def on_workitem
-    #p workitem.fields.keys
-    logger.debug "Entering #{workitem.participant_name} "
-    outputs = workitem.fields['Outputs']
-    logger.debug "In StackMate::Output.on_workitem #{outputs.inspect}"
-    outputs.each do |key, val|
-      v = val['Value']
-      constructed_value = intrinsic(v, workitem)
-      val['Value'] = constructed_value
-      logger.debug "Output: key = #{key}, value = #{constructed_value} descr = #{val['Description']}"
-    end
-
-    logger.debug "Output Done"
-    reply
+
+  class NoOpResource < Ruote::Participant
+    include Logging
+
+    def create
+      logger.debug "Creating #{participant_name} wfid=#{workitem.fei.wfid} fei=#{workitem.fei.to_h}"
+      workitem[participant_name] = {}
+      stackname = workitem.fields['ResolvedNames']['AWS::StackName']
+      logger.debug "physical id is  #{stackname}-#{participant_name} "
+      workitem[participant_name][:physical_id] =  stackname + '-' + participant_name
+      typ = workitem['Resources'][participant_name]['Type']
+      if AWS_FAKE_ATTRIB_VALUES[typ]
+        AWS_FAKE_ATTRIB_VALUES[typ].each do |k,v|
+          workitem[participant_name][k] = v
+        end
+      end
+    end
+
+    def delete
+      logger.debug "Deleting #{participant_name} wfid=#{workitem.fei.wfid} fei=#{workitem.fei.to_h}"
+    end
+
+    def on_workitem
+      @stackname = workitem['StackName']
+      if workitem['params']['operation'] == 'create'
+        create
+      else
+        delete
+      end
+      reply
+    end
   end
-end
-
-class NoOpResource < Ruote::Participant
-  include Logging
-
-  def on_workitem
-    logger.debug "Entering #{participant_name} wfid=#{workitem.fei.wfid} fei=#{workitem.fei.to_h}"
-    workitem[participant_name] = {}
-    stackname = workitem.fields['ResolvedNames']['AWS::StackName']
-    logger.debug "physical id is  #{stackname}-#{participant_name} "
-    workitem[participant_name][:physical_id] =  stackname + '-' + participant_name
-    typ = workitem['Resources'][participant_name]['Type']
-    if AWS_FAKE_ATTRIB_VALUES[typ]
-      AWS_FAKE_ATTRIB_VALUES[typ].each do |k,v| 
-        workitem[participant_name][k] = v
+
+  class StackNest < Ruote::Participant
+    include Logging
+    include Intrinsic
+    include Resolver
+    include StackPi
+
+    def create
+      logger.debug("Creating nested stack #{@stackname}")
+      #Get template from URL
+      #Call stackmate API for launching new stack
+      #read outputs from predefined location
+      #copy all critical fields from workitems
+
+      #CAVEAT - needs file storage. may be move to database backed
+      # needs a Outputs tag. Not a big deal since nested stacks anyways need
+      #outputs tags
+      workitem[@stack_name] = {}
+      params = workitem['ResolvedNames']
+      stack_props = workitem['Resources'][@stack_name]['Properties']
+      template_url = URI(get_resolved(stack_props['TemplateURL'],workitem))
+      logger.debug("Fetching template for #{@stack_name} from URL #{template_url}")
+      http = Net::HTTP.new(template_url.host,template_url.port)
+      if template_url.scheme == 'https'
+        http.use_ssl = true
+      end
+      http.start {
+        http.request_get(template_url.path) { |res|
+          File.open("/tmp/#{@stack_name}.template", 'w') { |file| file.write(res.body) }
+        }
+      }
+      stack_props['Parameters'].each_key do |k|
+        params[k] = get_resolved(stack_props['Parameters'][k],workitem)
+      end
+      params['stackrand'] = Time.now().to_i #TODO use subid from fei or something
+      params['isnested'] = "True"
+      StackMate::StackPi.create_stack("/tmp/#{@stack_name}.template",@stack_name,format_params(params),true)
+      file_name = "/tmp/#{@stack_name}.workitem.#{params['stackrand']}"
+      if(File.exists?(file_name))
+        output_workitem = YAML.load(File.read(file_name))
+        workitem[@stack_name]['ResolvedNames'] = output_workitem['ResolvedNames'].clone
+        workitem[@stack_name]['Outputs'] = output_workitem['Outputs'].clone
+        workitem[@stack_name]['Resources'] = output_workitem['Resources'].clone
+        workitem[@stack_name]['IdMap'] = output_workitem['IdMap'].clone
+        #copy all resources created
+        output_workitem['Resources'].each_key do |resource|
+          workitem[@stack_name][resource] = output_workitem[resource].clone
+        end
+        logger.debug("Successfully created nested stack #{@stack_name}")
+      else
+        logger.debug("Unable to create nested stack #{@stack_name}")
+        raise "Nested Stack Failed"
+      end
+    end
+
+    def delete
+      logger.debug("Deleting stack #{@stack_name}")
+      #TODO write code to roll back all stack info
+      #Probably nothing needed since new engine launched takes care of it
+      #No, need to clean up if parent stack fails after the nested stack is successfully created
+    end
+
+    def on_workitem
+      @stack_name = workitem.participant_name
+      if workitem['params']['operation'] == 'create'
+        create
+      else
+        #rollback / delete
+        delete
+      end
+      reply
+    end
+
+    def format_params(params)
+      result = ""
+      add_semi = false
+      params.each_key do |k|
+        result = result + ";" if add_semi
+        result = result + k + "=" + params[k].to_s if !params[k].nil?
+        add_semi = true
       end
+      result
     end
-    reply
   end
-end
 
-end
+end

data/lib/stackmate/participants/stacknest.rb

@@ -0,0 +1,6 @@
+require 'json'
+require 'cloudstack_ruby_client'
+require 'yaml'
+require 'stackmate/logging'
+require 'stackmate/intrinsic_functions'
+require 'stackmate/resolver'

data/lib/stackmate/resolver.rb

@@ -0,0 +1,122 @@
+#require 'stackmate/logging'
+require 'stackmate/intrinsic_functions'
+
+module StackMate
+  module Resolver
+    @devicename_map=
+    {
+      '/dev/sdb' => '1',
+      '/dev/sdc' => '2',
+      '/dev/sdd' => '3',
+      '/dev/sde' => '4',
+      '/dev/sdf' => '5',
+      '/dev/sdg' => '6',
+      '/dev/sdh' => '7',
+      '/dev/sdi' => '8',
+      '/dev/sdj' => '9',
+      '/dev/xvdb' => '1',
+      '/dev/xvdc' => '2',
+      '/dev/xvdd' => '3',
+      '/dev/xvde' => '4',
+      '/dev/xvdf' => '5',
+      '/dev/xvdg' => '6',
+      '/dev/xvdh' => '7',
+      '/dev/xvdi' => '8',
+      '/dev/xvdj' => '9',
+      'xvdb' => '1',
+      'xvdc' => '2',
+      'xvdd' => '3',
+      'xvde' => '4',
+      'xvdf' => '5',
+      'xvdg' => '6',
+      'xvdh' => '7',
+      'xvdi' => '8',
+      'xvdj' => '9',
+    }
+    #comes from awsapi reference
+    STRINGEXP = /.+/
+    INTEXP = /[0-9]+/
+    UUIDEXP = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/
+
+    def get_resolved(lookup_data,workitem)
+      case lookup_data
+      when String
+        lookup_data
+      when Hash
+        intrinsic(lookup_data,workitem)
+      end
+    end
+
+    def validate_param(value,type)
+      return true
+      begin
+        case type
+        when "boolean"
+          ["true","false"].include?(value)
+        when "date"
+          true
+        when "imageformat"
+          ["vhd","qcow"].include?(value)
+        when "int"
+          !INTEXP.match(value).nil?
+        when "integer"
+          !INTEXP.match(value).nil?
+        when "list"
+          #eval(value).kind_of?(Array)
+          true
+        when "long"
+          !INTEXP.match(value).nil?
+        when "map"
+          #eval(value).kind_of?(Hash)
+          true
+        when "set"
+          #eval(value).kind_of?(Array) and eval(value).uniq == eval(value)
+          true
+        when "short"
+          !INTEXP.match(value).nil?
+        when "state"
+          true
+        when "string"
+          value.kind_of?(String)
+        when "tzdate"
+          true
+        when "uuid"
+          !UUIDEXP.match(value).nil?
+        else
+          true
+        end
+      rescue Exception => e
+        false
+      end
+    end
+
+    def get_named_tag(tag_name,properties,workitem,default)
+      result = default
+      unless properties['Tags'].nil?
+        properties['Tags'].each do |tag|
+          k = tag['Key']
+          v = tag['Value']
+          if k.eql?(tag_name)
+            result = get_resolved(v,workitem)
+          end
+        end
+      end
+      result
+    end
+
+    def resolve_tags(tags_array,workitem)
+      result = {}
+      tags_array.each do |tag|
+        k = tag['key']
+        v = tag['value']
+        resolved_v = get_resolved(v,workitem)
+        result[k] = resolved_v
+      end
+      result
+    end
+
+    def resolve_to_deviceid(devicename)
+      @devicename_map[devicename.downcase]
+    end
+  end
+end

data/lib/stackmate/stack.rb

@@ -2,92 +2,148 @@ require 'json'
 require 'set'
 require 'tsort'
 require 'stackmate/logging'
+require 'yaml'
 
 module StackMate
 
-class Stacker
+  class Stacker
     include TSort
     include Logging
 
     attr_accessor :templ
 
     def initialize(stackstr, stackname, params)
-        @stackname = stackname
-        @resolved = params
-        @templ = JSON.parse(stackstr) 
-        @templ['StackName'] = @stackname
-        @param_names = @templ['Parameters']
-        @deps = {}
-        @pdeps = {}
-        validate_param_values
-        resolve_dependencies()
-        @templ['ResolvedNames'] = @resolved
+      @stackname = stackname
+      @resolved = params
+      @templ = JSON.parse(stackstr)
+      @templ['StackName'] = @stackname
+      @param_names = @templ['Parameters']
+      @deps = {}
+      @pdeps = {}
+      validate_param_values
+      resolve_dependencies()
+      validate_dependencies
+      @allowed_param_vales = get_allowed_values(@param_names)
+      @templ['ResolvedNames'] = populate_parameters(@param_names,@resolved)
+      #@templ['ResolvedNames']['StackId'] = SecureRandom.urlsafe_base64
+      @templ['IdMap'] = {}
+    end
+
+    def get_allowed_values(template_params)
+      allowed = {}
+      template_params.each_key do |k|
+        #Type is mandatory???
+        allowed[k] = {}
+        allowed[k]['Type'] = template_params[k]['Type'] if template_params[k].has_key?('Type')
+        allowed[k]['AllowedValues'] = template_params[k]['AllowedValues'] if template_params[k].has_key?('AllowedValues')
+      end
+      allowed
+    end
+
+    def populate_parameters(params,overrides)
+      populated={}
+      #Populate defaults
+      params.each_key do |k|
+        populated[k] = params[k]['Default']
+      end
+      #Then load local YAML mappings
+      begin
+        #TODO change to use stackid
+        #file_name = @stackname+".yml"
+        file_name = "local.yml"
+        localized = YAML.load_file(file_name)
+        localized.each_key do |k|
+          populated[k] = localized[k]
+        end
+      rescue
+        #raise "ERROR : Unable to load end point parameters"
+        logger.info("CAUTION: Unable to load end point parameters")
+      end
+      #Then override
+      overrides.each_key do |k|
+        populated[k] = overrides[k]
+      end
+      populated
     end
 
-    
     def validate_param_values
-        #TODO CloudFormation parameters have validity constraints specified
-        #Use them to validate parameter values (e.g., email addresses)
+      #TODO CloudFormation parameters have validity constraints specified
+      #Use them to validate parameter values (e.g., email addresses)
+      #As of now used only in actual cloudstack calls
+    end
+
+    def validate_dependencies
+      resources = @deps.keys
+      @deps.each_key do |k|
+        @deps[k].each do |resource|
+          if !resources.include?(resource)
+            raise "Bad reference or dependency on resource #{resource}"
+          end
+        end
+      end
     end
 
     def resolve_dependencies
-        @templ['Resources'].each { |key,val| 
-            deps = Set.new
-            pdeps = Set.new
-            find_refs(key, val, deps, pdeps)
-            deps << val['DependsOn'] if val['DependsOn']
-            #print key, " depends on ", deps.to_a, "\n"
-            @deps[key] = deps.to_a
-            @pdeps[key] = pdeps.to_a
-        }
-        @pdeps.keys.each do |k|
-            unres = @pdeps[k] - @resolved.keys
-            if ! unres.empty?
-                unres.each do |u|
-                    deflt = @param_names[u]['Default']
-                    #print "Found default value ", deflt, " for ", u, "\n" if deflt
-                    @resolved[u] = deflt if deflt
-                end
-                unres = @pdeps[k] - @resolved.keys
-                throw :unresolved, (@pdeps[k] - @resolved.keys) if !unres.empty?
-            end
+      @templ['Resources'].each { |key,val|
+        deps = Set.new
+        pdeps = Set.new
+        find_refs(key, val, deps, pdeps)
+        deps << val['DependsOn'] if val['DependsOn']
+        #print key, " depends on ", deps.to_a, "\n"
+        @deps[key] = deps.to_a
+        @pdeps[key] = pdeps.to_a
+      }
+      unresolved = []
+      @pdeps.keys.each do |k|
+        unres = @pdeps[k] - @resolved.keys
+        if ! unres.empty?
+          unres.each do |u|
+            deflt = @param_names[u]['Default']
+            #print "Found default value ", deflt, " for ", u, "\n" if deflt
+            @resolved[u] = deflt if deflt
+          end
+          unres = @pdeps[k] - @resolved.keys
+          unresolved = unresolved + unres
+          #throw :unresolved, (@pdeps[k] - @resolved.keys) if !unres.empty?
         end
+      end
+      throw :unresolved, unresolved.uniq if !unresolved.empty?
     end
 
 
     def find_refs (parent, jsn, deps, pdeps)
-        case jsn
-            when Array
-                jsn.each {|x| find_refs(parent, x, deps, pdeps)}
-                #print parent, ": ", jsn, "\n"
-            when Hash
-                jsn.keys.each do |k|
-                    #TODO Fn::GetAtt
-                    if k == "Ref"
-                        #only resolve dependencies on other resources for now
-                        if !@param_names.keys.index(jsn[k]) && jsn[k] != 'AWS::Region' && jsn[k] != 'AWS::StackId' && jsn[k] != 'AWS::StackName'
-                            deps << jsn[k]
-                            #print parent, ": ", deps.to_a, "\n"
-                        else if @param_names.keys.index(jsn[k])
-                            pdeps << jsn[k]
-                        end
-                        end
-                    else
-                        find_refs(parent, jsn[k], deps, pdeps)
-                    end
-                end
+      case jsn
+      when Array
+        jsn.each {|x| find_refs(parent, x, deps, pdeps)}
+        #print parent, ": ", jsn, "\n"
+      when Hash
+        jsn.keys.each do |k|
+          #TODO Fn::GetAtt
+          if k == "Ref"
+            #only resolve dependencies on other resources for now
+            if !@param_names.keys.index(jsn[k]) && !@resolved.keys.index(jsn[k]) && jsn[k] != 'AWS::Region' && jsn[k] != 'AWS::StackId' && jsn[k] != 'AWS::StackName'
+              deps << jsn[k]
+              #print parent, ": ", deps.to_a, "\n"
+            else if @param_names.keys.index(jsn[k])
+              pdeps << jsn[k]
+            end
+          end
+          else
+            find_refs(parent, jsn[k], deps, pdeps)
+          end
         end
-        return deps
+      end
+      return deps
     end
 
     def tsort_each_node(&block)
-        @deps.each_key(&block)
+      @deps.each_key(&block)
     end
 
     def tsort_each_child(name, &block)
-        @deps[name].each(&block) if @deps.has_key?(name)
+      @deps[name].each(&block) if @deps.has_key?(name)
     end
-    
-end
 
-end
+  end
+
+end