cfhighlander 0.2.0.alpha.10

data/lib/highlander.compiler.rb

@@ -0,0 +1,306 @@
+require 'cfndsl'
+require 'erb'
+require 'fileutils'
+require 'cfndsl/globals'
+require 'cfndsl/version'
+require 'json'
+require 'yaml'
+require 'open-uri'
+require 'net/http'
+require 'net/https'
+require 'highline/import'
+require 'zip'
+require_relative './util/zip.util'
+
+module Highlander
+
+  module Compiler
+
+    class ComponentCompiler
+
+      @@global_extensions_paths = []
+
+      attr_accessor :workdir,
+          :component,
+          :compiled_subcomponents,
+          :component_name,
+          :config_output_location,
+          :dsl_output_location,
+          :cfn_output_location,
+          :cfn_template_paths,
+          :silent_mode,
+          :lambda_src_paths
+
+      def initialize(component)
+
+        @workdir = ENV['HIGHLANDER_WORKDIR']
+        @component = component
+        @sub_components = []
+        @component_name = component.highlander_dsl.name.downcase
+        @cfndsl_compiled = false
+        @config_compiled = false
+        @cfn_template_paths = []
+        @lambdas_processed = false
+        @silent_mode = false
+        @lambda_src_paths = []
+
+        if @@global_extensions_paths.empty?
+          global_extensions_folder = "#{File.dirname(__FILE__)}/../cfndsl_ext"
+          Dir["#{global_extensions_folder}/*.rb"].each { |f| @@global_extensions_paths << f }
+        end
+
+        @component.highlander_dsl.components.each do |sub_component|
+          sub_component_compiler = Highlander::Compiler::ComponentCompiler.new(sub_component.component_loaded)
+          sub_component_compiler.component_name = sub_component.name
+          @sub_components << sub_component_compiler
+        end
+      end
+
+      def silent_mode=(value)
+        @silent_mode = value
+        @sub_components.each { |scc| scc.silent_mode=value }
+      end
+
+      def compileCfnDsl(out_format)
+        processLambdas unless @lambdas_processed
+        writeConfig unless @config_written
+        dsl = @component.highlander_dsl
+        component_cfndsl = @component.cfndsl_content
+
+        @component.highlander_dsl.components.each { |sc|
+          sc.distribution_format = out_format
+        }
+
+        # indent component cfndsl
+        component_cfndsl.gsub!("\n", "\n\t")
+        component_cfndsl.gsub!("\r\n", "\r\n\t")
+        # render cfndsl
+        renderer = ERB.new(File.read("#{__dir__}/../templates/cfndsl.component.template.erb"))
+        cfn_template = renderer.result(OpenStruct.new({
+            'dsl' => dsl,
+            'component_cfndsl' => component_cfndsl,
+            'component_requires' => (@@global_extensions_paths + @component.cfndsl_ext_files)
+        }).instance_eval { binding })
+
+        # write to output file
+        output_dir = "#{@workdir}/out/cfndsl"
+        @dsl_output_location = output_dir
+        output_path = "#{output_dir}/#{@component_name}.compiled.cfndsl.rb"
+        FileUtils.mkdir_p(output_dir) unless Dir.exist?(output_dir)
+        File.write(output_path, cfn_template)
+        puts "cfndsl template for #{dsl.name} written to #{output_path}"
+        @cfndsl_compiled_path = output_path
+
+        @sub_components.each { |subcomponent_compiler|
+          puts "Rendering sub-component cfndsl: #{subcomponent_compiler.component_name}"
+          subcomponent_compiler.compileCfnDsl out_format
+        }
+
+        @cfndsl_compiled = true
+
+      end
+
+      def compileCloudFormation(format = 'yaml')
+
+
+        #compile cfndsl templates first
+        compileCfnDsl format unless @cfndsl_compiled
+
+        dsl = @component.highlander_dsl
+        component_cfndsl = @component.cfndsl_content
+
+        # create out dir if not there
+        @cfn_output_location = "#{@workdir}/out/#{format}"
+        output_dir = @cfn_output_location
+        FileUtils.mkdir_p(output_dir) unless Dir.exist?(output_dir)
+
+        # write config
+        config_yaml_path = writeConfig
+
+
+        # compile templates
+        output_path = "#{output_dir}/#{component_name}.compiled.#{format}"
+        @cfn_template_paths << output_path
+        # configure cfndsl
+        cfndsl_opts = []
+        cfndsl_opts.push([:yaml, config_yaml_path])
+
+        # grab cfndsl model
+        model = CfnDsl.eval_file_with_extras(@cfndsl_compiled_path, cfndsl_opts, false)
+
+        # write resulting cloud formation template
+        if format == 'json'
+          output_content = JSON.pretty_generate(model)
+        elsif format == 'yaml'
+          output_content = JSON.parse(model.to_json).to_yaml
+        else
+          raise StandardError, "#{format} not supported for cfn generation"
+        end
+
+        File.write(output_path, output_content)
+        # `cfndsl #{@cfndsl_compiled_path} -p -f #{format} -o #{output_path} --disable-binding`
+        puts "CloudFormation #{format.upcase} template for #{dsl.name} written to #{output_path}"
+
+        # compile sub-component templates
+        @sub_components.each do |sub_component|
+          sub_component.compileCloudFormation format
+          @cfn_template_paths += sub_component.cfn_template_paths
+          @lambda_src_paths += sub_component.lambda_src_paths
+        end
+
+      end
+
+      def writeConfig(write_subcomponents_config = false)
+        @config_output_location = "#{@workdir}/out/config"
+        config_yaml_path = "#{@config_output_location}/#{@component_name}.config.yaml"
+        FileUtils.mkdir_p(@config_output_location) unless Dir.exist?(@config_output_location)
+
+        File.write(config_yaml_path, @component.config.to_yaml)
+        puts "Config for #{@component.highlander_dsl.name} written to #{config_yaml_path}"
+
+        if write_subcomponents_config
+          # compile sub-component templates
+          @sub_components.each do |sub_component|
+            sub_component.writeConfig write_subcomponents_config
+          end
+        end
+        @config_written = true
+        config_yaml_path
+      end
+
+      def processLambdas()
+
+        @component.highlander_dsl.lambda_functions_keys.each do |lfk|
+          resolver = LambdaResolver.new(@component,
+              lfk,
+              @workdir,
+              (not @silent_mode)
+          )
+          @lambda_src_paths += resolver.generateSourceArchives
+          resolver.mergeComponentConfig
+        end
+
+        @lambdas_processed = true
+
+      end
+
+    end
+
+
+    class LambdaResolver
+
+      def initialize(component, lambda_key, workdir, confirm_code_execution = true)
+        @component = component
+        @lambda_config = @component.config[lambda_key]
+        @component_dir = @component.component_dir
+        @workdir = workdir
+        @metadata = {
+            'path' => {},
+            'sha256' => {},
+            'version' => {}
+        }
+        @confirm_code_execution = confirm_code_execution
+      end
+
+      def generateSourceArchives
+
+        # Clear previous packages
+        FileUtils.rmtree "#{@workdir}/output/lambdas"
+
+        archive_paths = []
+
+        # Cached downloads map
+        cached_downloads = {}
+        @lambda_config['functions'].each do |name, lambda_config|
+          # create folder
+          out_folder = "#{@workdir}/out/lambdas/"
+          timestamp = Time.now.utc.to_i.to_s
+          file_name = "#{name}.#{@component.name}.#{@component.version}.#{timestamp}.zip"
+          @metadata['path'][name] = file_name
+          full_destination_path = "#{out_folder}#{file_name}"
+          archive_paths << full_destination_path
+          FileUtils.mkdir_p out_folder
+          # clear destination if already there
+          FileUtils.remove full_destination_path if File.exist? full_destination_path
+
+          # download file if code remote archive
+          puts "Packaging AWS Lambda function #{name}...\n"
+          if lambda_config['code'].include? 'http'
+            md5 = Digest::MD5.new
+            md5.update lambda_config['code']
+            hash = md5.hexdigest
+            cached_location = "#{@workdir}/.cache/lambdas/#{hash}"
+            if cached_downloads.key? lambda_config['code']
+              puts "Using already downloaded archive #{lambda_config['code']}"
+              FileUtils.copy(cached_downloads[lambda_config['code']], full_destination_path)
+            elsif File.file? cached_location
+              puts "Using cached download - '#{cached_location}'"
+              FileUtils.copy(cached_location, "#{out_folder}/src.zip")
+            else
+              puts "Downloading file #{lambda_config['code']} ..."
+              download = open(lambda_config['code'])
+              IO.copy_stream(download, "#{out_folder}/src.zip")
+              FileUtils.mkdir_p('.cache/lambdas')
+              FileUtils.copy("#{out_folder}/src.zip", cached_location)
+              puts "Download complete, caching in #{cached_location}"
+              cached_downloads[lambda_config['code']] = cached_location
+            end
+          else
+            # zip local code
+            zip_options = ''
+            full_path = "#{@component_dir}/lambdas/#{lambda_config['code']}"
+
+            # lambda source can be either path to file or directory within that file
+            # optionally, lambda source code
+            lambda_source_dir = File.dirname(full_path)
+            lambda_source_file = File.basename(full_path)
+            lambda_source_file = '.' if Pathname.new(full_path).directory?
+            lambda_source_dir = full_path if Pathname.new(full_path).directory?
+
+            # executing package command can generate files. We DO NOT want this file in source directory,
+            # but rather in intermediate directory
+            tmp_source_dir = "#{@workdir}/out/lambdas/tmp/#{name}"
+            FileUtils.mkpath(File.dirname(tmp_source_dir))
+            FileUtils.copy_entry(lambda_source_dir, tmp_source_dir)
+            lambda_source_dir = tmp_source_dir
+
+            # Lambda function source code allows pre-processing (e.g. install code dependencies)
+            unless lambda_config['package_cmd'].nil?
+              puts "Following code will be executed to generate lambda function #{name}:\n\n#{lambda_config['package_cmd']}\n\n"
+
+              if @confirm_code_execution
+                exit -7 unless HighLine.agree('Proceed (y/n)?')
+              end
+
+              package_cmd = "cd #{lambda_source_dir} && #{lambda_config['package_cmd']}"
+              puts 'Processing package command...'
+              package_result = system(package_cmd)
+              unless package_result
+                puts "Error packaging lambda function, following command failed\n\n#{package_cmd}\n\n"
+                exit -4
+              end
+            end
+            File.delete full_destination_path if File.exist? full_destination_path
+            zip_generator = Highlander::Util::ZipFileGenerator.new(lambda_source_dir, full_destination_path)
+            zip_generator.write
+
+          end
+
+          sha256 = Digest::SHA256.file full_destination_path
+          sha256 = sha256.base64digest
+          puts "Created zip package #{full_destination_path} for lambda #{name} with digest #{sha256}"
+          @metadata['sha256'][name] = sha256
+          @metadata['version'][name] = timestamp
+        end
+
+        return archive_paths
+      end
+
+      def mergeComponentConfig
+        @component.config['lambda_metadata'] = @metadata
+      end
+
+    end
+
+  end
+end

data/lib/highlander.dsl.base.rb

@@ -0,0 +1,25 @@
+
+module Highlander
+
+  module Dsl
+    class DslBase
+
+      attr_accessor :config
+
+      def initialize(parent)
+        @config = parent.config unless parent.nil?
+      end
+
+      def method_missing(method, *args)
+        if @config.nil?
+          raise StandardError, "#{self} no config!"
+        end
+        return @config["#{method}"] unless @config["#{method}"].nil?
+        raise StandardError, "#{self}Unknown method or variable #{method} in Highlander template"
+      end
+
+    end
+  end
+
+end
+

data/lib/highlander.dsl.component.rb

@@ -0,0 +1,243 @@
+require_relative './highlander.helper'
+require_relative './highlander.dsl.base'
+require_relative './highlander.factory'
+
+module Highlander
+
+  module Dsl
+
+    class SubcomponentParameter
+      attr_accessor :name, :cfndsl_value
+
+      def initialize
+
+      end
+
+    end
+
+
+    class Component < DslBase
+
+      attr_accessor :name,
+          :template,
+          :template_version,
+          :distribution_format,
+          :distribution_location,
+          :distribution_url,
+          :distribution_format,
+          :component_loaded,
+          :parameters,
+          :param_values,
+          :parent,
+          :component_config_override,
+          :export_config
+
+      def initialize(parent,
+          name,
+          template,
+          param_values,
+          component_sources = [],
+          config = {},
+          export_config = {},
+          distribution_format = 'yaml')
+
+        @parent = parent
+        @config = config
+        @export_config = export_config
+        @component_sources = component_sources
+
+        template_name = template
+        template_version = 'latest'
+        if template.include?('@') and not (template.start_with? 'git')
+          template_name = template.split('@')[0]
+          template_version = template.split('@')[1]
+        end
+
+        @template = template_name
+        @template_version = template_version
+        @name = name
+        @param_values = param_values
+
+        # distribution settings
+        @distribution_format = distribution_format
+        # by default components located at same location as master stack
+        @distribution_location = '.'
+        build_distribution_url
+
+        # load component
+        factory = Highlander::Factory::ComponentFactory.new(@component_sources)
+        @component_loaded = factory.findComponent(
+            @template,
+            @template_version
+        )
+        @component_loaded.config.extend @config
+
+        @parameters = []
+        # load_parameters
+      end
+
+      def version=(value)
+        @component_loaded.version = value
+      end
+
+      def distribute_bucket=(value)
+        @component_loaded.distribution_bucket = value
+      end
+
+      def distribute_prefix=(value)
+        @component_loaded.distribution_prefix = value
+      end
+
+      def distribution_format=(value)
+        @distribution_format = value
+        build_distribution_url
+      end
+
+      def build_distribution_url
+        @distribution_location = @parent.distribute_url unless @parent.distribute_url.nil?
+        @distribution_url = "#{@distribution_location}/#{@name}.compiled.#{@distribution_format}"
+      end
+
+      def load(component_config_override = {})
+        # check for component config on parent
+        parent = @parent
+
+        # Highest priority is DSL defined configuration
+        component_config_override.extend @config
+
+        @component_config_override = component_config_override
+
+        @component_loaded.load @component_config_override
+      end
+
+      # Parameters should be lazy loaded, that is late-binding should happen once
+      # all parameters and mappings are known
+      def load_parameters
+        component_dsl = @component_loaded.highlander_dsl
+        component_dsl.parameters.param_list.each do |component_param|
+          param = Highlander::Dsl::SubcomponentParameter.new
+          param.name = component_param.name
+          param.cfndsl_value = SubcomponentParamValueResolver.resolveValue(
+              @parent,
+              self,
+              component_param)
+          @parameters << param
+        end
+      end
+
+    end
+
+    class SubcomponentParamValueResolver
+      def self.resolveValue(component, sub_component, param)
+
+        puts("Resolving parameter #{component.name} -> #{sub_component.name}.#{param.name}")
+
+        # check if there are values defined on component itself
+        if sub_component.param_values.key?(param.name)
+          return Highlander::Helper.parameter_cfndsl_value(sub_component.param_values[param.name])
+        end
+
+        if param.class == Highlander::Dsl::StackParam
+          return self.resolveStackParamValue(component, sub_component, param)
+        elsif param.class == Highlander::Dsl::ComponentParam
+          return self.resolveComponentParamValue(component, sub_component, param)
+        elsif param.class == Highlander::Dsl::MappingParam
+          return self.resolveMappingParamValue(component, sub_component, param)
+        elsif param.class == Highlander::Dsl::OutputParam
+          return self.resolveOutputParamValue(component, sub_component, param)
+        else
+          raise "#{param.class} not resolvable to parameter value"
+        end
+      end
+
+      def self.resolveStackParamValue(component, sub_component, param)
+        param_name = param.is_global ? param.name : "#{sub_component.name}#{param.name}"
+        return "Ref('#{param_name}')"
+      end
+
+      def self.resolveComponentParamValue(component, sub_component, param)
+        # check component config for param value
+        # TODO
+        # check stack config for param value
+        # TODO
+        # return default value
+        return "'#{param.default_value}'"
+      end
+
+      def self.resolveMappingParamValue(component, sub_component, param)
+
+        # determine map name
+
+        provider = nil
+
+        mappings_name = param.mapName
+        actual_map_name = mappings_name
+
+        key_name = nil
+
+        # priority 0: stack-level parameter of map name
+        stack_param_mapname = component.parameters.param_list.find { |p| p.name == mappings_name }
+        unless stack_param_mapname.nil?
+          key_name = "Ref('#{mappings_name}')"
+        end
+
+        # priority 1 mapping provider keyName - used as lowest priority
+        if key_name.nil?
+          provider = mappings_provider(mappings_name)
+          if ((not provider.nil?) and (provider.respond_to?('getDefaultKey')))
+            key_name = provider.getDefaultKey
+          end
+        end
+
+        # priority 2: dsl defined key name
+        if key_name.nil?
+          key_name = param.mapKey
+          # could still be nil after this line
+        end
+
+        value =  mapping_value(component: component,
+            provider_name: mappings_name,
+            value_name: param.mapAttribute,
+            key_name: key_name
+        )
+
+        if value.nil?
+          return "'#{param.default_value}'" unless param.default_value.empty?
+          return "''"
+        end
+
+        return value
+
+
+        return value
+      end
+
+      def self.resolveOutputParamValue(component, sub_component, param)
+        component_name = param.component
+        resource_name = nil
+        if not sub_component.export_config.nil?
+          if sub_component.export_config.key? component_name
+            resource_name = sub_component.export_config[component_name]
+          end
+        end
+
+        if resource_name.nil?
+          # find by component
+          resource = component.components.find { |c| c.name == component_name }
+          resource_name = resource.name unless resource.nil?
+          if resource_name.nil?
+            resource = component.components.find { |c| c.template == component_name }
+            resource_name = resource.name unless resource.nil?
+          end
+        end
+
+        if resource_name.nil?
+          raise "#{sub_component.name}.Params.#{param.name}: Failed to resolve OutputParam '#{param.name}' with source '#{component_name}'. Component not found!"
+        end
+
+        return "FnGetAtt('#{resource_name}','Outputs.#{param.name}')"
+      end
+    end
+
+  end
+end

data/lib/highlander.dsl.params.rb

@@ -0,0 +1,113 @@
+require_relative './highlander.dsl.base'
+
+module Highlander
+
+  module Dsl
+    class Parameters < DslBase
+
+      attr_accessor :param_list
+
+      def initialize()
+        @param_list = []
+      end
+
+      def addParam(param)
+        existing_param = @param_list.find { |p| p.name == param.name }
+        if not existing_param.nil?
+          puts "Parameter being overwritten. Updating parameter #{param.name} with new definition..."
+          @param_list[@param_list.index(existing_param)] = param
+        else
+          @param_list << param
+        end
+      end
+
+      def StackParam(name, defaultValue='', isGlobal: false, noEcho: false)
+        param = StackParam.new(name, 'String', defaultValue)
+        param.is_global = isGlobal
+        param.config = @config
+        param.no_echo = noEcho
+        addParam param
+      end
+
+      def ComponentParam(name, defaultValue='')
+        param = ComponentParam.new(name, 'String', defaultValue)
+        param.config = @config
+        addParam param
+      end
+
+      def MappingParam(name, defaultValue='', &block)
+        param = MappingParam.new(name, 'String', defaultValue)
+        param.config = @config
+        param.instance_eval(&block)
+        addParam param
+      end
+
+      def OutputParam(component:, name:, default: '')
+        param = OutputParam.new(component, name, default)
+        param.config = @config
+        addParam param
+      end
+    end
+
+    class Parameter < DslBase
+      attr_accessor :name, :type, :default_value, :no_echo
+
+      def initialize(name, type, defaultValue, noEcho = false)
+        @no_echo = noEcho
+        @name = name
+        @type = type
+        @default_value = defaultValue
+      end
+    end
+
+    class StackParam < Parameter
+      attr_accessor :is_global
+    end
+
+    class ComponentParam < Parameter
+
+    end
+
+    class OutputParam < Parameter
+      attr_accessor :component
+
+      def initialize(component, name, default)
+        @component = component
+        @name = name
+        @default_value = default
+        @type = 'String'
+      end
+    end
+
+    class MappingParam < Parameter
+
+      attr_accessor :mapName, :mapKey, :mapAttribute
+
+      def method_missing(method, *args)
+        smethod = "#{method}"
+        if smethod.start_with?('Map')
+          puts smethod
+        end
+
+        super.method_missing(method)
+      end
+
+      def key(map_key)
+        @mapKey = map_key
+      end
+
+      def attribute(key)
+        @mapAttribute = key
+      end
+
+      def map(mapName)
+        @mapName = mapName
+      end
+
+      def mapProvider
+        mappings_provider(@mapName)
+      end
+
+    end
+  end
+end