cfhighlander 0.2.1 → 0.3.0.alpha.1528936037

data/lib/cfhighlander.factory.templatefinder.rb

@@ -0,0 +1,248 @@
+LOCAL_CFHIGHLANDER_CACHE_LOCATION = "#{ENV['HOME']}/.cfhighlander/components"
+
+require_relative './cfhighlander.model.templatemeta'
+
+module Cfhighlander
+
+  module Factory
+
+    class TemplateFinder
+
+      def initialize(component_sources = [])
+        ## First look in local $PWD/components folder
+        ## Then search for cached $HOME/.highlander/components
+        ## Then search in sources given by dsl
+        default_locations = [
+            LOCAL_CFHIGHLANDER_CACHE_LOCATION,
+            File.expand_path('components'),
+            File.expand_path('.')
+        ]
+        default_locations.each do |predefined_path|
+          component_sources.unshift(predefined_path)
+        end
+
+        @component_sources = component_sources
+      end
+
+      def findTemplateDefault(template_name, component_version)
+        default_lookup_url = 'https://github.com/theonestack'
+        default_lookup_url = ENV['HIGHLANDER_DEFAULT_COMPONENT_GIT_LOOKUP'] if ENV.key? 'HIGHLANDER_DEFAULT_COMPONENT_GIT_LOOKUP'
+        default_lookup_url = ENV['CFHIGHLANDER_DEFAULT_COMPONENT_GIT_LOOKUP'] if ENV.key? 'CFHIGHLANDER_DEFAULT_COMPONENT_GIT_LOOKUP'
+
+        git_url = "#{default_lookup_url}/hl-component-#{template_name}"
+
+        if component_version.nil? or component_version.empty? or component_version == 'latest'
+          branch = 'master'
+        else
+          branch = component_version
+        end
+        local_path = "#{LOCAL_CFHIGHLANDER_CACHE_LOCATION}/#{template_name}/#{component_version}"
+        return findTemplateGit(local_path, template_name, component_version, git_url, branch)
+
+      end
+
+      def findTemplateGit(cache_path, component_name, component_version, git_url, branch)
+        begin
+          cache_path = "#{cache_path}/" unless cache_path.end_with? '/'
+          # if this is snapshot, clean local cache
+          if branch.end_with? '.snapshot'
+            branch = branch.gsub('.snapshot', '')
+            FileUtils.rmtree cache_path if File.exist? cache_path and File.directory? cache_path
+          end
+
+          # if local cache exists, return from cache
+          if not Dir.glob("#{cache_path}*.highlander.rb").empty?
+            # if cache exists, just return from cache
+            component_name = Dir.glob("#{cache_path}*.highlander.rb")[0].gsub(cache_path, '').gsub('.highlander.rb', '')
+            STDERR.puts("DEPRECATED: #{component_name}: Use *.cfhighlander.rb template file name pattern")
+            return component_name, cache_path
+          end
+
+          if not Dir.glob("#{cache_path}*.cfhighlander.rb").empty?
+            # if cache exists, just return from cache
+            component_name = Dir.glob("#{cache_path}*.cfhighlander.rb")[0].gsub(cache_path, '').gsub('.cfhighlander.rb', '')
+            return component_name, cache_path
+          end
+
+          # shallow clone
+          puts "Trying to load #{component_name}/#{component_version} from #{git_url}##{branch} ... "
+          clone_opts = { depth: 1 }
+          clone_opts[:branch] = branch if not (branch.nil? or branch.empty?)
+          Git.clone git_url, cache_path, clone_opts
+          puts "\t .. cached in #{cache_path}\n"
+          # return from cache once it's cloned
+          return findTemplateGit(cache_path, component_name, component_version, git_url, branch)
+        rescue Exception => e
+          STDERR.puts "Failed to resolve component #{component_name}@#{component_version} from #{git_url}"
+          STDERR.puts e
+          return nil
+        end
+      end
+
+      def findTemplateS3(s3_location, component_name, component_version, use_legacy_extension = true)
+        parts = s3_location.split('/')
+        bucket = parts[2]
+        prefix = parts[3]
+        s3_key = "#{prefix}/#{component_name}/#{component_version}/#{component_name}.highlander.rb"
+        s3_prefix = "#{prefix}/#{component_name}/#{component_version}/"
+        local_destination = "#{LOCAL_CFHIGHLANDER_CACHE_LOCATION}/#{component_name}/#{component_version}"
+        begin
+          s3 = Aws::S3::Client.new({ region: s3_bucket_region(bucket) })
+          FileUtils.mkdir_p local_destination unless Dir.exist? local_destination
+
+          hl_content = s3.get_object({ bucket: bucket,
+              key: s3_key,
+              response_target: "#{local_destination}/#{component_name}.#{use_legacy_extension ? 'highlander' : 'cfhighlander'}.rb"
+          })
+          # if code execution got so far we consider file exists and download it locally
+          component_files = s3.list_objects_v2({ bucket: bucket, prefix: s3_prefix })
+          component_files.contents.each {|s3_object|
+            file_name = s3_object.key.gsub(s3_prefix, '')
+            destination_file = "#{local_destination}/#{file_name}"
+            destination_dir = File.dirname(destination_file)
+            print "Caching #{file_name} of #{component_name}@#{component_version} in #{destination_dir} ... "
+
+            FileUtils.mkpath(destination_dir) unless File.exists?(destination_dir)
+            s3.get_object({ bucket: bucket, key: s3_object.key, response_target: destination_file })
+            print " [OK] \n"
+          }
+          if use_legacy_extension
+            STDERR.puts "DEPRECATED: s3 load #{component_name} with *.highlander.rb template file name pattern"
+          end
+          return local_destination
+        rescue => e
+          # this handles both nonexisting key and bucket
+          if use_legacy_extension
+            return findTemplateS3(s3_location, component_name, component_version, false)
+          end
+          puts("#{component_name} not found in s3://#{bucket}/#{prefix}")
+          STDERR.puts(e.to_s) unless e.message.include? 'does not exist'
+          return nil
+        end
+      end
+
+      # will try and locate template based on git template location
+      # if template location is not git repo, returns nil
+      def tryFindTemplateGit(template_location, template_version)
+
+        # avoid any nres
+        template_version = '' if template_version.nil?
+
+        # if empty or latest branch is empty
+        if template_version.empty? or template_version == 'latest' or template_version == 'latest.snapshot'
+          branch = ''
+        else
+          # otherwise component version is actual branch
+          branch = template_version
+        end
+
+
+        git_url = nil
+        if template_location.start_with? 'git:'
+          git_url = template_location.gsub('git:', '')
+        elsif template_location.start_with? 'github:'
+          git_url = "https://github.com/#{template_location.gsub('github:', '')}"
+        elsif template_location.start_with? 'github.com:'
+          git_url = "https://github.com/#{template_location.gsub('github.com:', '')}"
+        end
+
+
+        if not git_url.nil?
+          local_path = "#{LOCAL_CFHIGHLANDER_CACHE_LOCATION}/#{template_location.gsub(':', '_').gsub(/\/+/, '/')}/#{template_version}"
+          template_name, location = findTemplateGit(local_path, template_location, template_version, git_url, branch)
+          if location.nil?
+            raise "Could not resolve component #{template_location}@#{template_version}"
+          else
+            return template_name, template_version, location
+          end
+        end
+
+        return nil
+      end
+
+      def findTemplate(template_name, template_version)
+        template_version_s = template_version.nil? ? 'latest' : template_version
+        template_version = nil if template_version == 'latest'
+        is_git_template = template_name.start_with? 'git'
+
+        if template_name.include? '@' and (not template_name.start_with? 'git')
+          parts = template_name.split('@')
+          template_name = parts[0]
+          template_version = parts[1]
+          template_version_s = template_version
+        end
+
+        if template_name.include? '#' and (is_git_template)
+          parts = template_name.split('#')
+          template_name = parts[0]
+          template_version = parts[1]
+          template_version_s = template_version
+        end
+
+        # if component specified as git location
+        if is_git_template
+          new_template_name, new_version, candidate_git = tryFindTemplateGit(template_name, template_version_s)
+          return build_meta(new_template_name, new_version, candidate_git) unless candidate_git.nil?
+        end
+
+        # if not git but has .snapshot lookup in default, it is allowed to reference default
+        # snapshots
+        if (not template_version.nil?) and template_version.end_with? '.snapshot'
+          new_template_name, snapshot_candidate_location = findTemplateDefault(template_name, template_version_s)
+          return build_meta(new_template_name, template_version, snapshot_candidate_location) unless snapshot_candidate_location.nil?
+        end
+
+        # try in all of the component sources
+        @component_sources.each do |source|
+          template_full_name = "#{template_name}@#{template_version.nil? ? 'latest' : template_version}"
+          # TODO handle http(s) sources and their download to local
+          if source.start_with?('http')
+            raise StandardError, 'http(s) sources not supported yet'
+          elsif source.start_with?('s3://')
+            # s3 candidate
+
+            s3_candidate = findTemplateS3(source, template_name, template_version_s)
+            # at this point all component files are download to local file system and consumed from there
+            return build_meta(template_name, template_version_s, s3_candidate) unless s3_candidate.nil?
+
+          else
+            # file system candidate
+            candidate = "#{source}/#{template_name}"
+            candidate = "#{candidate}/#{template_version}" unless template_version.nil?
+            candidate_hl_path = "#{candidate}/#{template_name}.highlander.rb"
+            candidate_cfhl_path = "#{candidate}/#{template_name}.cfhighlander.rb"
+            candidate2_hl_path = "#{source}/#{template_name}.highlander.rb"
+            candidate2_cfhl_path = "#{source}/#{template_name}.cfhighlander.rb"
+            puts "TRACE: Trying to load #{template_full_name} from #{candidate} ... "
+            if (File.exist?(candidate_hl_path) or File.exist?(candidate_cfhl_path))
+              return build_meta(template_name, template_version_s, candidate)
+            end
+
+            puts "TRACE: Trying to load #{template_full_name} from #{source} ... "
+            # if component version is latest it is allowed to search in path
+            # with no version component in it
+            if (File.exist?(candidate2_hl_path) or File.exist?(candidate2_cfhl_path))
+              return build_meta(template_name, 'latest', source)
+            end unless template_version_s != 'latest'
+          end
+        end
+
+        # try default component source on github
+        template_name, default_candidate = findTemplateDefault(template_name, template_version_s)
+        return build_meta(template_name, template_version_s, default_candidate) unless default_candidate.nil?
+
+        return nil
+      end
+
+      def build_meta(template_name, template_version, template_location)
+        return Cfhighlander::Model::TemplateMetadata.new(
+            template_name: template_name,
+            template_version: template_version,
+            template_location: template_location)
+      end
+
+    end
+
+  end
+
+end

data/lib/{highlander.helper.rb → cfhighlander.helper.rb}

@@ -1,4 +1,4 @@
-module Highlander
+module Cfhighlander
 
 
   module Helper
@@ -11,7 +11,7 @@ module Highlander
       end
 
       if value.class == Hash
-        return value.to_s
+        return value.to_json
       end
 
       return "'#{value}'"

data/lib/{highlander.mapproviders.rb → cfhighlander.mapproviders.rb}

@@ -1,4 +1,4 @@
-module Highlander
+module Cfhighlander
 
   ### Map Providers provide predefined maps or list of maps
   ### Aside from predefined mappings from providers

data/lib/cfhighlander.model.component.rb

@@ -0,0 +1,177 @@
+require 'yaml'
+
+module Cfhighlander
+
+  module Model
+
+    class Component
+
+      attr_accessor :component_dir,
+          :config,
+          :highlander_dsl_path,
+          :cfndsl_path,
+          :highlander_dsl,
+          :cfndsl_content,
+          :mappings,
+          :name,
+          :template,
+          :version,
+          :distribution_bucket,
+          :distribution_prefix,
+          :component_files,
+          :cfndsl_ext_files,
+          :lambda_src_files
+
+      attr_reader :cfn_model, :outputs
+
+      def initialize(template_meta, component_name)
+        @template = template_meta
+        @name = component_name
+        @component_dir = template_meta.template_location
+        @mappings = {}
+        @version = template_meta.template_version
+        @distribution_bucket = nil
+        @distribution_prefix = nil
+        @component_files = []
+        @cfndsl_ext_files = []
+        @lambda_src_files = []
+      end
+
+      # load component configuration files
+      def load_config()
+        @config = {} if @config.nil?
+        Dir["#{@component_dir}/*.config.yaml"].each do |config_file|
+          puts "INFO Loading config for #{@name}: read file:#{config_file} "
+          partial_config = YAML.load(File.read(config_file))
+          unless partial_config.nil?
+            @config.extend(partial_config)
+            @component_files << config_file
+          end
+        end
+      end
+
+      # load extensions
+      def loadDepandantExt()
+        @highlander_dsl.dependson_components.each do |requirement|
+          requirement.component_loaded.cfndsl_ext_files.each do |file|
+            cfndsl_ext_files << file
+          end
+        end
+      end
+
+      # evaluate components template
+      # @param [Hash] config_override
+      def load(config_override = nil)
+        if @component_dir.start_with? 'http'
+          raise StandardError, 'http(s) sources not supported yet'
+        end
+
+        legacy_cfhighlander_path = "#{@component_dir}/#{@template.template_name}.highlander.rb"
+        if File.exist? legacy_cfhighlander_path
+          STDERR.puts "DEPRECATED: #{legacy_cfhighlander_path} - Use *.cfhiglander.rb"
+          @highlander_dsl_path = legacy_cfhighlander_path
+        else
+          @highlander_dsl_path = "#{@component_dir}/#{@template.template_name}.cfhighlander.rb"
+        end
+
+        @cfndsl_path = "#{@component_dir}/#{@template.template_name}.cfndsl.rb"
+        candidate_mappings_path = "#{@component_dir}/*.mappings.yaml"
+        candidate_dynamic_mappings_path = "#{@component_dir}/#{@template.template_name}.mappings.rb"
+
+        @cfndsl_ext_files += Dir["#{@component_dir}/ext/cfndsl/*.rb"]
+        @lambda_src_files += Dir["#{@component_dir}/lambdas/**/*"].find_all {|p| not File.directory? p}
+        @component_files += @cfndsl_ext_files
+        @component_files += @lambda_src_files
+
+        @config = {} if @config.nil?
+
+        @config.extend config_override unless config_override.nil?
+        @config['component_version'] = @version unless @version.nil?
+        @config['component_name'] = @name
+        @config['template_name'] = @template.template_name
+        @config['template_version'] = @template.template_version
+
+
+        Dir[candidate_mappings_path].each do |mapping_file|
+          mappings = YAML.load(File.read(mapping_file))
+          @component_files << mapping_file
+          mappings.each do |name, map|
+            @mappings[name] = map
+          end unless mappings.nil?
+        end
+
+        if File.exist? candidate_dynamic_mappings_path
+          require candidate_dynamic_mappings_path
+          @component_files << candidate_dynamic_mappings_path
+        end
+
+        # 1st pass - parse the file
+        @component_files << @highlander_dsl_path
+        @highlander_dsl = eval(File.read(@highlander_dsl_path), binding)
+        # set version if not defined
+        @highlander_dsl.ComponentVersion(@version) unless @version.nil?
+
+
+        if @highlander_dsl.description.nil?
+          if template.template_name == @name
+            description = "#{@name}@#{template.template_version} - v#{@highlander_dsl.version}"
+          else
+            description = "#{@name} - v#{@highlander_dsl.version}"
+            description += " (#{template.template_name}@#{template.template_version})"
+          end
+
+          @highlander_dsl.Description(description)
+        end
+
+        # set (override) distribution options
+        @highlander_dsl.DistributionBucket(@distribution_bucket) unless @distribution_bucket.nil?
+        @highlander_dsl.DistributionPrefix(@distribution_prefix) unless @distribution_prefix.nil?
+
+        if File.exist? @cfndsl_path
+          @component_files << @cfndsl_path
+          @cfndsl_content = File.read(@cfndsl_path)
+          @cfndsl_content.strip!
+          # if there is CloudFormation do [content] end extract only contents
+          ### Regex \s is whitespace
+          match_data = /^CloudFormation do\s(.*)end\s?$/m.match(@cfndsl_content)
+          if not match_data.nil?
+            @cfndsl_content = match_data[1]
+          end
+
+        else
+          @cfndsl_content = ''
+        end
+
+        loadDepandantExt()
+      end
+
+      # evaluates cfndsl with current config
+      def eval_cfndsl
+        compiler = Cfhighlander::Compiler::ComponentCompiler.new self
+        @cfn_model = compiler.evaluateCloudFormation().as_json
+        @outputs = (
+        if @cfn_model.key? 'Outputs'
+        then
+          @cfn_model['Outputs'].map {|k, v| ComponentOutput.new self, k, v}
+        else
+          []
+        end
+        )
+      end
+    end
+
+    class ComponentOutput
+
+      attr_reader :component, :name, :value
+
+      def initialize(component, name, value)
+        @component = component
+        @name = name
+        @value = value
+      end
+    end
+
+  end
+
+end
+

data/lib/cfhighlander.model.templatemeta.rb

@@ -0,0 +1,25 @@
+module Cfhighlander
+
+  module Model
+
+    class TemplateMetadata
+
+      @template_name
+      @template_version
+      @template_location
+
+      attr_reader :template_location,
+          :template_version,
+          :template_name
+
+      def initialize(template_name:, template_version:, template_location:)
+        @template_name = template_name
+        @template_version = template_version
+        @template_location = template_location
+      end
+
+    end
+
+  end
+
+end

data/lib/{highlander.publisher.rb → cfhighlander.publisher.rb}

@@ -1,11 +1,11 @@
-require_relative './highlander.compiler'
+require_relative './cfhighlander.compiler'
 require 'aws-sdk-s3'
 
-module Highlander
+module Cfhighlander
 
   module Publisher
 
-    class Component
+    class ComponentPublisher
 
       def initialize(component, cleanup)
         @component = component

data/lib/{highlander.validator.rb → cfhighlander.validator.rb}

@@ -2,7 +2,7 @@ require 'aws-sdk-cloudformation'
 require 'aws-sdk-s3'
 require 'digest/md5'
 
-module Highlander
+module Cfhighlander
 
   module Cloudformation
 

data/lib/util/zip.util.rb

@@ -1,7 +1,7 @@
 require 'zip'
 
 
-module Highlander
+module Cfhighlander
   module Util
     ###
     ### taken from https://github.com/rubyzip/rubyzip/tree/master/samples

data/templates/cfndsl.component.template.erb

@@ -8,21 +8,21 @@ CloudFormation do
 <% end %>
 
 # render subcomponents
-<% for @component in dsl.components %>
-    CloudFormation_Stack('<%= @component.name.gsub('-','').gsub('_','') %>') do
-    TemplateURL '<%= @component.distribution_url %>'
-    Parameters ({
-    <% for @param in @component.parameters %>
-        '<%= @param.name %>' => <%= @param.cfndsl_value %>,
-    <% end %>
-    })
+<% for @subcomponent in dsl.subcomponents %>
+    CloudFormation_Stack('<%= @subcomponent.cfn_name %>') do
+        TemplateURL '<%= @subcomponent.distribution_url %>'
+        Parameters ({
+        <% for @param in @subcomponent.parameters %><%="\t"%>'<%=@param.name %>' => <%= @param.cfndsl_value %>,
+        <% end %>})
+        <% if @subcomponent.conditional %>Condition 'Enable<%= @subcomponent.cfn_name %>' <% end %>
     end
 <% end %>
 
 <%= component_cfndsl %>
 
-# render lambda functions
+
 <% unless dsl.lambda_functions_keys.nil? %>
+    # cfhighlander generated lambda functions
     <% for @key in dsl.lambda_functions_keys %>
         render_lambda_functions(self,
         <%= @key %>,
@@ -30,16 +30,17 @@ CloudFormation do
         {'bucket'=>'<%= dsl.distribution_bucket %>','prefix' => '<%= dsl.distribution_prefix %>', 'version'=>'<%= dsl.version %>'})
     <% end %>
 <% end %>
-# render parameters at the end, overriding any parameters defined in template itself
+    # cfhighlander generated parameters
 <% for @param in dsl.parameters.param_list %>
     Parameter('<%= @param.name %>') do
       Type '<%= @param.type %>'
       Default '<%= @param.default_value %>'
-      <% unless @param.no_echo.nil? %>
-      NoEcho <%= @param.no_echo.to_s %>
-      <% end %>
+      <% unless @param.no_echo.nil? %>NoEcho <%= @param.no_echo.to_s %> <% end %>
+      <% unless @param.allowed_values.nil? %>AllowedValues <%= @param.allowed_values.to_s %> <% end %>
     end
 <% end %>
 
-  Description '<%= dsl.description %>'
+<% for @condition in dsl.conditions %>    Condition('<%= @condition.name %>', <%= @condition.expression %>)
+<% end %>
+    Description '<%= dsl.description %>'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cfhighlander
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0.alpha.1528936037
 platform: ruby
 authors:
 - Nikola Tosic
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-28 00:00:00.000000000 Z
+date: 2018-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: highline
@@ -191,6 +191,26 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  name: netaddr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.1
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -214,7 +234,7 @@ extra_rdoc_files: []
 files:
 - README.md
 - bin/cfhighlander
-- bin/highlander.rb
+- bin/cfhighlander.rb
 - cfndsl_ext/config/managed_policies.yaml
 - cfndsl_ext/iam_helper.rb
 - cfndsl_ext/lambda_helper.rb
@@ -223,16 +243,19 @@ files:
 - hl_ext/common_helper.rb
 - hl_ext/mapping_helper.rb
 - hl_ext/vpc.rb
-- lib/highlander.compiler.rb
-- lib/highlander.dsl.base.rb
-- lib/highlander.dsl.component.rb
-- lib/highlander.dsl.params.rb
-- lib/highlander.dsl.rb
-- lib/highlander.factory.rb
-- lib/highlander.helper.rb
-- lib/highlander.mapproviders.rb
-- lib/highlander.publisher.rb
-- lib/highlander.validator.rb
+- lib/cfhighlander.compiler.rb
+- lib/cfhighlander.dsl.base.rb
+- lib/cfhighlander.dsl.params.rb
+- lib/cfhighlander.dsl.subcomponent.rb
+- lib/cfhighlander.dsl.template.rb
+- lib/cfhighlander.factory.rb
+- lib/cfhighlander.factory.templatefinder.rb
+- lib/cfhighlander.helper.rb
+- lib/cfhighlander.mapproviders.rb
+- lib/cfhighlander.model.component.rb
+- lib/cfhighlander.model.templatemeta.rb
+- lib/cfhighlander.publisher.rb
+- lib/cfhighlander.validator.rb
 - lib/util/zip.util.rb
 - templates/cfndsl.component.template.erb
 homepage: https://github.com/theonestack/cfhighlander/blob/master/README.md
@@ -250,9 +273,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.7.7