sfn 3.0.10 → 3.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1dbf7d97959e1120a0d7b1194371c15b7a483ebd
-  data.tar.gz: beaddca76c30c1d453449939de41693a6ec84554
+  metadata.gz: 844e08d945b817d040cd227631e30a62bb2bc591
+  data.tar.gz: a0b766931a02119319fa21c1b1e9158706ec2320
 SHA512:
-  metadata.gz: ab20f19b50ed8db52d845c51a2194720b875627225d2a44574124b8a06e5a87ebc55382520e4368e068776703d3649738f99e6bfc41b0f074735a5eb33152496
-  data.tar.gz: 5bce247327ec8b0cbd793c3b28a9b4a994dda012f9f5fb0031e48e17a1306826ff8678c90a7255ffada556c9d35af4677ce577c7fa9b3679b3fd08fff0ea529a
+  metadata.gz: 1fcb7c461101e3c7de5e253e4e024c3cbcd05c41277ae258e325f696a4dc88c10e74d491ea932be375075e43e7587409402c7147d7aa78f235f1348fb9d800b5
+  data.tar.gz: f5944d0b75b1356a5467ca6180ccecd68e5661f888982726f13d1dad7b3f8e98ffbac0fd78ac267a18c7192c87749adb70e4c2137c0cd5e75488970b78dbcf22

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# v3.0.12
+* [feature] New linting foundation and `lint` command (#208)
+
 # v3.0.10
 * [fix] Template prompting within collections (#204)
 * [enhancement] Stack parameter matching via configuration (#203)

data/docs/README.md

@@ -13,3 +13,4 @@
   - [Enabling Callbacks](callbacks.md#enabling-callbacks)
   - [Builtin Callbacks](callbacks.md#builtin-callbacks)
   - [Custom Callbacks](callbacks.md#custom-callbacks)
+- [SparklePacks](sparkle-packs.md)

data/docs/lint.md

@@ -0,0 +1,72 @@
+---
+title: "Lint"
+weight: 5
+---
+
+## Lint
+
+The lint framework built within the sfn tool utilizes the JMESPath query language
+for identifying patterns and apply validation rules.
+
+### Lint RuleSets
+
+#### Local
+
+Create rule sets using the generator. Each file must contain a single
+lint rule set. Below is a simple rule set used to flag non-AWS type
+resources:
+
+~~~ruby
+# tests/lint/resource_type_check.rb
+
+RuleSet.build(:test) do
+  rule :aws_resources_only do
+    definition 'Resources.[*][0][*].Type' do |search|
+      unless(search.nil?)
+        result = search.find_all{|i| !i.start_with?('AWS')}
+        result.empty? ? true : result
+      else
+        true
+      end
+    end
+
+    fail_message 'All types must be within AWS root namespace'
+  end
+end
+~~~
+
+#### Library
+
+When a rule set is defined within a library, it must use the full constant namespace and
+must register to allow access to the rule set:
+
+~~~ruby
+my_ruleset = Sfn::Lint::RuleSet.build(:test) do
+  rule :aws_resources_only do
+    definition 'Resources.[*][0][*].Type' do |search|
+      unless(search.nil?)
+        result = search.find_all{|i| !i.start_with?('AWS')}
+        result.empty? ? true : result
+      else
+        true
+      end
+    end
+
+    fail_message 'All types must be within AWS root namespace'
+  end
+end
+Sfn::Lint::RuleSet.register(my_ruleset)
+~~~
+
+### Usage
+
+By default `sfn` will apply any registered rule sets that are defined for the target provider.
+
+#### Local
+
+Provide a template and lint directory to the `lint` command. For example,
+if lint rule sets are defined within `tests/lint`:
+
+~~~
+$ sfn lint --file my-template --lint-directory tests/lint
+~~~

data/docs/sparkle-packs.md

@@ -0,0 +1,75 @@
+---
+title: "SparklePacks"
+weight: 4
+anchors:
+  - title: "Enabling SparklePacks"
+    url: "#enabling-sparklepacks"
+---
+
+## What is a SparklePack?
+
+SparklePacks are implemented as a feature of the SparkleFormation library,
+providing a means to package SparkleFormation building blocks
+and templates as reusable, redistributable software artifacts.
+A SparklePack may package up any combination of SparkleFormation
+[building blocks](http://www.sparkleformation.io/docs/sparkle_formation/building-blocks.html) and templates.
+
+sfn supports loading SparklePacks distributed as [Ruby gems](http://www.sparkleformation.io/docs/sparkle_formation/sparkle-packs.html#distribution).
+You can find published SparklePacks on the RubyGems site by
+[searching for the sparkle-pack prefix](https://rubygems.org/search?query=sparkle-pack).
+
+### Enabling SparklePacks
+
+The following examples use the [sparkle-pack-aws-availability-zones](https://rubygems.org/gems/sparkle-pack-aws-availability-zones) gem.
+In reviewing [the source code of that project on Github](https://github.com/hw-labs/sparkle-pack-aws-availability-zones),
+note that it provides a [`zones` registry](https://github.com/hw-labs/sparkle-pack-aws-availability-zones/blob/v0.1.2/lib/sparkleformation/registry/get_azs.rb)
+which uses the aws-sdk-core library to return an array of available AZs.
+
+When using sfn with Bundler, we'll add any SparklePacks we
+want to enable to the `sfn` group in our Gemfile:
+
+~~~ruby
+# Gemfile
+source 'https://rubygems.org'
+
+gem 'sfn'
+
+group :sfn do
+  gem 'sparkle-pack-aws-availability-zones'
+end
+~~~
+
+After running `bundle`, the SparklePack is installed but not yet enabled:
+
+~~~
+$ cat sparkleformation/zones_test.rb
+SparkleFormation.new(:zones_test) do
+  zones registry!(:zones)
+end
+
+$ bundle exec sfn print --file zones_test
+ERROR: SparkleFormation::Error::NotFound::Registry: Failed to locate item named: `zones`
+~~~
+
+Adding the gem to an array of `sparkle_packs` in
+the `.sfn` configuration file will activate it for use:
+
+~~~ruby
+Configuration.new do
+  sparkle_pack [ 'sparkle-pack-aws-availability-zones' ]
+end
+~~~
+
+Invoking `zones` registry in the template is now functional:
+
+~~~
+$ bundle exec sfn print --file zones_test
+{
+  "Zones": [
+    "us-east-1a",
+    "us-east-1b",
+    "us-east-1c",
+    "us-east-1e"
+  ]
+}
+~~~

data/lib/sfn.rb

@@ -17,5 +17,6 @@ module Sfn
   autoload :Command, 'sfn/command'
   autoload :CommandModule, 'sfn/command_module'
   autoload :Planner, 'sfn/planner'
+  autoload :Lint, 'sfn/lint'
 
 end

data/lib/sfn/command.rb

@@ -17,6 +17,7 @@ module Sfn
     autoload :Import, 'sfn/command/import'
     autoload :Init, 'sfn/command/init'
     autoload :Inspect, 'sfn/command/inspect'
+    autoload :Lint, 'sfn/command/lint'
     autoload :List, 'sfn/command/list'
     autoload :Print, 'sfn/command/print'
     autoload :Promote, 'sfn/command/promote'

data/lib/sfn/command/lint.rb

@@ -0,0 +1,87 @@
+require 'sfn'
+
+module Sfn
+  class Command
+    # Lint command
+    class Lint < Command
+
+      include Sfn::CommandModule::Base
+      include Sfn::CommandModule::Template
+
+      # Perform linting
+      def execute!
+        print_only_original = config[:print_only]
+        config[:print_only] = true
+        file = load_template_file
+        ui.info "#{ui.color("Template Linting (#{provider.connection.provider}): ", :bold)} #{config[:file].sub(Dir.pwd, '').sub(%r{^/}, '')}"
+        config[:print_only] = print_only_original
+
+        raw_template = parameter_scrub!(template_content(file))
+
+        if(config[:print_only])
+          ui.puts raw_template
+        else
+          result = lint_template(raw_template)
+          if(result == true)
+            ui.puts ui.color('  -> VALID', :green, :bold)
+          else
+            ui.puts ui.color('  -> INVALID', :red, :bold)
+            result.each do |failure|
+              ui.error "Result Set: #{ui.color(failure[:rule_set].name, :red, :bold)}"
+              failure[:failures].each do |f_msg|
+                ui.puts "#{ui.color('  *', :red, :bold)} #{f_msg}"
+              end
+            end
+            raise 'Linting failure'
+          end
+        end
+      end
+
+      # Apply linting to given template
+      #
+      # @param template [Hash]
+      # @return [TrueClass, Array<Smash[:rule_set, :failures]>]
+      def lint_template(template)
+        results = rule_sets.map do |set|
+          result = set.apply(template)
+          unless(result == true)
+            Smash.new(:rule_set => set, :failures => result)
+          end
+        end.compact
+        results.empty? ? true : results
+      end
+
+      # @return [Array<Sfn::Lint::RuleSet>]
+      def rule_sets
+        sets = [config[:lint_directory]].flatten.compact.map do |directory|
+          if(File.directory?(directory))
+            files = Dir.glob(File.join(directory, '**', '**', '*.rb'))
+            files.map do |path|
+              begin
+                Sfn::Lint.class_eval(
+                  IO.read(path), path, 1
+                )
+              rescue
+                ui.warn "Failed to load detected file: #{path}"
+                nil
+              end
+            end
+          end
+        end.flatten.compact.find_all{|rs| rs.provider == provider.connection.provider}
+        unless(config[:local_rule_sets_only])
+          sets += Sfn::Lint::RuleSet.get_all(provider.connection.provider)
+        end
+        if(config[:disabled_rule_set])
+          disabled = [config[:disabled_rule_set]].flatten.compact
+          sets.delete_if{|i| disabled.include?(i.name.to_s) }
+        end
+        if(config[:enabled_rule_set])
+          enabled = [config[:enabled_rule_set]].flatten.compact
+          sets.delete_if{|i| enabled.include?(i.name.to_s) }
+        end
+        sets
+      end
+
+    end
+  end
+end

data/lib/sfn/config.rb

@@ -52,6 +52,7 @@ module Sfn
     autoload :Import, 'sfn/config/import'
     autoload :Init, 'sfn/config/init'
     autoload :Inspect, 'sfn/config/inspect'
+    autoload :Lint, 'sfn/config/lint'
     autoload :List, 'sfn/config/list'
     autoload :Print, 'sfn/config/print'
     autoload :Promote, 'sfn/config/promote'

data/lib/sfn/config/lint.rb

@@ -0,0 +1,28 @@
+require 'sfn'
+
+module Sfn
+  class Config
+    # Lint command configuration
+    class Lint < Validate
+      attribute(
+        :lint_directory, String,
+        :description => 'Directory containing lint rule sets',
+        :multiple => true
+      )
+      attribute(
+        :disabled_rule_set, String,
+        :description => 'Disable rule set from being applied',
+        :multiple => true
+      )
+      attribute(
+        :enabled_rule_set, String,
+        :description => 'Only apply this rule set',
+        :multiple => true
+      )
+      attribute(
+        :local_rule_sets_only, [TrueClass, FalseClass],
+        :description => 'Only apply rule sets provided by lint directory'
+      )
+    end
+  end
+end

data/lib/sfn/lint.rb

@@ -0,0 +1,12 @@
+require 'sfn'
+require 'jmespath'
+
+module Sfn
+  module Lint
+
+    autoload :Definition, 'sfn/lint/definition'
+    autoload :Rule, 'sfn/lint/rule'
+    autoload :RuleSet, 'sfn/lint/rule_set'
+
+  end
+end

data/lib/sfn/lint/definition.rb

@@ -0,0 +1,57 @@
+require 'sfn'
+
+module Sfn
+  module Lint
+    # Lint defition
+    class Definition
+
+      # @return [String] search expression used for matching
+      attr_reader :search_expression
+      # @return [Proc-ish] must respond to #call
+      attr_reader :evaluator
+      # @return [Symbol] target provider
+      attr_reader :provider
+
+      # Create a new definition
+      #
+      # @param expr [String] search expression used for matching
+      # @param provider [String, Symbol] target provider
+      # @param evaluator [Proc] logic used to handle match
+      # @return [self]
+      def initialize(expr, provider=:aws, evaluator=nil, &block)
+        if(evaluator && block)
+          raise ArgumentError.new 'Only evaluator or block can be provided, not both.'
+        end
+        @provider = Bogo::Utility.snake(provider).to_sym
+        @search_expression = expr
+        @evaluator = evaluator || block
+      end
+
+      # Apply definition to template
+      #
+      # @param template [Hash] template being processed
+      # @return [TrueClass, Array<String>] true if passed. List of string results that failed
+      def apply(template)
+        result = JMESPath.search(search_expression, template)
+        run(result, template)
+      end
+
+      protected
+
+      # Check result of search expression
+      #
+      # @param result [Object] result(s) of search expression
+      # @param template [Hash] full template
+      # @return [TrueClass, Array<String>] true if passed. List of string results that failed
+      # @note override this method when subclassing
+      def run(result, template)
+        unless(evaluator)
+          raise NotImplementedError.new 'No evaluator has been defined for this definition!'
+        end
+        evaluator.call(result, template)
+      end
+
+    end
+
+  end
+end

data/lib/sfn/lint/rule.rb

@@ -0,0 +1,118 @@
+require 'sfn'
+
+module Sfn
+  module Lint
+    # Composition of definitions
+    class Rule
+
+      # @return [Symbol] name of rule
+      attr_reader :name
+      # @return [Array<Definition>] definitions composing rule
+      attr_reader :definitions
+      # @return [String] message describing failure
+      attr_reader :fail_message
+      # @return [Symbol] target provider
+      attr_reader :provider
+
+      # Create a new rule
+      #
+      # @param name [String, Symbol] name of rule
+      # @param definitions [Array<Definition>] definitions composing rule
+      # @param fail_message [String] message to describe failure
+      # @param provider [String, Symbol] target provider
+      # @return [self]
+      def initialize(name, definitions, fail_message, provider=:aws)
+        @name = name.to_sym
+        @definitions = definitions.dup.uniq.freeze
+        @fail_message = fail_message
+        @provider = Bogo::Utility.snake(provider).to_sym
+        validate_definitions!
+      end
+
+      # Generate the failure message for this rule with given failure
+      # result set.
+      def generate_fail_message(results)
+        msg = fail_message.dup
+        unless(results.empty?)
+          failed_items = results.map do |item|
+            f_item = item[:failures]
+            next if f_item.nil? || f_item == true || f_item == false
+            f_item
+          end.flatten.compact.map(&:to_s)
+          unless(failed_items.empty?)
+            msg = "#{msg} (failures: `#{failed_items.join('`, `')}`)"
+          end
+        end
+        msg
+      end
+
+      # Apply all definitions to template
+      #
+      # @param template [Hash]
+      # @return [TrueClass, Array<Smash[:definition, :failures]>] true if passed. Definition failures if failed.
+      def apply(template)
+        results = definitions.map do |definition|
+          result = definition.apply(template)
+          result == true ? result : Smash.new(:definition => definition, :failures => result)
+        end
+        if(results.all?{|item| item == true})
+          true
+        else
+          results.delete_if{|item| item == true}
+          results
+        end
+      end
+
+      # Check if template passes this rule
+      #
+      # @param template [Hash]
+      # @return [TrueClass, FalseClass]
+      def pass?(template)
+        apply(template) == true
+      end
+
+      # Check if template fails this rule
+      #
+      # @param template [Hash]
+      # @return [TrueClass, FalseClass]
+      def fail?(template)
+        !pass?(template)
+      end
+
+      # Add a new definition to the collection
+      #
+      # @param definition [Definition] new definition to add
+      # @return [self]
+      def add_definition(definition)
+        new_defs = definitions.dup
+        new_defs << definition
+        @definitions = new_defs.uniq.freeze
+        validate_definitions!
+        self
+      end
+
+      # Remove a definition from the collection
+      #
+      # @param definition [Definition] definition to remove
+      # @return [self]
+      def remove_definition(definition)
+        new_defs = definitions.dup
+        new_defs.delete(definition)
+        @definitions = new_defs.uniq.freeze
+        self
+      end
+
+      # Check that provided definitions provider match rule defined provider
+      def validate_definitions!
+        non_match = definitions.find_all do |definition|
+          definition.provider != provider
+        end
+        unless(non_match.empty?)
+          raise ArgumentError.new "Rule defines `#{provider}` as provider but includes definitions for " \
+            "non matching providers. (#{non_match.map(&:provider).map(&:to_s).uniq.sort.join(', ')})"
+        end
+      end
+
+    end
+  end
+end

data/lib/sfn/lint/rule_set.rb

@@ -0,0 +1,177 @@
+require 'sfn'
+
+module Sfn
+  module Lint
+    # Named collection of rules
+    class RuleSet
+
+      # Helper class for ruleset generation
+      class Creator
+
+        attr_reader :items, :provider
+
+        def initialize(provider)
+          @provider = provider
+          @items = []
+        end
+
+        class RuleSet < Creator
+
+          def rule(name, &block)
+            r = Rule.new(provider)
+            r.instance_exec(&block)
+            items << Sfn::Lint::Rule.new(name, r.items, r.fail_message, provider)
+          end
+
+        end
+
+        class Rule < Creator
+
+          def definition(expr, evaluator=nil, &block)
+            items << Sfn::Lint::Definition.new(expr, provider, evaluator, &block)
+          end
+
+          def fail_message(val=nil)
+            unless(val.nil?)
+              @fail_message = val
+            end
+            @fail_message
+          end
+
+        end
+
+      end
+
+      class << self
+
+        @@_rule_set_registry = Smash.new
+
+        # RuleSet generator helper for quickly building simple rule sets
+        #
+        # @param name [String] name of rule set
+        # @param provider [String, Symbol] target provider
+        # @yieldblock rule set content
+        def build(name, provider=:aws, &block)
+          provider = Bogo::Utility.snake(provider).to_sym
+          rs = Creator::RuleSet.new(provider)
+          rs.instance_exec(&block)
+          self.new(name, provider, rs.items)
+        end
+
+        # Register a rule set
+        #
+        # @param rule_set [RuleSet]
+        # @return [TrueClass]
+        def register(rule_set)
+          @@_rule_set_registry.set(rule_set.provider, rule_set.name, rule_set)
+          true
+        end
+
+        # Get registered rule set
+        #
+        # @param name [String] name of rule set
+        # @param provider [String] target provider
+        # @return [RuleSet, NilClass]
+        def get(name, provider=:aws)
+          provider = Bogo::Utility.snake(provider)
+          @@_rule_set_registry.get(provider, name)
+        end
+
+        # Get all rule sets for specified provider
+        #
+        # @param provider [String] target provider
+        # @return [Array<RuleSet>]
+        def get_all(provider=:aws)
+          @@_rule_set_registry.fetch(provider, {}).values
+        end
+
+      end
+
+      include Bogo::Memoization
+
+      # @return [Symbol] name
+      attr_reader :name
+      # @return [Symbol] target provider
+      attr_reader :provider
+      # @return [Array<Rule>] rules of set
+      attr_reader :rules
+
+      # Create new rule set
+      #
+      # @param name [String, Symbol] name of rule set
+      # @param provider [String, Symbol] name of target provider
+      # @param rules [Array<Rule>] list of rules defining this set
+      # @return [self]
+      def initialize(name, provider=:aws, rules=[])
+        @name = name.to_sym
+        @provider = Bogo::Utility.snake(provider).to_sym
+        @rules = rules.dup.uniq.freeze
+        validate_rules!
+      end
+
+      # Add a new rule to the collection
+      #
+      # @param rule [Rule] new rule to add
+      # @return [self]
+      def add_rule(rule)
+        new_rules = rules.dup
+        new_rules << rule
+        @rules = new_rules.uniq.freeze
+        validate_rules!
+        self
+      end
+
+      # Remove a rule from the collection
+      #
+      # @param rule [Rule] rule to remove
+      # @return [self]
+      def remove_rule(rule)
+        new_rules = rules.dup
+        new_rules.delete(rule)
+        @rules = new_rules.uniq.freeze
+        self
+      end
+
+      # Apply rule set to template.
+      #
+      # @param template [Hash]
+      # @return [TrueClass, Array<String>] true on success, list failure messages on failure
+      def apply(template)
+        failures = collect_failures(template)
+        if(failures.empty?)
+          true
+        else
+          failures.map do |failure|
+            failure[:rule].generate_fail_message(failure[:result])
+          end
+        end
+      end
+
+      # Process template through rules defined in this set and
+      # store failure information
+      #
+      # @param template [Hash]
+      # @return [Array<Rule>] list of failures
+      def collect_failures(template)
+        results = rules.map do |rule|
+          result = rule.apply(template)
+          result == true ? true : Smash.new(:rule => rule, :result => result)
+        end
+        results.delete_if{|i| i == true}
+        results
+      end
+
+      # Check that provided rules provider match rule set defined provider
+      def validate_rules!
+        non_match = rules.find_all do |rule|
+          rule.provider != provider
+        end
+        unless(non_match.empty?)
+          raise ArgumentError.new "Rule set defines `#{provider}` as provider but includes rules for " \
+            "non matching providers. (#{non_match.map(&:provider).map(&:to_s).uniq.sort.join(', ')})"
+        end
+      end
+
+    end
+  end
+end

data/lib/sfn/version.rb

@@ -1,4 +1,4 @@
 module Sfn
   # Current library version
-  VERSION = Gem::Version.new('3.0.10')
+  VERSION = Gem::Version.new('3.0.12')
 end

data/sfn.gemspec

@@ -18,6 +18,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'miasma-open-stack', '>= 0.1.0', '< 0.3'
   s.add_runtime_dependency 'miasma-rackspace', '>= 0.1.0', '< 0.3'
   s.add_runtime_dependency 'miasma-google', '>= 0.1.0', '< 0.3'
+  s.add_runtime_dependency 'jmespath'
   s.add_runtime_dependency 'net-ssh'
   s.add_runtime_dependency 'sparkle_formation', '>= 3.0.3', '< 4'
   s.add_runtime_dependency 'hashdiff', '~> 0.2.2'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sfn
 version: !ruby/object:Gem::Version
-  version: 3.0.10
+  version: 3.0.12
 platform: ruby
 authors:
 - Chris Roberts
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-19 00:00:00.000000000 Z
+date: 2016-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bogo-cli
@@ -170,6 +170,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: jmespath
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement
@@ -317,7 +331,9 @@ files:
 - docs/images/d_list.png
 - docs/images/d_update.png
 - docs/images/d_validate.png
+- docs/lint.md
 - docs/overview.md
+- docs/sparkle-packs.md
 - docs/usage.md
 - docs/v/0.3.2/marked.js
 - docs/v/bootstrap.min.css
@@ -347,6 +363,7 @@ files:
 - lib/sfn/command/import.rb
 - lib/sfn/command/init.rb
 - lib/sfn/command/inspect.rb
+- lib/sfn/command/lint.rb
 - lib/sfn/command/list.rb
 - lib/sfn/command/print.rb
 - lib/sfn/command/promote.rb
@@ -369,11 +386,16 @@ files:
 - lib/sfn/config/import.rb
 - lib/sfn/config/init.rb
 - lib/sfn/config/inspect.rb
+- lib/sfn/config/lint.rb
 - lib/sfn/config/list.rb
 - lib/sfn/config/print.rb
 - lib/sfn/config/promote.rb
 - lib/sfn/config/update.rb
 - lib/sfn/config/validate.rb
+- lib/sfn/lint.rb
+- lib/sfn/lint/definition.rb
+- lib/sfn/lint/rule.rb
+- lib/sfn/lint/rule_set.rb
 - lib/sfn/monkey_patch.rb
 - lib/sfn/monkey_patch/stack.rb
 - lib/sfn/monkey_patch/stack/azure.rb