cfndsl 0.17.5 → 1.0.0.pre.1

data/lib/cfndsl/rake_task.rb

@@ -1,28 +1,225 @@
 # frozen_string_literal: true
 
-require 'rake'
 require 'rake/tasklib'
+require_relative 'globals'
+require_relative 'specification'
 
-require 'cfndsl'
+# Monkey patch Rake
+module Rake
+  # Default verbosity to false
+  def self.verbose?
+    verbose && verbose != FileUtilsExt::DEFAULT
+  end
+end
 
 module CfnDsl
-  # Rake Task
+  # Rake TaskLib
+  # rubocop:disable Metrics/ClassLength
   class RakeTask < Rake::TaskLib
+    # @deprecated pre 1.x rake task generator
     attr_accessor :cfndsl_opts
 
+    # Creates Cloudformation generation tasks
+    #
+    # @example
+    # directory 'tmp'
+    #
+    # namespace :samples do
+    #
+    #   CfnDsl::RakeTask.new do |t|
+    #     t.specification(file: 'tmp/cloudformation_resources.json')
+    #
+    #     desc 'Generate CloudFormation Json'
+    #     t.json(name: :json, files: ["sample/*.rb"], pathmap: 'tmp/%f.json', pretty: true, extras: FileList.new('sample/*.yaml') )
+    #
+    #     t.yaml(name: :yaml, files: 'sample/t1.rb', pathmap: 'tmp/%f.yaml', extras: '%X.yaml')
+    #   end
+    # end
     def initialize(name = nil)
+      @tasks = []
+      @spec_task = nil
+
+      last_desc = ::Rake.application.last_description
+      desc nil
+
       yield self if block_given?
 
-      desc 'Generate Cloudformation' unless ::Rake.application.last_description
-      task(name || :generate) do |_t, _args|
-        cfndsl_opts[:files].each do |opts|
-          generate(opts)
-        end
+      if cfndsl_opts
+        desc last_desc if last_desc
+        task(name || :generate) { |_t, _args| cfndsl_opts[:files].each { |opts| generate(opts) } }
+      else
+        define_base_tasks(name || :generate, last_desc)
+      end
+    end
+
+    # Use a custom specification file
+    #
+    # This specification file will be used for any generation tasks
+    #
+    # Creates a file task to download from upstream source, with an optional additional named task that depends on it.
+    #
+    # The minimum required version can be specified with the version parameter, which can be overriden by invoking
+    # rake with the 'cfn_spec_task' argument
+    #
+    # @example
+    # CfnDsl::RakeTask.new() do |t|
+    #     t.specification(name: :update_spec, file: 'tmp/cloudformation_resources.json', version: '6.3'0)
+    # end
+    #
+    # # rake update_spec
+    # #      >> Will ensure 'tmp/cloudformation_resources.json' exists and is at least 6.3.0
+    # # rake update_spec[latest]
+    # #      >> Will always try to download the latest available spec
+    #
+    #
+    # @param [String] file the number of the specification file to use
+    # @param [Symbol] name A pretty name for the task to update this file from upstream source
+    # @param [String] version The minimum version required, and the default version used when downloading
+    #    if not specified and not overriden by 'cfn_spec_version' rake task argument
+    #    then any existing file is considered sufficient, and 'latest' is the version used for downloading
+    #
+    # @todo Add capability to provide a user spec/patches dir
+    def specification(name: nil, file:, version: nil)
+      if name
+        desc 'Update Resource Specification' unless ::Rake.application.last_description
+        task name, [:cfn_spec_version] => file
+      end
+
+      @spec_task = file(file, :cfn_spec_version) do |t, args|
+        update_specification(t.name, args.with_defaults(cfn_spec_version: version)[:cfn_spec_version])
+      end
+      @spec_task.define_singleton_method(:needed?) { true } # We always need to check
+      self
+    end
+
+    # Convert DSL sources to json
+    #
+    # Generates file tasks for each of the matching file in the files FileList.  Each task is dependant on the source,
+    #   the specification file, and the required extras files such that if the timestamp of any of these is earlier
+    #   the target file will be regenerated.
+    #
+    # Finally task <name> is generated that depends on all the generated targets.
+    #
+    # @param [Symbol|String] name the name of a task that is dependant on all the files being converted
+    # @param [Rake::FileList | Array<String> ] files source file list
+    # @param [String] pathmap expression to map source files to target files
+    # @param [Rake::Filelist|Array<String>] extras a list of files to load as external parameters
+    #   Note String values containing '%' are treated as a pathmap from the source
+    #   The resulting generated FileList is used as a dependency for generation so any entries not
+    #   containing '*' MUST exist.
+    # @param [Boolean] pretty use JSON.pretty_generate
+    # @param [Hash] json_opts, other options to pass to JSON generator
+    # rubocop:disable Metrics/ParameterLists
+    def json(name:, files:, pathmap:, pretty: false, extras: [], **json_opts)
+      json_method = pretty ? :pretty_generate : :generate
+      generate_model_tasks(name: name, files: files, pathmap: pathmap, extras: extras) do |model, f|
+        f.write(JSON.send(json_method, model, **json_opts))
       end
+      self
     end
 
+    # Convert DSL sources to yaml
+    #
+    # Generates file tasks for each of the matching file in the files FileList.  Each task is dependant on the source,
+    #   the specification file, and the required extras files such that if the timestamp of any of these is earlier
+    #   the target file will be regenerated.
+    #
+    # Finally task <name> is generated that depends on all the generated targets.
+    # @param [Symbol|String] name the name of a task that is dependant on all the files being converted
+    # @param [Rake::FileList | Array<String> ] files source file list
+    # @param [String] pathmap expression to map source files to target files
+    # @param [Rake::Filelist|Array<String>] extras a list of files to load as external parameters
+    #   Note String values containing '%' are treated as a pathmap from the source
+    #   The resulting generated FileList is used as a dependency for generation so any entries not
+    #   containing '*' MUST exist.
+    # @param [Hash] yaml_opts, other options to pass to YAML generator
+    def yaml(name:, files:, pathmap:, extras: [], **yaml_opts)
+      generate_model_tasks(name: name, files: files, pathmap: pathmap, extras: extras) do |model, f|
+        YAML.dump(model, f, **yaml_opts)
+      end
+      self
+    end
+    # rubocop:enable Metrics/ParameterLists
+
     private
 
+    attr_reader :tasks, :spec_task
+
+    def define_base_tasks(name, description)
+      return if tasks.empty?
+
+      desc description || 'Generate Cloudformation'
+      task(name || :generate, [:cfn_spec_version] => tasks)
+
+      load_spec = task load_spec_types: [spec_task].compact do |_t, args|
+        CfnDsl.specification_file = spec_task&.name || CfnDsl::LOCAL_SPEC_FILE
+        require_relative 'cloudformation'
+        CfnDsl::CloudFormationTemplate.check_types(file: spec_task&.name, version: @updated_version || args[:cfn_spec_version])
+        verbose&.puts "Using Cloudformation types generated from #{CfnDsl::CloudFormationTemplate.template_types.values_at('File', 'Version').join(' ')}"
+      end
+      load_spec.define_singleton_method(:timestamp) { Rake::EARLY } # Prevent this task from causing generation tasks.
+
+      tasks.unshift(load_spec)
+    end
+
+    def update_specification(file, minimum_version)
+      if CfnDsl::Specification.update_required?(version: minimum_version, file: file)
+
+        safe_update_embedded_spec if file == CfnDsl::LOCAL_SPEC_FILE
+
+        result = CfnDsl.update_specification_file(file: file, version: minimum_version)
+        puts "Specification #{result[:file]} updated to version #{result[:version]}"
+        @updated_version = result[:version]
+      elsif minimum_version
+        verbose&.puts "Specification #{file} is already >= #{minimum_version}"
+      end
+    end
+
+    def safe_update_embedded_spec
+      return if File.fnmatch?(File.join(File.dirname(File.expand_path(Rake.application.rakefile)), '*'), CfnDsl::LOCAL_SPEC_FILE)
+
+      raise Error, 'Refusing to update CloudFormation spec embedded in cfndsl gem'
+    end
+
+    def generate_model_tasks(name:, files:, pathmap:, extras:)
+      files = Rake::FileList.new(*files) unless files.is_a?(Rake::FileList)
+
+      tasks << task(name, [:cfn_spec_version] => files.pathmap(pathmap))
+
+      files.each do |source|
+        matched_extras = build_extras_filelist(source, extras)
+
+        file source.pathmap(pathmap) => [source, :load_spec_types, matched_extras] do |task|
+          eval_extras = matched_extras.map { |e| [:yaml, e] } # eval treats yaml and json same
+          puts "Generating Cloudformation for #{source} to #{task.name}"
+          model = CfnDsl.eval_file_with_extras(source, eval_extras, verbose)
+          File.open(task.name, 'w') { |f| yield model, f }
+        end
+      end
+
+      self
+    end
+
+    def build_extras_filelist(source, extras)
+      extras = [extras] unless extras.respond_to?(:each)
+      extras.each_with_object(Rake::FileList.new) do |extra, result|
+        case extra
+        when Rake::FileList
+          result.add(extra)
+        when Array
+          result.add(*extra)
+        when /%/
+          result.add(source.pathmap(extra))
+        else
+          result.add(extra)
+        end
+      end
+    end
+
+    def verbose
+      (Rake.verbose? || cfndsl_opts&.fetch(:verbose, false)) && STDERR
+    end
+
     def generate(opts)
       log(opts)
       outputter(opts) do |output|
@@ -55,12 +252,9 @@ module CfnDsl
       cfndsl_opts.fetch(:extras, [])
     end
 
-    def verbose
-      cfndsl_opts[:verbose] && STDERR
-    end
-
     def file_output(path)
       File.open(File.expand_path(path), 'w') { |f| yield f }
     end
   end
+  # rubocop:enable Metrics/ClassLength
 end

data/lib/cfndsl/ref_check.rb

@@ -3,26 +3,34 @@
 # This module defines some methods for walking the reference tree
 # of various objects.
 module RefCheck
-  # Build up a set of references.
-  def build_references(refs)
-    raise 'Circular reference' if @_visited
+  class SelfReference < StandardError
+  end
 
-    @_visited = true
+  class NullReference < StandardError
+  end
 
-    if respond_to?(:all_refs)
-      all_refs.each do |ref|
-        refs[ref.to_s] = 1
+  # Build up a set of references.
+  # rubocop:disable Metrics/PerceivedComplexity
+  def build_references(refs = [], origin = nil, method = :all_refs)
+    if respond_to?(method)
+      send(method).each do |ref|
+        raise SelfReference, "#{origin} references itself at #{to_json}" if origin && ref.to_s == origin
+        raise NullReference, "#{origin} contains null value reference at #{to_json}" if origin && ref.nil?
+
+        refs << ref
       end
     end
 
     ref_children.each do |elem|
-      elem.build_references(refs) if elem.respond_to?(:build_references)
-    end
+      # Nulls are not permitted in Cloudformation templates.
+      raise NullReference, "#{origin} contains null value reference at #{to_json}" if origin && elem.nil?
 
-    @_visited = nil
+      elem.build_references(refs, origin, method) if elem.respond_to?(:build_references)
+    end
 
     refs
   end
+  # rubocop:enable Metrics/PerceivedComplexity
 
   def ref_children
     []
@@ -37,7 +45,7 @@ class Array
   end
 end
 
-# Mixin to Array
+# Mixin to Hash
 class Hash
   include RefCheck
   def ref_children

data/lib/cfndsl/resources.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
-require 'cfndsl/jsonable'
-require 'cfndsl/properties'
-require 'cfndsl/update_policy'
+require_relative 'jsonable'
 
 module CfnDsl
   # Handles Resource objects
@@ -10,10 +8,6 @@ module CfnDsl
     dsl_attr_setter :Type, :DependsOn, :DeletionPolicy, :Condition, :Metadata
     dsl_content_object :Property, :UpdatePolicy, :CreationPolicy
 
-    def addTag(name, value, propagate = nil)
-      add_tag(name, value, propagate)
-    end
-
     def add_tag(name, value, propagate = nil)
       send(:Tag) do
         Key name
@@ -22,19 +16,12 @@ module CfnDsl
       end
     end
 
-    def all_refs
-      refs = []
-      if @DependsOn
-        if @DependsOn.respond_to?(:each)
-          @DependsOn.each do |dep|
-            refs.push dep
-          end
-        end
-
-        refs.push @DependsOn if @DependsOn.instance_of?(String)
-      end
+    def condition_refs
+      [@Condition].flatten.compact.map(&:to_s)
+    end
 
-      refs
+    def all_refs
+      [@DependsOn].flatten.compact.map(&:to_s)
     end
   end
 end

data/lib/cfndsl/rules.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'cfndsl/jsonable'
+require_relative 'jsonable'
 
 module CfnDsl
   # Handles Rule objects

data/lib/cfndsl/runner.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+require 'optparse'
+require 'json'
+require_relative 'globals'
+# Defer require of other capabilities (particularly loading dynamic Types) until required
+
+module CfnDsl
+  # Runner class to handle commandline invocation
+  class Runner
+    # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
+    def self.invoke!
+      options = {}
+
+      optparse = OptionParser.new do |opts|
+        opts.version = CfnDsl::VERSION
+        opts.banner = 'Usage: cfndsl [options] FILE'
+
+        # Define the options, and what they do
+        options[:output] = '-'
+        opts.on('-o', '--output FILE', 'Write output to file') do |file|
+          options[:output] = file
+        end
+
+        options[:extras] = []
+        opts.on('-y', '--yaml FILE', 'Import yaml file as local variables') do |file|
+          options[:extras].push([:yaml, File.expand_path(file)])
+        end
+
+        opts.on('-j', '--json FILE', 'Import json file as local variables') do |file|
+          options[:extras].push([:json, File.expand_path(file)])
+        end
+
+        opts.on('-p', '--pretty', 'Pretty-format output JSON') do
+          options[:pretty] = true
+        end
+
+        opts.on('-f', '--format FORMAT', 'Specify the output format (JSON default)') do |format|
+          raise "Format #{format} not supported" unless %w[json yaml].include? format
+
+          options[:outformat] = format
+        end
+
+        opts.on('-D', '--define "VARIABLE=VALUE"', 'Directly set local VARIABLE as VALUE') do |file|
+          options[:extras].push([:raw, file])
+        end
+
+        options[:verbose] = false
+        opts.on('-v', '--verbose', 'Turn on verbose ouptut') do
+          options[:verbose] = true
+        end
+
+        opts.on('-m', '--disable-deep-merge', 'Disable deep merging of yaml') do
+          CfnDsl.disable_deep_merge
+        end
+
+        # TODO: Support options to add a spec/patches dir
+        opts.on('-s', '--specification-file FILE', 'Location of Cloudformation Resource Specification file') do |file|
+          CfnDsl.specification_file File.expand_path(file)
+        end
+
+        opts.on('-u', '--update-specification [VERSION]', 'Update the Resource Specification file to latest, or specific version') do |file|
+          options[:spec_version] = file || 'latest'
+          options[:update_spec] = true
+        end
+
+        opts.on('-g', '--generate RESOURCE_TYPE,RESOURCE_LOGICAL_NAME', 'Add resource type and logical name') do |r|
+          options[:lego] = true
+          options[:resources] << r
+        end
+
+        opts.on('-a', '--assetversion', 'Print out the specification version') do
+          options[:assetversion] = true
+        end
+
+        opts.on('-l', '--list', 'List supported resources') do
+          require_relative 'cfnlego'
+          puts Cfnlego.Resources.sort
+          exit
+        end
+
+        # This displays the help screen, all programs are
+        # assumed to have this option.
+        opts.on('-h', '--help', 'Display this screen') do
+          puts opts
+          exit
+        end
+      end
+
+      optparse.parse!
+
+      if options[:update_spec]
+        warn 'Updating specification file'
+        result = CfnDsl.update_specification_file(version: options[:spec_version])
+        warn "Specification #{result[:version]} successfully written to #{result[:file]}"
+      end
+
+      if options[:assetversion]
+        spec_file = JSON.parse File.read(CfnDsl.specification_file)
+        warn spec_file['ResourceSpecificationVersion']
+      end
+
+      if options[:lego]
+        require 'cfnlego'
+        puts Cfnlego.run(options)
+        exit
+      end
+
+      if ARGV.empty?
+        if options[:update_spec] || options[:assetversion]
+          exit 0
+        else
+          puts optparse.help
+          exit 1
+        end
+      end
+
+      filename = File.expand_path(ARGV[0])
+      verbose = options[:verbose] && STDERR
+
+      verbose.puts "Using specification file #{CfnDsl.specification_file}" if verbose
+
+      require_relative 'cloudformation'
+      model = CfnDsl.eval_file_with_extras(filename, options[:extras], verbose)
+
+      output = STDOUT
+      if options[:output] != '-'
+        verbose.puts("Writing to #{options[:output]}") if verbose
+        output = File.open(File.expand_path(options[:output]), 'w')
+      elsif verbose
+        verbose.puts('Writing to STDOUT')
+      end
+
+      if options[:outformat] == 'yaml'
+        data = model.to_json
+        output.puts JSON.parse(data).to_yaml
+      else
+        output.puts options[:pretty] ? JSON.pretty_generate(model) : model.to_json
+      end
+    end
+  end
+end
+# rubocop:enable Metrics/MethodLength, Metrics/AbcSize, Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity