hecks 2.5.1 → 2.7.0

data/lib/hecks/projections/deploy/template_diff/comparer.rb

@@ -0,0 +1,231 @@
+module Hecks
+  module Projections
+    module Deploy
+      module TemplateDiff
+        # Compares two loaded CloudFormation templates.
+        #
+        # Resources, parameters, outputs, conditions and mappings are matched
+        # by name, then compared property by property. A list of mappings that
+        # each carry a `Name`, `Id`, `Key` or similar field (container
+        # definitions, environment variables, tags, origins) is matched by that
+        # field, not by position, so inserting one entry reports one addition
+        # instead of a cascade of shifted values. Cache behaviors are matched
+        # the same way but their order is also checked, because CloudFront
+        # takes the first pattern that matches.
+        #
+        # Two values that differ only in how they are written (a number and
+        # the same digits as text, a `!Sub` with nothing but one `${Name}` and
+        # a `!Ref Name`) are equal after normalization or marked `cosmetic`.
+        module Comparer
+          module_function
+
+          Change = Struct.new(:path, :kind, :before, :after, :cosmetic, keyword_init: true)
+          Entity = Struct.new(:name, :type, :changes, :replaced, keyword_init: true)
+          SectionDiff = Struct.new(:added, :removed, :changed, keyword_init: true) do
+            # Tells whether nothing in the section differs.
+            #
+            # @return [Boolean] true when no entity was added, removed or changed
+            def empty? = added.empty? && removed.empty? && changed.empty?
+          end
+
+          ENTITY_SECTIONS = %w[Parameters Resources Outputs Conditions Mappings].freeze
+          IDENTITY_KEYS = %w[Name Id Key PolicyName PathPattern HeaderName ContainerName Sid Field].freeze
+          ORDERED_BY = %w[PathPattern].freeze
+
+          # Compares two templates.
+          #
+          # @param before [Hash{String => Object}] the template as `Loader.load` returns it
+          # @param after [Hash{String => Object}] the template to compare it with
+          # @return [Hash{String => SectionDiff, Array<Change>}] one `SectionDiff` per entity
+          #   section
+          #   that either template has, and `"Template"` mapped to the changes in every other
+          #   top-level key such as `Description`
+          def compare(before, after)
+            before = normalize(before)
+            after = normalize(after)
+            diff = ENTITY_SECTIONS.each_with_object({}) do |section, sections|
+              next unless before.key?(section) || after.key?(section)
+
+              sections[section] = section_diff(section, before[section] || {}, after[section] || {})
+            end
+            diff["Template"] = template_changes(before, after)
+            diff
+          end
+
+          def normalize(value)
+            case value
+            when Hash then normalize_hash(value)
+            when Array then value.map { |item| normalize(item) }
+            else value
+            end
+          end
+          private_class_method :normalize
+
+          def normalize_hash(hash)
+            result = hash.to_h { |key, item| [key, key == "DependsOn" ? depends_on(item) : normalize(item)] }
+            substitution?(result) ? simplify_sub(result) : result
+          end
+          private_class_method :normalize_hash
+
+          def depends_on(value)
+            Array(value).map(&:to_s).sort
+          end
+          private_class_method :depends_on
+
+          def substitution?(hash)
+            hash.size == 1 && hash["Fn::Sub"].is_a?(String)
+          end
+          private_class_method :substitution?
+
+          # `!Sub "${Name}"` is `!Ref Name`, `!Sub "${A.B}"` is `!GetAtt A.B`, and a `!Sub` with no
+          # variable is the plain string.
+          def simplify_sub(hash)
+            text = hash["Fn::Sub"]
+            return text unless text.include?("${")
+
+            match = text.match(/\A\$\{([^}!]+)\}\z/)
+            return hash unless match
+
+            name = match[1]
+            name.include?(".") && !name.start_with?("AWS::") ? { "Fn::GetAtt" => name.split(".", 2) } : { "Ref" => name }
+          end
+          private_class_method :simplify_sub
+
+          def section_diff(section, before, after)
+            SectionDiff.new(
+              added:   (after.keys - before.keys).sort.map { |name| [name, type_of(section, after[name])] },
+              removed: (before.keys - after.keys).sort.map { |name| [name, type_of(section, before[name])] },
+              changed: (before.keys & after.keys).sort.filter_map { |name| entity_diff(section, name, before[name], after[name]) }
+            )
+          end
+          private_class_method :section_diff
+
+          def type_of(section, entity)
+            entity.is_a?(Hash) && %w[Resources Parameters].include?(section) ? entity["Type"] : nil
+          end
+          private_class_method :type_of
+
+          def entity_diff(section, name, before, after)
+            return nil if before == after
+
+            changes = []
+            diff_values("", before, after, changes)
+            return nil if changes.empty?
+
+            replaced = section == "Resources" && before.is_a?(Hash) && after.is_a?(Hash) && before["Type"] != after["Type"]
+            Entity.new(name: name, type: type_of(section, after), changes: changes, replaced: replaced)
+          end
+          private_class_method :entity_diff
+
+          def template_changes(before, after)
+            changes = []
+            others = (before.keys | after.keys) - ENTITY_SECTIONS
+            others.sort.each { |key| diff_values(key, before[key], after[key], changes) unless before[key] == after[key] }
+            changes
+          end
+          private_class_method :template_changes
+
+          def diff_values(path, before, after, changes)
+            return if before == after
+
+            if before.is_a?(Hash) && after.is_a?(Hash)
+              diff_hashes(path, before, after, changes)
+            elsif before.is_a?(Array) && after.is_a?(Array)
+              diff_arrays(path, before, after, changes)
+            else
+              changes << Change.new(path: path, kind: :changed, before: before, after: after, cosmetic: same_text?(before, after))
+            end
+          end
+          private_class_method :diff_values
+
+          def diff_hashes(path, before, after, changes)
+            (before.keys | after.keys).sort.each do |key|
+              child = path.empty? ? key : "#{path}.#{key}"
+              if !after.key?(key)
+                changes << Change.new(path: child, kind: :removed, before: before[key])
+              elsif !before.key?(key)
+                changes << Change.new(path: child, kind: :added, after: after[key])
+              else
+                diff_values(child, before[key], after[key], changes)
+              end
+            end
+          end
+          private_class_method :diff_hashes
+
+          def diff_arrays(path, before, after, changes)
+            key = identity_key(before, after)
+            return diff_keyed(path, key, before, after, changes) if key
+            if before.sort_by(&:to_s) == after.sort_by(&:to_s)
+              return changes << Change.new(path: path, kind: :reordered, before: before, after: after,
+                                           cosmetic: true)
+            end
+
+            (0...[before.size, after.size].max).each do |index|
+              child = "#{path}[#{index}]"
+              if index >= after.size then changes << Change.new(path: child, kind: :removed, before: before[index])
+              elsif index >= before.size then changes << Change.new(path: child, kind: :added, after: after[index])
+              else diff_values(child, before[index], after[index], changes)
+              end
+            end
+          end
+          private_class_method :diff_arrays
+
+          def diff_keyed(path, key, before, after, changes)
+            left = before.to_h { |item| [item[key].to_s, item] }
+            right = after.to_h { |item| [item[key].to_s, item] }
+            (left.keys | right.keys).each do |name|
+              child = "#{path}[#{key}=#{name}]"
+              if !right.key?(name) then changes << Change.new(path: child, kind: :removed, before: left[name])
+              elsif !left.key?(name) then changes << Change.new(path: child, kind: :added, after: right[name])
+              else diff_values(child, left[name], right[name], changes)
+              end
+            end
+            check_order(path, key, left.keys, right.keys, changes)
+          end
+          private_class_method :diff_keyed
+
+          def check_order(path, key, left, right, changes)
+            return unless ORDERED_BY.include?(key)
+
+            common = left & right
+            return if left.select { |name| common.include?(name) } == right.select { |name| common.include?(name) }
+
+            changes << Change.new(path: path, kind: :reordered, before: left, after: right, cosmetic: false)
+          end
+          private_class_method :check_order
+
+          # A list of mappings is matched by a field only when every entry in both lists has it and
+          # no two entries share its value, so a match is never ambiguous.
+          def identity_key(before, after)
+            lists = [before, after]
+            return nil if lists.any? { |list| list.empty? || !list.all?(Hash) }
+
+            IDENTITY_KEYS.find do |key|
+              lists.all? do |list|
+                list.all? { |item| scalar?(item[key]) } && list.map do |item|
+                  item[key].to_s
+                end.uniq.size == list.size
+              end
+            end
+          end
+          private_class_method :identity_key
+
+          def scalar?(value)
+            value.is_a?(String) || value.is_a?(Integer)
+          end
+          private_class_method :scalar?
+
+          def same_text?(before, after)
+            scalars = [before, after].all? do |value|
+              [String, Integer, Float, TrueClass, FalseClass].any? do |type|
+                value.is_a?(type)
+              end
+            end
+            scalars && before.to_s == after.to_s
+          end
+          private_class_method :same_text?
+        end
+      end
+    end
+  end
+end

data/lib/hecks/projections/deploy/template_diff/loader.rb

@@ -0,0 +1,105 @@
+require "psych"
+
+module Hecks
+  module Projections
+    module Deploy
+      module TemplateDiff
+        # Reads a CloudFormation template in YAML into plain Ruby data.
+        #
+        # CloudFormation's short forms (`!Ref Name`, `!GetAtt A.B`, `!Sub "..."`)
+        # are YAML tags that a stock loader refuses or drops. This loader walks
+        # the parsed document and turns each into the long form CloudFormation
+        # itself reads (`{"Ref" => "Name"}`, `{"Fn::GetAtt" => ["A", "B"]}`), so
+        # a template written with short forms and one written with long forms
+        # load to the same data. Comments and key order do not survive loading,
+        # which is what a comparison of two templates wants.
+        #
+        # Nothing here evaluates a template or contacts AWS.
+        module Loader
+          module_function
+
+          # Turns template text into data.
+          #
+          # @param text [String] a CloudFormation template written in YAML
+          # @return [Hash{String => Object}] the template, with intrinsic tags in their long form
+          # @raise [ArgumentError] if the text is not one YAML mapping, or uses an alias
+          def load(text)
+            document = Psych.parse(text)
+            raise ArgumentError, "the template is empty" unless document
+
+            value = convert(document.root)
+            raise ArgumentError, "a CloudFormation template is a mapping at the top level" unless value.is_a?(Hash)
+
+            value
+          rescue Psych::SyntaxError => e
+            raise ArgumentError, "the template is not valid YAML: #{e.message}"
+          end
+
+          # Reads a template from a file.
+          #
+          # @param path [String] the path of a YAML template
+          # @return [Hash{String => Object}] the template, as `load` returns it
+          # @raise [ArgumentError] if the file is missing or is not a template
+          def load_file(path)
+            raise ArgumentError, "#{path} does not exist" unless File.file?(path)
+
+            load(File.read(path))
+          rescue ArgumentError => e
+            raise if e.message.start_with?(path)
+
+            raise ArgumentError, "#{path}: #{e.message}"
+          end
+
+          def convert(node)
+            raise ArgumentError, "YAML aliases are not supported in a template" if node.is_a?(Psych::Nodes::Alias)
+
+            value =
+              case node
+              when Psych::Nodes::Mapping then mapping(node)
+              when Psych::Nodes::Sequence then node.children.map { |child| convert(child) }
+              else scalar(node)
+              end
+            intrinsic?(node.tag) ? intrinsic(node.tag, value) : value
+          end
+          private_class_method :convert
+
+          def mapping(node)
+            node.children.each_slice(2).to_h { |key, value| [convert(key).to_s, convert(value)] }
+          end
+          private_class_method :mapping
+
+          # A plain scalar is typed the way YAML types it (`8080` is a number, `true` a boolean); a
+          # quoted or block scalar is always text.
+          def scalar(node)
+            return node.value unless node.plain && node.tag.nil?
+
+            Psych::ScalarScanner.new(Psych::ClassLoader::Restricted.new([], [])).tokenize(node.value)
+          end
+          private_class_method :scalar
+
+          def intrinsic?(tag)
+            tag&.start_with?("!") && !tag.start_with?("!!")
+          end
+          private_class_method :intrinsic?
+
+          def intrinsic(tag, value)
+            name = tag.delete_prefix("!")
+            case name
+            when "Ref", "Condition" then { name => value }
+            when "GetAtt" then { "Fn::GetAtt" => get_att(value) }
+            else { "Fn::#{name}" => value }
+            end
+          end
+          private_class_method :intrinsic
+
+          # `!GetAtt A.B` is the short form of `["A", "B"]`; only the first dot separates the
+          # resource from the attribute, since an attribute may itself contain dots.
+          def get_att(value)
+            value.is_a?(String) ? value.split(".", 2) : value
+          end
+          private_class_method :get_att
+        end
+      end
+    end
+  end
+end

data/lib/hecks/projections/deploy/template_diff.rb

@@ -0,0 +1,173 @@
+require "json"
+require_relative "template_diff/loader"
+require_relative "template_diff/comparer"
+
+module Hecks
+  module Projections
+    module Deploy
+      # An offline comparison of two CloudFormation templates.
+      #
+      # It answers "what would change if this template replaced that one"
+      # from the two files alone, the offline half of checking that a
+      # generated template leaves a live stack untouched: when the generated
+      # template and the one already deployed differ in nothing but comments,
+      # ordering and spelling, a change set against the live stack should be
+      # empty too. It reads no AWS state and changes nothing.
+      #
+      # ## What it reports
+      #
+      # Resources, parameters, outputs, conditions and mappings that were
+      # added, removed or changed, by logical id, and for a changed one every
+      # property that differs, with the value before and after. Other
+      # top-level keys, such as `Description`, are compared as values.
+      # Comments and the order of keys do not count, and the short and long
+      # forms of an intrinsic (`!Ref Name`, `{"Ref": "Name"}`) are the same
+      # thing. A change to a resource's `Type` is marked as a replacement.
+      # Differences that are only in how a value is written are marked
+      # cosmetic and do not count as a difference unless `strict` is set.
+      #
+      # ## What it does not do
+      #
+      # It does not know which property changes CloudFormation applies in
+      # place and which replace the resource beyond a change of `Type`, so a
+      # changed immutable property is listed as a change, not flagged.
+      module TemplateDiff
+        module_function
+
+        Report = Struct.new(:sections, :strict, keyword_init: true) do
+          # Tells whether the two templates differ in a way that counts.
+          #
+          # @return [Boolean] false when every difference is cosmetic and the report is not strict
+          def different?
+            sections.any? do |name, diff|
+              if name == "Template"
+                counted(diff).any?
+              else
+                !diff.added.empty? || !diff.removed.empty? || diff.changed.any? { |entity| counted(entity.changes).any? }
+              end
+            end
+          end
+
+          # Keeps the changes that count towards a difference.
+          #
+          # @param changes [Array<Comparer::Change>] the changes to filter
+          # @return [Array<Comparer::Change>] every change, or only the non-cosmetic ones unless
+          #   strict
+          def counted(changes) = strict ? changes : changes.reject(&:cosmetic)
+        end
+
+        # Compares two template texts.
+        #
+        # @param before [String] the first template, YAML
+        # @param after [String] the second template, YAML
+        # @param strict [Boolean] whether cosmetic differences count as differences
+        # @return [Report] the differences, by section
+        # @raise [ArgumentError] if either text is not a valid template
+        def diff(before, after, strict: false)
+          Report.new(sections: Comparer.compare(Loader.load(before), Loader.load(after)), strict: strict)
+        end
+
+        # Compares two template files.
+        #
+        # @param before_path [String] the path of the first template
+        # @param after_path [String] the path of the template to compare it with
+        # @param strict [Boolean] whether cosmetic differences count as differences
+        # @return [Report] the differences, by section
+        # @raise [ArgumentError] if either file is missing or is not a valid template
+        def diff_files(before_path, after_path, strict: false)
+          Report.new(
+            sections: Comparer.compare(Loader.load_file(before_path), Loader.load_file(after_path)),
+            strict:   strict
+          )
+        end
+
+        # Writes a report as text for a person to read.
+        #
+        # @param report [Report] the differences to describe
+        # @return [String] one section per kind of entity, each entity on a `+`, `-` or `~` line,
+        #   or `no differences` when nothing differs
+        def render(report)
+          lines = report.sections.flat_map do |name, diff|
+            name == "Template" ? template_lines(report, diff) : section_lines(report, name, diff)
+          end
+          return "no differences\n" if lines.empty?
+
+          note = report.different? ? [] : ["", "only cosmetic differences; --strict counts them"]
+          "#{(lines + note).join("\n")}\n"
+        end
+
+        # Writes a report as JSON for a program to read.
+        #
+        # @param report [Report] the differences to describe
+        # @return [String] a JSON document with `different` and one entry per section
+        def render_json(report)
+          sections = report.sections.to_h do |name, diff|
+            [name, name == "Template" ? diff.map { |change| change_hash(change) } : section_hash(diff)]
+          end
+          JSON.pretty_generate("different" => report.different?, "sections" => sections)
+        end
+
+        def template_lines(report, changes)
+          return [] if changes.empty?
+
+          ["Template"] + changes.flat_map { |change| change_lines(change, report, "  ") }
+        end
+        private_class_method :template_lines
+
+        def section_lines(report, name, diff)
+          return [] if diff.empty?
+
+          [name] +
+            diff.added.map { |id, type| "  + #{id}#{" (#{type})" if type}" } +
+            diff.removed.map { |id, type| "  - #{id}#{" (#{type})" if type}" } +
+            diff.changed.flat_map { |entity| entity_lines(entity, report) }
+        end
+        private_class_method :section_lines
+
+        def entity_lines(entity, report)
+          head = "  ~ #{entity.name}#{" (#{entity.type})" if entity.type}#{'  REPLACEMENT: the type changed' if entity.replaced}"
+          [head] + entity.changes.flat_map { |change| change_lines(change, report, "      ") }
+        end
+        private_class_method :entity_lines
+
+        def change_lines(change, report, indent)
+          marker = change.cosmetic && !report.strict ? " (cosmetic)" : ""
+          case change.kind
+          when :added then ["#{indent}+ #{change.path}: #{show(change.after)}"]
+          when :removed then ["#{indent}- #{change.path}: #{show(change.before)}"]
+          when :reordered then ["#{indent}~ #{change.path}: order #{show(change.before)} -> #{show(change.after)}#{marker}"]
+          else ["#{indent}~ #{change.path}: #{show(change.before)} -> #{show(change.after)}#{marker}"]
+          end
+        end
+        private_class_method :change_lines
+
+        def show(value)
+          text = value.is_a?(String) ? value.inspect : JSON.generate(value)
+          text.length > 140 ? "#{text[0, 137]}..." : text
+        end
+        private_class_method :show
+
+        def section_hash(diff)
+          {
+            "added"   => diff.added.map { |id, type| { "id" => id, "type" => type }.compact },
+            "removed" => diff.removed.map { |id, type| { "id" => id, "type" => type }.compact },
+            "changed" => diff.changed.map { |entity| entity_hash(entity) }
+          }
+        end
+        private_class_method :section_hash
+
+        def entity_hash(entity)
+          changes = entity.changes.map { |change| change_hash(change) }
+          { "id" => entity.name, "type" => entity.type, "replaced" => entity.replaced, "changes" => changes }.compact
+        end
+        private_class_method :entity_hash
+
+        def change_hash(change)
+          { "path" => change.path, "kind" => change.kind.to_s, "before" => change.before, "after" => change.after,
+"cosmetic" => change.cosmetic }.compact
+        end
+        private_class_method :change_hash
+      end
+    end
+  end
+end

data/lib/hecks/projections/glossary/sensitivity.rb

@@ -7,7 +7,7 @@ module Hecks
       # deployment declares apart from the domain (`Privacy::Marking`, or a
       # `has_phi(readable_by:)` chain in a `.hecksagon`) and handed in as
       # `options[:markings]`. A marking names an aggregate by its qualified
-      # name (`"Lifeadelics::Registration"`) and a field by its dotted path
+      # name (`"Site::Registration"`) and a field by its dotted path
       # (`"attendee.medications"`). Its category is spoken as the acronym it
       # is, uppercased and never reworded, because the category vocabulary is
       # the deployment's own and open-ended.

data/lib/hecks/projections.rb

@@ -36,3 +36,5 @@ require_relative "projections/glossary"
 require_relative "projections/deploy/shared"
 require_relative "projections/deploy/lambda"
 require_relative "projections/deploy/fargate"
+require_relative "projections/deploy/smoke"
+require_relative "projections/deploy/template_diff"

data/lib/hecks/release/gem_pin/published_versions.rb

@@ -0,0 +1,58 @@
+require "json"
+require "net/http"
+require "uri"
+
+module Hecks
+  module Release
+    class GemPin
+      # Raised when a pin cannot be resolved or is refused.
+      class Error < StandardError; end
+
+      # The hecks releases RubyGems publishes, fetched on first use.
+      #
+      # `GemPin` takes any object answering `include?` and `newest_satisfying`,
+      # so a caller with no network, or a spec, hands it a stand-in and this
+      # class is never asked to dial out.
+      class PublishedVersions
+        URL = URI("https://rubygems.org/api/v1/versions/hecks.json")
+
+        # Says whether RubyGems publishes a version.
+        #
+        # @param version [String, Gem::Version] the version to look for
+        # @return [Boolean] true when it is in the published list, prereleases included
+        # @raise [GemPin::Error] if RubyGems cannot be reached or does not answer with the list
+        def include?(version)
+          versions.include?(Gem::Version.new(version))
+        end
+
+        # Finds the newest stable release a requirement allows.
+        #
+        # @param requirement [Gem::Requirement] the constraint to satisfy
+        # @return [Gem::Version, nil] the newest non-prerelease version that satisfies it, or nil
+        # @raise [GemPin::Error] if RubyGems cannot be reached or does not answer with the list
+        def newest_satisfying(requirement)
+          versions.reject(&:prerelease?).select { |version| requirement.satisfied_by?(version) }.max
+        end
+
+        private
+
+        def versions
+          @versions ||= fetch
+        end
+
+        def fetch
+          response = Net::HTTP.start(URL.host, URL.port, use_ssl: true, open_timeout: 10, read_timeout: 20) do |http|
+            http.get(URL.request_uri)
+          end
+          unless response.is_a?(Net::HTTPSuccess)
+            raise Error, "could not list published hecks versions (RubyGems answered #{response.code})"
+          end
+
+          JSON.parse(response.body).map { |entry| Gem::Version.new(entry.fetch("number")) }
+        rescue SocketError, Timeout::Error, SystemCallError => e
+          raise Error, "could not reach RubyGems to check the hecks version (#{e.class})"
+        end
+      end
+    end
+  end
+end