kennel 1.130.0 → 1.131.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d6ee3df09b69dd5a7af7285c219f4415c4e94ce2ab4f6533c5fff03be848e9c4
-  data.tar.gz: d7f923b69ad9b63774141ac0e9c6c04de7a26a67abd732c41c5f19d478704656
+  metadata.gz: ac3eeb3c1ec0e2b03c7acbf2b2d7c68335207a7e79408775e9b876c6c4acd400
+  data.tar.gz: db96fddb04bce543efcea1a505a1d360447161259e9e9fdd481acfe785bad194
 SHA512:
-  metadata.gz: 7c967bf54743cc757a0c0c911e981701d4b5b393f6ebac66538432541c7c6dfecef2cd8c1c0e5389f90c7d0b91a54e79a29e0e1abf9d16fd67adad4bcd6209fe
-  data.tar.gz: 297239ea9305e4eadbe40a363160b0ca2aaf70dad237f95a448b00b30d3db9ed9f707b2736b2596ef73e8981d333b600af87da40978f5d1c9fc8e0dee3d7f9ea
+  metadata.gz: a710b2e2ae6932cf36bdd12fbb06c34f8b538944e693a484b294e7614b7d9941e7c2da498cf7cdcb56bda255a0b2c874383306437ee5b71456d900ae1e9a5c2f
+  data.tar.gz: 90e767722863bb640f3bd96436937c390d40d8c8b868d3ee51a444ff6e042e36f03048cc1b5c9cec49d96207d53d99df6f529318658c3881fd9078ed55c2b7d8

data/lib/kennel/api.rb

@@ -7,6 +7,8 @@ module Kennel
 
     def self.tag(api_resource, reply)
       klass = Models::Record.api_resource_map[api_resource]
+      return reply unless klass # do not blow up on unknown models
+
       reply.merge(
         klass: klass,
         tracking_id: klass.parse_tracking_id(reply)

data/lib/kennel/file_cache.rb

@@ -57,7 +57,9 @@ module Kennel
     # - delete expired keys
     # - delete what would be deleted anyway when updating
     def expire_old_data
-      @data.reject! { |_, (_, (_, cv), expires)| expires < @now || cv != @cache_version }
+      @data.reject! do |(_api_resource, _id), (_value, (_key_version, cache_version), expires)|
+        expires < @now || cache_version != @cache_version
+      end
     end
   end
 end

data/lib/kennel/models/dashboard.rb

@@ -202,7 +202,7 @@ module Kennel
         end
       end
 
-      def validate_update!(_actuals, diffs)
+      def validate_update!(diffs)
         _, path, from, to = diffs.find { |diff| diff[1] == "layout_type" }
         invalid_update!(path, from, to) if path
       end

data/lib/kennel/models/monitor.rb

@@ -137,7 +137,7 @@ module Kennel
         end
       end
 
-      def validate_update!(_actuals, diffs)
+      def validate_update!(diffs)
         # ensure type does not change, but not if it's metric->query which is supported and used by importer.rb
         _, path, from, to = diffs.detect { |_, path, _, _| path == "type" }
         if path && !(from == "metric alert" && to == "query alert")

data/lib/kennel/models/record.rb

@@ -167,7 +167,7 @@ module Kennel
       end
 
       # Can raise DisallowedUpdateError
-      def validate_update!(*)
+      def validate_update!(_diffs)
       end
 
       def invalid_update!(field, old_value, new_value)

data/lib/kennel/syncer.rb

@@ -6,47 +6,43 @@ module Kennel
     LINE_UP = "\e[1A\033[K" # go up and clear
 
     Plan = Struct.new(:changes, keyword_init: true)
+
+    InternalPlan = Struct.new(:creates, :updates, :deletes) do
+      def empty?
+        creates.empty? && updates.empty? && deletes.empty?
+      end
+    end
+
     Change = Struct.new(:type, :api_resource, :tracking_id, :id)
 
     def initialize(api, expected, actual, strict_imports: true, project_filter: nil, tracking_id_filter: nil)
       @api = api
-      @expected = Set.new expected # need Set to speed up deletion
-      @actual = actual
       @strict_imports = strict_imports
       @project_filter = project_filter
       @tracking_id_filter = tracking_id_filter
 
-      @attribute_differ = AttributeDiffer.new
+      @resolver = Resolver.new(expected: expected, project_filter: @project_filter, tracking_id_filter: @tracking_id_filter)
 
-      calculate_changes
-      validate_changes
-      prevent_irreversible_partial_updates
+      internal_plan = calculate_changes(expected: expected, actual: actual)
+      validate_changes(internal_plan)
+      @internal_plan = internal_plan
 
       @warnings.each { |message| Kennel.out.puts Console.color(:yellow, "Warning: #{message}") }
     end
 
     def plan
+      ip = @internal_plan
       Plan.new(
-        changes:
-          @create.map { |_id, e, _a| Change.new(:create, e.class.api_resource, e.tracking_id, nil) } +
-            @update.map { |id, e, _a| Change.new(:update, e.class.api_resource, e.tracking_id, id) } +
-            @delete.map { |id, _e, a| Change.new(:delete, a.fetch(:klass).api_resource, a.fetch(:tracking_id), id) }
+        changes: (ip.creates + ip.updates + ip.deletes).map(&:change)
       )
     end
 
     def print_plan
-      Kennel.out.puts "Plan:"
-      if noop?
-        Kennel.out.puts Console.color(:green, "Nothing to do")
-      else
-        print_changes "Create", @create, :green
-        print_changes "Update", @update, :yellow
-        print_changes "Delete", @delete, :red
-      end
+      PlanDisplayer.new.display(internal_plan)
     end
 
     def confirm
-      return false if noop?
+      return false if internal_plan.empty?
       return true if ENV["CI"] || !STDIN.tty? || !Kennel.err.tty?
       Console.ask?("Execute Plan ?")
     end
@@ -54,121 +50,77 @@ module Kennel
     def update
       changes = []
 
-      each_resolved @create do |_, e|
-        message = "#{e.class.api_resource} #{e.tracking_id}"
+      internal_plan.deletes.each do |item|
+        message = "#{item.api_resource} #{item.tracking_id} #{item.id}"
+        Kennel.out.puts "Deleting #{message}"
+        @api.delete item.api_resource, item.id
+        changes << item.change
+        Kennel.out.puts "#{LINE_UP}Deleted #{message}"
+      end
+
+      resolver.each_resolved internal_plan.creates do |item|
+        message = "#{item.api_resource} #{item.tracking_id}"
         Kennel.out.puts "Creating #{message}"
-        reply = @api.create e.class.api_resource, e.as_json
+        reply = @api.create item.api_resource, item.expected.as_json
         id = reply.fetch(:id)
-        changes << Change.new(:create, e.class.api_resource, e.tracking_id, id)
-        populate_id_map [], [reply] # allow resolving ids we could previously no resolve
-        Kennel.out.puts "#{LINE_UP}Created #{message} #{e.class.url(id)}"
+        changes << item.change(id)
+        resolver.add_actual [reply] # allow resolving ids we could previously not resolve
+        Kennel.out.puts "#{LINE_UP}Created #{message} #{item.url(id)}"
       end
 
-      each_resolved @update do |id, e|
-        message = "#{e.class.api_resource} #{e.tracking_id} #{e.class.url(id)}"
+      resolver.each_resolved internal_plan.updates do |item|
+        message = "#{item.api_resource} #{item.tracking_id} #{item.url}"
         Kennel.out.puts "Updating #{message}"
-        @api.update e.class.api_resource, id, e.as_json
-        changes << Change.new(:update, e.class.api_resource, e.tracking_id, id)
+        @api.update item.api_resource, item.id, item.expected.as_json
+        changes << item.change
         Kennel.out.puts "#{LINE_UP}Updated #{message}"
       end
 
-      @delete.each do |id, _, a|
-        klass = a.fetch(:klass)
-        message = "#{klass.api_resource} #{a.fetch(:tracking_id)} #{id}"
-        Kennel.out.puts "Deleting #{message}"
-        @api.delete klass.api_resource, id
-        changes << Change.new(:delete, klass.api_resource, a.fetch(:tracking_id), id)
-        Kennel.out.puts "#{LINE_UP}Deleted #{message}"
-      end
-
       Plan.new(changes: changes)
     end
 
     private
 
-    # loop over items until everything is resolved or crash when we get stuck
-    # this solves cases like composite monitors depending on each other or monitor->monitor slo->slo monitor chains
-    def each_resolved(list)
-      list = list.dup
-      loop do
-        return if list.empty?
-        list.reject! do |id, e|
-          if resolved?(e)
-            yield id, e
-            true
-          else
-            false
-          end
-        end ||
-          assert_resolved(list[0][1]) # resolve something or show a circular dependency error
-      end
-    end
-
-    # TODO: optimize by storing an instance variable if already resolved
-    def resolved?(e)
-      assert_resolved e
-      true
-    rescue UnresolvableIdError
-      false
-    end
-
-    # raises UnresolvableIdError when not resolved
-    def assert_resolved(e)
-      resolve_linked_tracking_ids! [e], force: true
-    end
-
-    def noop?
-      @create.empty? && @update.empty? && @delete.empty?
-    end
+    attr_reader :resolver, :internal_plan
 
-    def calculate_changes
+    def calculate_changes(expected:, actual:)
       @warnings = []
-      @id_map = IdMap.new
 
       Progress.progress "Diffing" do
-        populate_id_map @expected, @actual
-        filter_actual! @actual
-        resolve_linked_tracking_ids! @expected # resolve dependencies to avoid diff
-        @expected.each(&:add_tracking_id) # avoid diff with actual, which has tracking_id
+        resolver.add_actual actual
+        filter_actual! actual
+        resolver.resolve_as_much_as_possible(expected) # resolve as many dependencies as possible to reduce the diff
 
         # see which expected match the actual
-        matching, unmatched_expected, unmatched_actual = partition_matched_expected
+        matching, unmatched_expected, unmatched_actual = MatchedExpected.partition(expected, actual)
         validate_expected_id_not_missing unmatched_expected
         fill_details! matching # need details to diff later
 
         # update matching if needed
-        @update = matching.map do |e, a|
+        updates = matching.map do |e, a|
+          # Refuse to "adopt" existing items into kennel while running with a filter (i.e. on a branch).
+          # Without this, we'd adopt an item, then the next CI run would delete it
+          # (instead of "unadopting" it).
+          e.add_tracking_id unless @project_filter && a.fetch(:tracking_id).nil?
           id = a.fetch(:id)
           diff = e.diff(a)
-          [id, e, a, diff] if diff.any?
+          a[:id] = id
+          Types::PlannedUpdate.new(e, a, diff) if diff.any?
         end.compact
 
         # delete previously managed
-        @delete = unmatched_actual.map { |a| [a.fetch(:id), nil, a] if a.fetch(:tracking_id) }.compact
+        deletes = unmatched_actual.map { |a| Types::PlannedDelete.new(a) if a.fetch(:tracking_id) }.compact
 
         # unmatched expected need to be created
-        @create = unmatched_expected.map { |e| [nil, e] }
+        unmatched_expected.each(&:add_tracking_id)
+        creates = unmatched_expected.map { |e| Types::PlannedCreate.new(e) }
 
         # order to avoid deadlocks
-        @delete.sort_by! { |_, _, a| DELETE_ORDER.index a.fetch(:klass).api_resource }
-        @update.sort_by! { |_, e, _| DELETE_ORDER.index e.class.api_resource } # slo needs to come before slo alert
-      end
-    end
+        deletes.sort_by! { |item| DELETE_ORDER.index item.api_resource }
+        updates.sort_by! { |item| DELETE_ORDER.index item.api_resource } # slo needs to come before slo alert
 
-    def partition_matched_expected
-      lookup_map = matching_expected_lookup_map
-      unmatched_expected = @expected.dup
-      unmatched_actual = []
-      matched = []
-      @actual.each do |a|
-        e = matching_expected(a, lookup_map)
-        if e && unmatched_expected.delete?(e)
-          matched << [e, a]
-        else
-          unmatched_actual << a
-        end
-      end.compact
-      [matched, unmatched_expected, unmatched_actual]
+        InternalPlan.new(creates, updates, deletes)
+      end
     end
 
     # fill details of things we need to compare
@@ -189,108 +141,242 @@ module Kennel
       end
     end
 
-    # index list by all the thing we look up by: tracking id and actual id
-    def matching_expected_lookup_map
-      @expected.each_with_object({}) do |e, all|
-        keys = [e.tracking_id]
-        keys << "#{e.class.api_resource}:#{e.id}" if e.id
-        keys.compact.each do |key|
-          raise "Lookup #{key} is duplicated" if all[key]
-          all[key] = e
-        end
+    # We've already validated the desired objects ('generated') in isolation.
+    # Now that we have made the plan, we can perform some more validation.
+    def validate_changes(internal_plan)
+      internal_plan.updates.each do |item|
+        item.expected.validate_update!(item.diff)
       end
     end
 
-    def matching_expected(a, map)
-      klass = a.fetch(:klass)
-      map["#{klass.api_resource}:#{a.fetch(:id)}"] || map[a.fetch(:tracking_id)]
+    def filter_actual!(actual)
+      if @tracking_id_filter
+        actual.select! do |a|
+          tracking_id = a.fetch(:tracking_id)
+          !tracking_id || @tracking_id_filter.include?(tracking_id)
+        end
+      elsif @project_filter
+        project_prefixes = @project_filter.map { |p| "#{p}:" }
+        actual.select! do |a|
+          tracking_id = a.fetch(:tracking_id)
+          !tracking_id || tracking_id.start_with?(*project_prefixes)
+        end
+      end
     end
 
-    def print_changes(step, list, color)
-      return if list.empty?
-      list.each do |_, e, a, diff|
-        klass = (e ? e.class : a.fetch(:klass))
-        Kennel.out.puts Console.color(color, "#{step} #{klass.api_resource} #{e&.tracking_id || a.fetch(:tracking_id)}")
-        diff&.each { |args| Kennel.out.puts @attribute_differ.format(*args) } # only for update
+    class PlanDisplayer
+      def initialize
+        @attribute_differ = AttributeDiffer.new
       end
-    end
 
-    # We've already validated the desired objects ('generated') in isolation.
-    # Now that we have made the plan, we can perform some more validation.
-    def validate_changes
-      @update.each do |_, expected, actuals, diffs|
-        expected.validate_update!(actuals, diffs)
+      def display(internal_plan)
+        Kennel.out.puts "Plan:"
+        if internal_plan.empty?
+          Kennel.out.puts Console.color(:green, "Nothing to do")
+        else
+          print_changes "Create", internal_plan.creates, :green
+          print_changes "Update", internal_plan.updates, :yellow
+          print_changes "Delete", internal_plan.deletes, :red
+        end
+      end
+
+      private
+
+      def print_changes(step, list, color)
+        return if list.empty?
+        list.each do |item|
+          Kennel.out.puts Console.color(color, "#{step} #{item.api_resource} #{item.tracking_id}")
+          if item.class::TYPE == :update
+            item.diff.each { |args| Kennel.out.puts @attribute_differ.format(*args) } # only for update
+          end
+        end
       end
     end
 
-    # - do not add tracking-id when working with existing ids on a branch,
-    #   so resource do not get deleted when running an update on master (for example merge->CI)
-    # - ideally we'd never add tracking in the first place, but when adding tracking we do not know the diff yet
-    def prevent_irreversible_partial_updates
-      return unless @project_filter # full update, so we are not on a branch
-      @update.select! do |_, e, _, diff| # ensure clean diff, by removing noop-update
-        next true unless e.id # safe to add tracking when not having id
-
-        diff.select! do |field_diff|
-          (_, field, actual) = field_diff
-          # TODO: refactor this so TRACKING_FIELD stays record-private
-          next true if e.class::TRACKING_FIELD != field.to_sym # need to sym here because Hashdiff produces strings
-          next true if e.class.parse_tracking_id(field.to_sym => actual) # already has tracking id
-
-          field_diff[3] = e.remove_tracking_id # make `rake plan` output match what we are sending
-          actual != field_diff[3] # discard diff if now nothing changes
+    class Resolver
+      def initialize(expected:, project_filter:, tracking_id_filter:)
+        @id_map = IdMap.new
+        @project_filter = project_filter
+        @tracking_id_filter = tracking_id_filter
+
+        # mark everything as new
+        expected.each do |e|
+          id_map.set(e.class.api_resource, e.tracking_id, IdMap::NEW)
+          if e.class.api_resource == "synthetics/tests"
+            id_map.set(Kennel::Models::Monitor.api_resource, e.tracking_id, IdMap::NEW)
+          end
         end
+      end
 
-        diff.any?
+      def add_actual(actual)
+        # override resources that exist with their id
+        project_prefixes = project_filter&.map { |p| "#{p}:" }
+
+        actual.each do |a|
+          # ignore when not managed by kennel
+          next unless tracking_id = a.fetch(:tracking_id)
+
+          # ignore when deleted from the codebase
+          # (when running with filters we cannot see the other resources in the codebase)
+          api_resource = a.fetch(:klass).api_resource
+          next if
+            !id_map.get(api_resource, tracking_id) &&
+            (!project_prefixes || tracking_id.start_with?(*project_prefixes)) &&
+            (!tracking_id_filter || tracking_id_filter.include?(tracking_id))
+
+          id_map.set(api_resource, tracking_id, a.fetch(:id))
+          if a.fetch(:klass).api_resource == "synthetics/tests"
+            id_map.set(Kennel::Models::Monitor.api_resource, tracking_id, a.fetch(:monitor_id))
+          end
+        end
       end
-    end
 
-    def populate_id_map(expected, actual)
-      # mark everything as new
-      expected.each do |e|
-        @id_map.set(e.class.api_resource, e.tracking_id, IdMap::NEW)
-        if e.class.api_resource == "synthetics/tests"
-          @id_map.set(Kennel::Models::Monitor.api_resource, e.tracking_id, IdMap::NEW)
+      def resolve_as_much_as_possible(expected)
+        expected.each do |e|
+          e.resolve_linked_tracking_ids!(id_map, force: false)
         end
       end
 
-      # override resources that exist with their id
-      project_prefixes = @project_filter&.map { |p| "#{p}:" }
-      actual.each do |a|
-        # ignore when not managed by kennel
-        next unless tracking_id = a.fetch(:tracking_id)
-
-        # ignore when deleted from the codebase
-        # (when running with filters we cannot see the other resources in the codebase)
-        api_resource = a.fetch(:klass).api_resource
-        next if
-          !@id_map.get(api_resource, tracking_id) &&
-          (!project_prefixes || tracking_id.start_with?(*project_prefixes)) &&
-          (!@tracking_id_filter || @tracking_id_filter.include?(tracking_id))
-
-        @id_map.set(api_resource, tracking_id, a.fetch(:id))
-        if a.fetch(:klass).api_resource == "synthetics/tests"
-          @id_map.set(Kennel::Models::Monitor.api_resource, tracking_id, a.fetch(:monitor_id))
+      # loop over items until everything is resolved or crash when we get stuck
+      # this solves cases like composite monitors depending on each other or monitor->monitor slo->slo monitor chains
+      def each_resolved(list)
+        list = list.dup
+        loop do
+          return if list.empty?
+          list.reject! do |item|
+            if resolved?(item.expected)
+              yield item
+              true
+            else
+              false
+            end
+          end ||
+            assert_resolved(list[0].expected) # resolve something or show a circular dependency error
         end
       end
+
+      private
+
+      attr_reader :id_map, :project_filter, :tracking_id_filter
+
+      # TODO: optimize by storing an instance variable if already resolved
+      def resolved?(e)
+        assert_resolved e
+        true
+      rescue UnresolvableIdError
+        false
+      end
+
+      # raises UnresolvableIdError when not resolved
+      def assert_resolved(e)
+        e.resolve_linked_tracking_ids!(id_map, force: true)
+      end
     end
 
-    def resolve_linked_tracking_ids!(list, force: false)
-      list.each { |e| e.resolve_linked_tracking_ids!(@id_map, force: force) }
+    module MatchedExpected
+      class << self
+        def partition(expected, actual)
+          lookup_map = matching_expected_lookup_map(expected)
+          unmatched_expected = Set.new(expected) # for efficient deletion
+          unmatched_actual = []
+          matched = []
+          actual.each do |a|
+            e = matching_expected(a, lookup_map)
+            if e && unmatched_expected.delete?(e)
+              matched << [e, a]
+            else
+              unmatched_actual << a
+            end
+          end.compact
+          [matched, unmatched_expected.to_a, unmatched_actual]
+        end
+
+        private
+
+        # index list by all the thing we look up by: tracking id and actual id
+        def matching_expected_lookup_map(expected)
+          expected.each_with_object({}) do |e, all|
+            keys = [e.tracking_id]
+            keys << "#{e.class.api_resource}:#{e.id}" if e.id
+            keys.compact.each do |key|
+              raise "Lookup #{key} is duplicated" if all[key]
+              all[key] = e
+            end
+          end
+        end
+
+        def matching_expected(a, map)
+          klass = a.fetch(:klass)
+          map["#{klass.api_resource}:#{a.fetch(:id)}"] || map[a.fetch(:tracking_id)]
+        end
+      end
     end
 
-    def filter_actual!(actual)
-      if @tracking_id_filter
-        actual.select! do |a|
-          tracking_id = a.fetch(:tracking_id)
-          !tracking_id || @tracking_id_filter.include?(tracking_id)
+    module Types
+      class PlannedChange
+        def initialize(klass, tracking_id)
+          @klass = klass
+          @tracking_id = tracking_id
         end
-      elsif @project_filter
-        project_prefixes = @project_filter.map { |p| "#{p}:" }
-        actual.select! do |a|
-          tracking_id = a.fetch(:tracking_id)
-          !tracking_id || tracking_id.start_with?(*project_prefixes)
+
+        def api_resource
+          klass.api_resource
+        end
+
+        def url(id = nil)
+          klass.url(id || self.id)
+        end
+
+        def change(id = nil)
+          Change.new(self.class::TYPE, api_resource, tracking_id, id)
         end
+
+        attr_reader :klass, :tracking_id
+      end
+
+      class PlannedCreate < PlannedChange
+        TYPE = :create
+
+        def initialize(expected)
+          super(expected.class, expected.tracking_id)
+          @expected = expected
+        end
+
+        attr_reader :expected
+      end
+
+      class PlannedUpdate < PlannedChange
+        TYPE = :update
+
+        def initialize(expected, actual, diff)
+          super(expected.class, expected.tracking_id)
+          @expected = expected
+          @actual = actual
+          @diff = diff
+          @id = actual.fetch(:id)
+        end
+
+        def change
+          super(id)
+        end
+
+        attr_reader :expected, :actual, :diff, :id
+      end
+
+      class PlannedDelete < PlannedChange
+        TYPE = :delete
+
+        def initialize(actual)
+          super(actual.fetch(:klass), actual.fetch(:tracking_id))
+          @actual = actual
+          @id = actual.fetch(:id)
+        end
+
+        def change
+          super(id)
+        end
+
+        attr_reader :actual, :id
       end
     end
   end

data/lib/kennel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Kennel
-  VERSION = "1.130.0"
+  VERSION = "1.131.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kennel
 version: !ruby/object:Gem::Version
-  version: 1.130.0
+  version: 1.131.0
 platform: ruby
 authors:
 - Michael Grosser
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-12-30 00:00:00.000000000 Z
+date: 2023-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: diff-lcs