kennel 1.129.0 → 1.131.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7690d961a150ea0ba0649edcab513c22ec828988fec2ab631636497274305c7
-  data.tar.gz: c5bc7244c4168466f5167e5c322f3a2cdb8e6b9b8ec23a2d12fb343a7648ae06
+  metadata.gz: ac3eeb3c1ec0e2b03c7acbf2b2d7c68335207a7e79408775e9b876c6c4acd400
+  data.tar.gz: db96fddb04bce543efcea1a505a1d360447161259e9e9fdd481acfe785bad194
 SHA512:
-  metadata.gz: 51b47377a8cf688dafd17600b543cc1734eb8511a6ed1b627d8b1180bf70fc66c86f9d4a8e4b2a3ff5e7223edf366e5483d983d9465c5907aaa07f5b9c36ef46
-  data.tar.gz: 698c23a4de558382467342b90e7d3eac143a0279653efe9f1b711b3e114a37d8d72f4a9c5d97e652a6646007298f5e93f296c1125caaa4c498195089b0cac730
+  metadata.gz: a710b2e2ae6932cf36bdd12fbb06c34f8b538944e693a484b294e7614b7d9941e7c2da498cf7cdcb56bda255a0b2c874383306437ee5b71456d900ae1e9a5c2f
+  data.tar.gz: 90e767722863bb640f3bd96436937c390d40d8c8b868d3ee51a444ff6e042e36f03048cc1b5c9cec49d96207d53d99df6f529318658c3881fd9078ed55c2b7d8

data/lib/kennel/api.rb

@@ -5,6 +5,16 @@ module Kennel
   class Api
     CACHE_FILE = "tmp/cache/details"
 
+    def self.tag(api_resource, reply)
+      klass = Models::Record.api_resource_map[api_resource]
+      return reply unless klass # do not blow up on unknown models
+
+      reply.merge(
+        klass: klass,
+        tracking_id: klass.parse_tracking_id(reply)
+      )
+    end
+
     def initialize(app_key = nil, api_key = nil)
       @app_key = app_key || ENV.fetch("DATADOG_APP_KEY")
       @api_key = api_key || ENV.fetch("DATADOG_API_KEY")
@@ -16,7 +26,7 @@ module Kennel
       response = request :get, "/api/v1/#{api_resource}/#{id}", params: params
       response = response.fetch(:data) if api_resource == "slo"
       response[:id] = response.delete(:public_id) if api_resource == "synthetics/tests"
-      response
+      self.class.tag(api_resource, response)
     end
 
     def list(api_resource, params = {})
@@ -32,7 +42,7 @@ module Kennel
         # ignore monitor synthetics create and that inherit the kennel_id, we do not directly manage them
         response.reject! { |m| m[:type] == "synthetics alert" } if api_resource == "monitor"
 
-        response
+        response.map { |r| self.class.tag(api_resource, r) }
       end
     end
 
@@ -40,13 +50,13 @@ module Kennel
       response = request :post, "/api/v1/#{api_resource}", body: attributes
       response = response.fetch(:data).first if api_resource == "slo"
       response[:id] = response.delete(:public_id) if api_resource == "synthetics/tests"
-      response
+      self.class.tag(api_resource, response)
     end
 
     def update(api_resource, id, attributes)
       response = request :put, "/api/v1/#{api_resource}/#{id}", body: attributes
       response[:id] = response.delete(:public_id) if api_resource == "synthetics/tests"
-      response
+      self.class.tag(api_resource, response)
     end
 
     # - force=true to not dead-lock on dependent monitors+slos

data/lib/kennel/attribute_differ.rb

@@ -46,13 +46,13 @@ module Kennel
       Diff::LCS.sdiff(old.split("\n", -1), new.split("\n", -1)).flat_map do |diff|
         case diff.action
         when "-"
-          Utils.color(:red, "- #{diff.old_element}")
+          Console.color(:red, "- #{diff.old_element}")
         when "+"
-          Utils.color(:green, "+ #{diff.new_element}")
+          Console.color(:green, "+ #{diff.new_element}")
         when "!"
           [
-            Utils.color(:red, "- #{diff.old_element}"),
-            Utils.color(:green, "+ #{diff.new_element}")
+            Console.color(:red, "- #{diff.old_element}"),
+            Console.color(:green, "+ #{diff.new_element}")
           ]
         else
           "  #{diff.old_element}"
@@ -61,12 +61,12 @@ module Kennel
     end
 
     def truncate(message)
-      warning = Utils.color(
+      warning = Console.color(
         :magenta,
         "  (Diff for this item truncated after #{@max_diff_lines} lines. " \
         "Rerun with MAX_DIFF_LINES=#{@max_diff_lines * 2} to see more)"
       )
-      Utils.truncate_lines(message, to: @max_diff_lines, warning: warning)
+      StringUtils.truncate_lines(message, to: @max_diff_lines, warning: warning)
     end
 
     # TODO: use awesome-print or similar, but it has too many monkey-patches

data/lib/kennel/console.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+module Kennel
+  module Console
+    COLORS = { red: 31, green: 32, yellow: 33, cyan: 36, magenta: 35, default: 0 }.freeze
+
+    class TeeIO < IO
+      def initialize(ios)
+        super(0) # called with fake file descriptor 0, so we can call super and get a proper class
+        @ios = ios
+      end
+
+      def write(string)
+        @ios.each { |io| io.write string }
+      end
+    end
+
+    class << self
+      def ask?(question)
+        Kennel.err.printf color(:red, "#{question} -  press 'y' to continue: ", force: true)
+        begin
+          STDIN.gets.chomp == "y"
+        rescue Interrupt # do not show a backtrace if user decides to Ctrl+C here
+          Kennel.err.print "\n"
+          exit 1
+        end
+      end
+
+      def color(color, text, force: false)
+        return text unless force || Kennel.out.tty?
+
+        "\e[#{COLORS.fetch(color)}m#{text}\e[0m"
+      end
+
+      def capture_stdout
+        old = Kennel.out
+        Kennel.out = StringIO.new
+        yield
+        Kennel.out.string
+      ensure
+        Kennel.out = old
+      end
+
+      def capture_stderr
+        old = Kennel.err
+        Kennel.err = StringIO.new
+        yield
+        Kennel.err.string
+      ensure
+        Kennel.err = old
+      end
+
+      def tee_output
+        old_stdout = Kennel.out
+        old_stderr = Kennel.err
+        capture = StringIO.new
+        Kennel.out = TeeIO.new([capture, Kennel.out])
+        Kennel.err = TeeIO.new([capture, Kennel.err])
+        yield
+        capture.string
+      ensure
+        Kennel.out = old_stdout
+        Kennel.err = old_stderr
+      end
+    end
+  end
+end

data/lib/kennel/file_cache.rb

@@ -57,7 +57,9 @@ module Kennel
     # - delete expired keys
     # - delete what would be deleted anyway when updating
     def expire_old_data
-      @data.reject! { |_, (_, (_, cv), expires)| expires < @now || cv != @cache_version }
+      @data.reject! do |(_api_resource, _id), (_value, (_key_version, cache_version), expires)|
+        expires < @now || cache_version != @cache_version
+      end
     end
   end
 end

data/lib/kennel/github_reporter.rb

@@ -24,7 +24,7 @@ module Kennel
     end
 
     def report(&block)
-      output = Utils.tee_output(&block).strip
+      output = Console.tee_output(&block).strip
     rescue StandardError
       output = "Error:\n#{$ERROR_INFO.message}"
       raise

data/lib/kennel/importer.rb

@@ -33,7 +33,7 @@ module Kennel
           model.remove_tracking_id(data)
           tracking_id.split(":").last
         else
-          Kennel::Utils.parameterize(title)
+          Kennel::StringUtils.parameterize(title)
         end
 
       case resource

data/lib/kennel/models/base.rb

@@ -10,7 +10,7 @@ module Kennel
       SETTING_OVERRIDABLE_METHODS = [:name, :kennel_id].freeze
 
       def kennel_id
-        @kennel_id ||= Utils.snake_case kennel_id_base
+        @kennel_id ||= StringUtils.snake_case kennel_id_base
       end
 
       def name

data/lib/kennel/models/dashboard.rb

@@ -202,7 +202,7 @@ module Kennel
         end
       end
 
-      def validate_update!(_actuals, diffs)
+      def validate_update!(diffs)
         _, path, from, to = diffs.find { |diff| diff[1] == "layout_type" }
         invalid_update!(path, from, to) if path
       end

data/lib/kennel/models/monitor.rb

@@ -137,7 +137,7 @@ module Kennel
         end
       end
 
-      def validate_update!(_actuals, diffs)
+      def validate_update!(diffs)
         # ensure type does not change, but not if it's metric->query which is supported and used by importer.rb
         _, path, from, to = diffs.detect { |_, path, _, _| path == "type" }
         if path && !(from == "metric alert" && to == "query alert")

data/lib/kennel/models/record.rb

@@ -167,7 +167,7 @@ module Kennel
       end
 
       # Can raise DisallowedUpdateError
-      def validate_update!(*)
+      def validate_update!(_diffs)
       end
 
       def invalid_update!(field, old_value, new_value)

data/lib/kennel/string_utils.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+module Kennel
+  module StringUtils
+    class << self
+      def snake_case(string)
+        string
+          .gsub(/::/, "_") # Foo::Bar -> foo_bar
+          .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2') # FOOBar -> foo_bar
+          .gsub(/([a-z\d])([A-Z])/, '\1_\2') # fooBar -> foo_bar
+          .tr("-", "_") # foo-bar -> foo_bar
+          .downcase
+      end
+
+      # for child projects, not used internally
+      def title_case(string)
+        string.split(/[\s_]/).map(&:capitalize) * " "
+      end
+
+      # simplified version of https://apidock.com/rails/ActiveSupport/Inflector/parameterize
+      def parameterize(string)
+        string
+          .downcase
+          .gsub(/[^a-z0-9\-_]+/, "-") # remove unsupported
+          .gsub(/-{2,}/, "-") # remove duplicates
+          .gsub(/^-|-$/, "") # remove leading/trailing
+      end
+
+      def truncate_lines(text, to:, warning:)
+        lines = text.split(/\n/, to + 1)
+        lines[-1] = warning if lines.size > to
+        lines.join("\n")
+      end
+    end
+  end
+end

data/lib/kennel/syncer.rb

@@ -6,170 +6,121 @@ module Kennel
     LINE_UP = "\e[1A\033[K" # go up and clear
 
     Plan = Struct.new(:changes, keyword_init: true)
+
+    InternalPlan = Struct.new(:creates, :updates, :deletes) do
+      def empty?
+        creates.empty? && updates.empty? && deletes.empty?
+      end
+    end
+
     Change = Struct.new(:type, :api_resource, :tracking_id, :id)
 
     def initialize(api, expected, actual, strict_imports: true, project_filter: nil, tracking_id_filter: nil)
       @api = api
-      @expected = Set.new expected # need Set to speed up deletion
-      @actual = actual
       @strict_imports = strict_imports
       @project_filter = project_filter
       @tracking_id_filter = tracking_id_filter
 
-      @attribute_differ = AttributeDiffer.new
+      @resolver = Resolver.new(expected: expected, project_filter: @project_filter, tracking_id_filter: @tracking_id_filter)
 
-      calculate_changes
-      validate_changes
-      prevent_irreversible_partial_updates
+      internal_plan = calculate_changes(expected: expected, actual: actual)
+      validate_changes(internal_plan)
+      @internal_plan = internal_plan
 
-      @warnings.each { |message| Kennel.out.puts Utils.color(:yellow, "Warning: #{message}") }
+      @warnings.each { |message| Kennel.out.puts Console.color(:yellow, "Warning: #{message}") }
     end
 
     def plan
+      ip = @internal_plan
       Plan.new(
-        changes:
-          @create.map { |_id, e, _a| Change.new(:create, e.class.api_resource, e.tracking_id, nil) } +
-            @update.map { |id, e, _a| Change.new(:update, e.class.api_resource, e.tracking_id, id) } +
-            @delete.map { |id, _e, a| Change.new(:delete, a.fetch(:klass).api_resource, a.fetch(:tracking_id), id) }
+        changes: (ip.creates + ip.updates + ip.deletes).map(&:change)
       )
     end
 
     def print_plan
-      Kennel.out.puts "Plan:"
-      if noop?
-        Kennel.out.puts Utils.color(:green, "Nothing to do")
-      else
-        print_changes "Create", @create, :green
-        print_changes "Update", @update, :yellow
-        print_changes "Delete", @delete, :red
-      end
+      PlanDisplayer.new.display(internal_plan)
     end
 
     def confirm
-      return false if noop?
+      return false if internal_plan.empty?
       return true if ENV["CI"] || !STDIN.tty? || !Kennel.err.tty?
-      Utils.ask("Execute Plan ?")
+      Console.ask?("Execute Plan ?")
     end
 
     def update
       changes = []
 
-      each_resolved @create do |_, e|
-        message = "#{e.class.api_resource} #{e.tracking_id}"
+      internal_plan.deletes.each do |item|
+        message = "#{item.api_resource} #{item.tracking_id} #{item.id}"
+        Kennel.out.puts "Deleting #{message}"
+        @api.delete item.api_resource, item.id
+        changes << item.change
+        Kennel.out.puts "#{LINE_UP}Deleted #{message}"
+      end
+
+      resolver.each_resolved internal_plan.creates do |item|
+        message = "#{item.api_resource} #{item.tracking_id}"
         Kennel.out.puts "Creating #{message}"
-        reply = @api.create e.class.api_resource, e.as_json
-        Utils.inline_resource_metadata reply, e.class
+        reply = @api.create item.api_resource, item.expected.as_json
         id = reply.fetch(:id)
-        changes << Change.new(:create, e.class.api_resource, e.tracking_id, id)
-        populate_id_map [], [reply] # allow resolving ids we could previously no resolve
-        Kennel.out.puts "#{LINE_UP}Created #{message} #{e.class.url(id)}"
+        changes << item.change(id)
+        resolver.add_actual [reply] # allow resolving ids we could previously not resolve
+        Kennel.out.puts "#{LINE_UP}Created #{message} #{item.url(id)}"
       end
 
-      each_resolved @update do |id, e|
-        message = "#{e.class.api_resource} #{e.tracking_id} #{e.class.url(id)}"
+      resolver.each_resolved internal_plan.updates do |item|
+        message = "#{item.api_resource} #{item.tracking_id} #{item.url}"
         Kennel.out.puts "Updating #{message}"
-        @api.update e.class.api_resource, id, e.as_json
-        changes << Change.new(:update, e.class.api_resource, e.tracking_id, id)
+        @api.update item.api_resource, item.id, item.expected.as_json
+        changes << item.change
         Kennel.out.puts "#{LINE_UP}Updated #{message}"
       end
 
-      @delete.each do |id, _, a|
-        klass = a.fetch(:klass)
-        message = "#{klass.api_resource} #{a.fetch(:tracking_id)} #{id}"
-        Kennel.out.puts "Deleting #{message}"
-        @api.delete klass.api_resource, id
-        changes << Change.new(:delete, klass.api_resource, a.fetch(:tracking_id), id)
-        Kennel.out.puts "#{LINE_UP}Deleted #{message}"
-      end
-
       Plan.new(changes: changes)
     end
 
     private
 
-    # loop over items until everything is resolved or crash when we get stuck
-    # this solves cases like composite monitors depending on each other or monitor->monitor slo->slo monitor chains
-    def each_resolved(list)
-      list = list.dup
-      loop do
-        return if list.empty?
-        list.reject! do |id, e|
-          if resolved?(e)
-            yield id, e
-            true
-          else
-            false
-          end
-        end ||
-          assert_resolved(list[0][1]) # resolve something or show a circular dependency error
-      end
-    end
-
-    # TODO: optimize by storing an instance variable if already resolved
-    def resolved?(e)
-      assert_resolved e
-      true
-    rescue UnresolvableIdError
-      false
-    end
-
-    # raises UnresolvableIdError when not resolved
-    def assert_resolved(e)
-      resolve_linked_tracking_ids! [e], force: true
-    end
-
-    def noop?
-      @create.empty? && @update.empty? && @delete.empty?
-    end
+    attr_reader :resolver, :internal_plan
 
-    def calculate_changes
+    def calculate_changes(expected:, actual:)
       @warnings = []
-      @id_map = IdMap.new
 
       Progress.progress "Diffing" do
-        populate_id_map @expected, @actual
-        filter_actual! @actual
-        resolve_linked_tracking_ids! @expected # resolve dependencies to avoid diff
-        @expected.each(&:add_tracking_id) # avoid diff with actual, which has tracking_id
+        resolver.add_actual actual
+        filter_actual! actual
+        resolver.resolve_as_much_as_possible(expected) # resolve as many dependencies as possible to reduce the diff
 
         # see which expected match the actual
-        matching, unmatched_expected, unmatched_actual = partition_matched_expected
+        matching, unmatched_expected, unmatched_actual = MatchedExpected.partition(expected, actual)
         validate_expected_id_not_missing unmatched_expected
         fill_details! matching # need details to diff later
 
         # update matching if needed
-        @update = matching.map do |e, a|
+        updates = matching.map do |e, a|
+          # Refuse to "adopt" existing items into kennel while running with a filter (i.e. on a branch).
+          # Without this, we'd adopt an item, then the next CI run would delete it
+          # (instead of "unadopting" it).
+          e.add_tracking_id unless @project_filter && a.fetch(:tracking_id).nil?
           id = a.fetch(:id)
           diff = e.diff(a)
-          [id, e, a, diff] if diff.any?
+          a[:id] = id
+          Types::PlannedUpdate.new(e, a, diff) if diff.any?
         end.compact
 
         # delete previously managed
-        @delete = unmatched_actual.map { |a| [a.fetch(:id), nil, a] if a.fetch(:tracking_id) }.compact
+        deletes = unmatched_actual.map { |a| Types::PlannedDelete.new(a) if a.fetch(:tracking_id) }.compact
 
         # unmatched expected need to be created
-        @create = unmatched_expected.map { |e| [nil, e] }
+        unmatched_expected.each(&:add_tracking_id)
+        creates = unmatched_expected.map { |e| Types::PlannedCreate.new(e) }
 
         # order to avoid deadlocks
-        @delete.sort_by! { |_, _, a| DELETE_ORDER.index a.fetch(:klass).api_resource }
-        @update.sort_by! { |_, e, _| DELETE_ORDER.index e.class.api_resource } # slo needs to come before slo alert
-      end
-    end
+        deletes.sort_by! { |item| DELETE_ORDER.index item.api_resource }
+        updates.sort_by! { |item| DELETE_ORDER.index item.api_resource } # slo needs to come before slo alert
 
-    def partition_matched_expected
-      lookup_map = matching_expected_lookup_map
-      unmatched_expected = @expected.dup
-      unmatched_actual = []
-      matched = []
-      @actual.each do |a|
-        e = matching_expected(a, lookup_map)
-        if e && unmatched_expected.delete?(e)
-          matched << [e, a]
-        else
-          unmatched_actual << a
-        end
-      end.compact
-      [matched, unmatched_expected, unmatched_actual]
+        InternalPlan.new(creates, updates, deletes)
+      end
     end
 
     # fill details of things we need to compare
@@ -190,108 +141,242 @@ module Kennel
       end
     end
 
-    # index list by all the thing we look up by: tracking id and actual id
-    def matching_expected_lookup_map
-      @expected.each_with_object({}) do |e, all|
-        keys = [e.tracking_id]
-        keys << "#{e.class.api_resource}:#{e.id}" if e.id
-        keys.compact.each do |key|
-          raise "Lookup #{key} is duplicated" if all[key]
-          all[key] = e
-        end
+    # We've already validated the desired objects ('generated') in isolation.
+    # Now that we have made the plan, we can perform some more validation.
+    def validate_changes(internal_plan)
+      internal_plan.updates.each do |item|
+        item.expected.validate_update!(item.diff)
       end
     end
 
-    def matching_expected(a, map)
-      klass = a.fetch(:klass)
-      map["#{klass.api_resource}:#{a.fetch(:id)}"] || map[a.fetch(:tracking_id)]
+    def filter_actual!(actual)
+      if @tracking_id_filter
+        actual.select! do |a|
+          tracking_id = a.fetch(:tracking_id)
+          !tracking_id || @tracking_id_filter.include?(tracking_id)
+        end
+      elsif @project_filter
+        project_prefixes = @project_filter.map { |p| "#{p}:" }
+        actual.select! do |a|
+          tracking_id = a.fetch(:tracking_id)
+          !tracking_id || tracking_id.start_with?(*project_prefixes)
+        end
+      end
     end
 
-    def print_changes(step, list, color)
-      return if list.empty?
-      list.each do |_, e, a, diff|
-        klass = (e ? e.class : a.fetch(:klass))
-        Kennel.out.puts Utils.color(color, "#{step} #{klass.api_resource} #{e&.tracking_id || a.fetch(:tracking_id)}")
-        diff&.each { |args| Kennel.out.puts @attribute_differ.format(*args) } # only for update
+    class PlanDisplayer
+      def initialize
+        @attribute_differ = AttributeDiffer.new
       end
-    end
 
-    # We've already validated the desired objects ('generated') in isolation.
-    # Now that we have made the plan, we can perform some more validation.
-    def validate_changes
-      @update.each do |_, expected, actuals, diffs|
-        expected.validate_update!(actuals, diffs)
+      def display(internal_plan)
+        Kennel.out.puts "Plan:"
+        if internal_plan.empty?
+          Kennel.out.puts Console.color(:green, "Nothing to do")
+        else
+          print_changes "Create", internal_plan.creates, :green
+          print_changes "Update", internal_plan.updates, :yellow
+          print_changes "Delete", internal_plan.deletes, :red
+        end
+      end
+
+      private
+
+      def print_changes(step, list, color)
+        return if list.empty?
+        list.each do |item|
+          Kennel.out.puts Console.color(color, "#{step} #{item.api_resource} #{item.tracking_id}")
+          if item.class::TYPE == :update
+            item.diff.each { |args| Kennel.out.puts @attribute_differ.format(*args) } # only for update
+          end
+        end
       end
     end
 
-    # - do not add tracking-id when working with existing ids on a branch,
-    #   so resource do not get deleted when running an update on master (for example merge->CI)
-    # - ideally we'd never add tracking in the first place, but when adding tracking we do not know the diff yet
-    def prevent_irreversible_partial_updates
-      return unless @project_filter # full update, so we are not on a branch
-      @update.select! do |_, e, _, diff| # ensure clean diff, by removing noop-update
-        next true unless e.id # safe to add tracking when not having id
-
-        diff.select! do |field_diff|
-          (_, field, actual) = field_diff
-          # TODO: refactor this so TRACKING_FIELD stays record-private
-          next true if e.class::TRACKING_FIELD != field.to_sym # need to sym here because Hashdiff produces strings
-          next true if e.class.parse_tracking_id(field.to_sym => actual) # already has tracking id
-
-          field_diff[3] = e.remove_tracking_id # make `rake plan` output match what we are sending
-          actual != field_diff[3] # discard diff if now nothing changes
+    class Resolver
+      def initialize(expected:, project_filter:, tracking_id_filter:)
+        @id_map = IdMap.new
+        @project_filter = project_filter
+        @tracking_id_filter = tracking_id_filter
+
+        # mark everything as new
+        expected.each do |e|
+          id_map.set(e.class.api_resource, e.tracking_id, IdMap::NEW)
+          if e.class.api_resource == "synthetics/tests"
+            id_map.set(Kennel::Models::Monitor.api_resource, e.tracking_id, IdMap::NEW)
+          end
         end
+      end
 
-        diff.any?
+      def add_actual(actual)
+        # override resources that exist with their id
+        project_prefixes = project_filter&.map { |p| "#{p}:" }
+
+        actual.each do |a|
+          # ignore when not managed by kennel
+          next unless tracking_id = a.fetch(:tracking_id)
+
+          # ignore when deleted from the codebase
+          # (when running with filters we cannot see the other resources in the codebase)
+          api_resource = a.fetch(:klass).api_resource
+          next if
+            !id_map.get(api_resource, tracking_id) &&
+            (!project_prefixes || tracking_id.start_with?(*project_prefixes)) &&
+            (!tracking_id_filter || tracking_id_filter.include?(tracking_id))
+
+          id_map.set(api_resource, tracking_id, a.fetch(:id))
+          if a.fetch(:klass).api_resource == "synthetics/tests"
+            id_map.set(Kennel::Models::Monitor.api_resource, tracking_id, a.fetch(:monitor_id))
+          end
+        end
       end
-    end
 
-    def populate_id_map(expected, actual)
-      # mark everything as new
-      expected.each do |e|
-        @id_map.set(e.class.api_resource, e.tracking_id, IdMap::NEW)
-        if e.class.api_resource == "synthetics/tests"
-          @id_map.set(Kennel::Models::Monitor.api_resource, e.tracking_id, IdMap::NEW)
+      def resolve_as_much_as_possible(expected)
+        expected.each do |e|
+          e.resolve_linked_tracking_ids!(id_map, force: false)
         end
       end
 
-      # override resources that exist with their id
-      project_prefixes = @project_filter&.map { |p| "#{p}:" }
-      actual.each do |a|
-        # ignore when not managed by kennel
-        next unless tracking_id = a.fetch(:tracking_id)
-
-        # ignore when deleted from the codebase
-        # (when running with filters we cannot see the other resources in the codebase)
-        api_resource = a.fetch(:klass).api_resource
-        next if
-          !@id_map.get(api_resource, tracking_id) &&
-          (!project_prefixes || tracking_id.start_with?(*project_prefixes)) &&
-          (!@tracking_id_filter || @tracking_id_filter.include?(tracking_id))
-
-        @id_map.set(api_resource, tracking_id, a.fetch(:id))
-        if a[:klass].api_resource == "synthetics/tests"
-          @id_map.set(Kennel::Models::Monitor.api_resource, tracking_id, a.fetch(:monitor_id))
+      # loop over items until everything is resolved or crash when we get stuck
+      # this solves cases like composite monitors depending on each other or monitor->monitor slo->slo monitor chains
+      def each_resolved(list)
+        list = list.dup
+        loop do
+          return if list.empty?
+          list.reject! do |item|
+            if resolved?(item.expected)
+              yield item
+              true
+            else
+              false
+            end
+          end ||
+            assert_resolved(list[0].expected) # resolve something or show a circular dependency error
         end
       end
+
+      private
+
+      attr_reader :id_map, :project_filter, :tracking_id_filter
+
+      # TODO: optimize by storing an instance variable if already resolved
+      def resolved?(e)
+        assert_resolved e
+        true
+      rescue UnresolvableIdError
+        false
+      end
+
+      # raises UnresolvableIdError when not resolved
+      def assert_resolved(e)
+        e.resolve_linked_tracking_ids!(id_map, force: true)
+      end
     end
 
-    def resolve_linked_tracking_ids!(list, force: false)
-      list.each { |e| e.resolve_linked_tracking_ids!(@id_map, force: force) }
+    module MatchedExpected
+      class << self
+        def partition(expected, actual)
+          lookup_map = matching_expected_lookup_map(expected)
+          unmatched_expected = Set.new(expected) # for efficient deletion
+          unmatched_actual = []
+          matched = []
+          actual.each do |a|
+            e = matching_expected(a, lookup_map)
+            if e && unmatched_expected.delete?(e)
+              matched << [e, a]
+            else
+              unmatched_actual << a
+            end
+          end.compact
+          [matched, unmatched_expected.to_a, unmatched_actual]
+        end
+
+        private
+
+        # index list by all the thing we look up by: tracking id and actual id
+        def matching_expected_lookup_map(expected)
+          expected.each_with_object({}) do |e, all|
+            keys = [e.tracking_id]
+            keys << "#{e.class.api_resource}:#{e.id}" if e.id
+            keys.compact.each do |key|
+              raise "Lookup #{key} is duplicated" if all[key]
+              all[key] = e
+            end
+          end
+        end
+
+        def matching_expected(a, map)
+          klass = a.fetch(:klass)
+          map["#{klass.api_resource}:#{a.fetch(:id)}"] || map[a.fetch(:tracking_id)]
+        end
+      end
     end
 
-    def filter_actual!(actual)
-      if @tracking_id_filter
-        actual.select! do |a|
-          tracking_id = a.fetch(:tracking_id)
-          !tracking_id || @tracking_id_filter.include?(tracking_id)
+    module Types
+      class PlannedChange
+        def initialize(klass, tracking_id)
+          @klass = klass
+          @tracking_id = tracking_id
         end
-      elsif @project_filter
-        project_prefixes = @project_filter.map { |p| "#{p}:" }
-        actual.select! do |a|
-          tracking_id = a.fetch(:tracking_id)
-          !tracking_id || tracking_id.start_with?(*project_prefixes)
+
+        def api_resource
+          klass.api_resource
+        end
+
+        def url(id = nil)
+          klass.url(id || self.id)
+        end
+
+        def change(id = nil)
+          Change.new(self.class::TYPE, api_resource, tracking_id, id)
         end
+
+        attr_reader :klass, :tracking_id
+      end
+
+      class PlannedCreate < PlannedChange
+        TYPE = :create
+
+        def initialize(expected)
+          super(expected.class, expected.tracking_id)
+          @expected = expected
+        end
+
+        attr_reader :expected
+      end
+
+      class PlannedUpdate < PlannedChange
+        TYPE = :update
+
+        def initialize(expected, actual, diff)
+          super(expected.class, expected.tracking_id)
+          @expected = expected
+          @actual = actual
+          @diff = diff
+          @id = actual.fetch(:id)
+        end
+
+        def change
+          super(id)
+        end
+
+        attr_reader :expected, :actual, :diff, :id
+      end
+
+      class PlannedDelete < PlannedChange
+        TYPE = :delete
+
+        def initialize(actual)
+          super(actual.fetch(:klass), actual.fetch(:tracking_id))
+          @actual = actual
+          @id = actual.fetch(:id)
+        end
+
+        def change
+          super(id)
+        end
+
+        attr_reader :actual, :id
       end
     end
   end

data/lib/kennel/unmuted_alerts.rb

@@ -22,7 +22,7 @@ module Kennel
             m[:state][:groups].each do |g|
               color = COLORS[g[:status]] || :default
               since = "\t#{time_since(g[:last_triggered_ts])}"
-              Kennel.out.puts "#{Kennel::Utils.color(color, g[:status])}\t#{g[:name]}#{since}"
+              Kennel.out.puts "#{Kennel::Console.color(color, g[:status])}\t#{g[:name]}#{since}"
             end
             Kennel.out.puts
           end

data/lib/kennel/utils.rb

@@ -1,100 +1,11 @@
 # frozen_string_literal: true
 module Kennel
   module Utils
-    COLORS = { red: 31, green: 32, yellow: 33, cyan: 36, magenta: 35, default: 0 }.freeze
-
-    class TeeIO < IO
-      def initialize(ios)
-        super(0) # called with fake file descriptor 0, so we can call super and get a proper class
-        @ios = ios
-      end
-
-      def write(string)
-        @ios.each { |io| io.write string }
-      end
-    end
-
     class << self
-      def snake_case(string)
-        string
-          .gsub(/::/, "_") # Foo::Bar -> foo_bar
-          .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2') # FOOBar -> foo_bar
-          .gsub(/([a-z\d])([A-Z])/, '\1_\2') # fooBar -> foo_bar
-          .tr("-", "_") # foo-bar -> foo_bar
-          .downcase
-      end
-
-      # for child projects, not used internally
-      def title_case(string)
-        string.split(/[\s_]/).map(&:capitalize) * " "
-      end
-
-      # simplified version of https://apidock.com/rails/ActiveSupport/Inflector/parameterize
-      def parameterize(string)
-        string
-          .downcase
-          .gsub(/[^a-z0-9\-_]+/, "-") # remove unsupported
-          .gsub(/-{2,}/, "-") # remove duplicates
-          .gsub(/^-|-$/, "") # remove leading/trailing
-      end
-
       def presence(value)
         value.nil? || value.empty? ? nil : value
       end
 
-      def ask(question)
-        Kennel.err.printf color(:red, "#{question} -  press 'y' to continue: ", force: true)
-        begin
-          STDIN.gets.chomp == "y"
-        rescue Interrupt # do not show a backtrace if user decides to Ctrl+C here
-          Kennel.err.print "\n"
-          exit 1
-        end
-      end
-
-      def color(color, text, force: false)
-        return text unless force || Kennel.out.tty?
-
-        "\e[#{COLORS.fetch(color)}m#{text}\e[0m"
-      end
-
-      def truncate_lines(text, to:, warning:)
-        lines = text.split(/\n/, to + 1)
-        lines[-1] = warning if lines.size > to
-        lines.join("\n")
-      end
-
-      def capture_stdout
-        old = Kennel.out
-        Kennel.out = StringIO.new
-        yield
-        Kennel.out.string
-      ensure
-        Kennel.out = old
-      end
-
-      def capture_stderr
-        old = Kennel.err
-        Kennel.err = StringIO.new
-        yield
-        Kennel.err.string
-      ensure
-        Kennel.err = old
-      end
-
-      def tee_output
-        old_stdout = Kennel.out
-        old_stderr = Kennel.err
-        capture = StringIO.new
-        Kennel.out = TeeIO.new([capture, Kennel.out])
-        Kennel.err = TeeIO.new([capture, Kennel.err])
-        yield
-        capture.string
-      ensure
-        Kennel.out = old_stdout
-        Kennel.err = old_stderr
-      end
-
       def capture_sh(command)
         result = `#{command} 2>&1`
         raise "Command failed:\n#{command}\n#{result}" unless $CHILD_STATUS.success?
@@ -150,11 +61,6 @@ module Kennel
         else []
         end
       end
-
-      def inline_resource_metadata(resource, klass)
-        resource[:klass] = klass
-        resource[:tracking_id] = klass.parse_tracking_id(resource)
-      end
     end
   end
 end

data/lib/kennel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Kennel
-  VERSION = "1.129.0"
+  VERSION = "1.131.0"
 end

data/lib/kennel.rb

@@ -5,6 +5,8 @@ require "zeitwerk"
 require "English"
 
 require "kennel/version"
+require "kennel/console"
+require "kennel/string_utils"
 require "kennel/utils"
 require "kennel/progress"
 require "kennel/filter"
@@ -133,8 +135,7 @@ module Kennel
     def definitions(**kwargs)
       @definitions ||= Progress.progress("Downloading definitions", **kwargs) do
         Utils.parallel(Models::Record.subclasses) do |klass|
-          results = api.list(klass.api_resource, with_downtimes: false) # lookup monitors without adding unnecessary downtime information
-          results.each { |a| Utils.inline_resource_metadata(a, klass) }
+          api.list(klass.api_resource, with_downtimes: false) # lookup monitors without adding unnecessary downtime information
         end.flatten(1)
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kennel
 version: !ruby/object:Gem::Version
-  version: 1.129.0
+  version: 1.131.0
 platform: ruby
 authors:
 - Michael Grosser
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-12-16 00:00:00.000000000 Z
+date: 2023-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: diff-lcs
@@ -90,6 +90,7 @@ files:
 - lib/kennel.rb
 - lib/kennel/api.rb
 - lib/kennel/attribute_differ.rb
+- lib/kennel/console.rb
 - lib/kennel/file_cache.rb
 - lib/kennel/filter.rb
 - lib/kennel/github_reporter.rb
@@ -108,6 +109,7 @@ files:
 - lib/kennel/progress.rb
 - lib/kennel/projects_provider.rb
 - lib/kennel/settings_as_methods.rb
+- lib/kennel/string_utils.rb
 - lib/kennel/subclass_tracking.rb
 - lib/kennel/syncer.rb
 - lib/kennel/tasks.rb