kennel 1.127.0 → 1.129.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ec72f834db1c82d19ba21eabd482e5f46a540b0c9a3e0e51ecc20a80c192a6d7
-  data.tar.gz: c5b549fb934ce3e274d88149df509b4dcf16e1adc6be3271e94b3101dbee1ad7
+  metadata.gz: c7690d961a150ea0ba0649edcab513c22ec828988fec2ab631636497274305c7
+  data.tar.gz: c5bc7244c4168466f5167e5c322f3a2cdb8e6b9b8ec23a2d12fb343a7648ae06
 SHA512:
-  metadata.gz: e0bff1a22428c19da30025f79e8e7904675ea27d79eb2cfdf1a6cf0638a33efbdee44c10f898480b9e7d91d73296368e3bab58ce495282beb43cb13c975c9153
-  data.tar.gz: 445cf62d6e525c42808a42ba80e941829cab5c679ef74f71590799ec46e93f3d41ff8162bed775342510c042249a2acfaf10dea06507a2b5f7a7b0c8a450fb3c
+  metadata.gz: 51b47377a8cf688dafd17600b543cc1734eb8511a6ed1b627d8b1180bf70fc66c86f9d4a8e4b2a3ff5e7223edf366e5483d983d9465c5907aaa07f5b9c36ef46
+  data.tar.gz: 698c23a4de558382467342b90e7d3eac143a0279653efe9f1b711b3e114a37d8d72f4a9c5d97e652a6646007298f5e93f296c1125caaa4c498195089b0cac730

data/lib/kennel/attribute_differ.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "diff/lcs"
+
+module Kennel
+  class AttributeDiffer
+    def initialize
+      # min '2' because: -1 makes no sense, 0 does not work with * 2 math, 1 says '1 lines'
+      @max_diff_lines = [Integer(ENV.fetch("MAX_DIFF_LINES", "50")), 2].max
+      super
+    end
+
+    def format(type, field, old, new = nil)
+      multiline = false
+      if type == "+"
+        temp = pretty_inspect(new)
+        new = pretty_inspect(old)
+        old = temp
+      elsif old.is_a?(String) && new.is_a?(String) && (old.include?("\n") || new.include?("\n"))
+        multiline = true
+      else # ~ and -
+        old = pretty_inspect(old)
+        new = pretty_inspect(new)
+      end
+
+      message =
+        if multiline
+          "  #{type}#{field}\n" +
+            multiline_diff(old, new).map { |l| "    #{l}" }.join("\n")
+        elsif (old + new).size > 100
+          "  #{type}#{field}\n" \
+          "    #{old} ->\n" \
+          "    #{new}"
+        else
+          "  #{type}#{field} #{old} -> #{new}"
+        end
+
+      truncate(message)
+    end
+
+    private
+
+    # display diff for multi-line strings
+    # must stay readable when color is off too
+    def multiline_diff(old, new)
+      Diff::LCS.sdiff(old.split("\n", -1), new.split("\n", -1)).flat_map do |diff|
+        case diff.action
+        when "-"
+          Utils.color(:red, "- #{diff.old_element}")
+        when "+"
+          Utils.color(:green, "+ #{diff.new_element}")
+        when "!"
+          [
+            Utils.color(:red, "- #{diff.old_element}"),
+            Utils.color(:green, "+ #{diff.new_element}")
+          ]
+        else
+          "  #{diff.old_element}"
+        end
+      end
+    end
+
+    def truncate(message)
+      warning = Utils.color(
+        :magenta,
+        "  (Diff for this item truncated after #{@max_diff_lines} lines. " \
+        "Rerun with MAX_DIFF_LINES=#{@max_diff_lines * 2} to see more)"
+      )
+      Utils.truncate_lines(message, to: @max_diff_lines, warning: warning)
+    end
+
+    # TODO: use awesome-print or similar, but it has too many monkey-patches
+    # https://github.com/amazing-print/amazing_print/issues/36
+    def pretty_inspect(object)
+      string = object.inspect.dup
+      string.gsub!(/:([a-z_]+)=>/, "\\1: ")
+      10.times do
+        string.gsub!(/{(\S.*?\S)}/, "{ \\1 }") || break
+      end
+      string
+    end
+  end
+end

data/lib/kennel/importer.rb

@@ -189,7 +189,7 @@ module Kennel
 
     # important to the front and rest deterministic
     def sort_hash(hash)
-      Hash[hash.sort_by { |k, _| [SORT_ORDER.index(k) || 999, k] }]
+      hash.sort_by { |k, _| [SORT_ORDER.index(k) || 999, k] }.to_h
     end
   end
 end

data/lib/kennel/models/record.rb

@@ -35,8 +35,8 @@ module Kennel
         :klass, :tracking_id # added by syncer.rb
       ].freeze
       ALLOWED_KENNEL_ID_CHARS = "a-zA-Z_\\d.-"
-      ALLOWED_KENNEL_ID_FULL = "[#{ALLOWED_KENNEL_ID_CHARS}]+:[#{ALLOWED_KENNEL_ID_CHARS}]+"
-      ALLOWED_KENNEL_ID_REGEX = /\A#{ALLOWED_KENNEL_ID_FULL}\z/.freeze
+      ALLOWED_KENNEL_ID_FULL = "[#{ALLOWED_KENNEL_ID_CHARS}]+:[#{ALLOWED_KENNEL_ID_CHARS}]+".freeze
+      ALLOWED_KENNEL_ID_REGEX = /\A#{ALLOWED_KENNEL_ID_FULL}\z/
 
       settings :id, :kennel_id
 
@@ -52,7 +52,7 @@ module Kennel
         end
 
         def api_resource_map
-          subclasses.map { |s| [s.api_resource, s] }.to_h
+          subclasses.to_h { |s| [s.api_resource, s] }
         end
 
         def parse_tracking_id(a)
@@ -98,6 +98,8 @@ module Kennel
 
         self.class.send(:normalize, expected, actual)
 
+        return [] if actual == expected # Hashdiff is slow, this is fast
+
         # strict: ignore Integer vs Float
         # similarity: show diff when not 100% similar
         # use_lcs: saner output

data/lib/kennel/models/synthetic_test.rb

@@ -3,8 +3,7 @@ module Kennel
   module Models
     class SyntheticTest < Record
       TRACKING_FIELD = :message
-      DEFAULTS = {
-      }.freeze
+      DEFAULTS = {}.freeze
       READONLY_ATTRIBUTES = superclass::READONLY_ATTRIBUTES + [:status, :monitor_id]
       LOCATIONS = ["aws:ca-central-1", "aws:eu-north-1", "aws:eu-west-1", "aws:eu-west-3", "aws:eu-west-2", "aws:ap-south-1", "aws:us-west-2", "aws:us-west-1", "aws:sa-east-1", "aws:us-east-2", "aws:ap-northeast-1", "aws:ap-northeast-2", "aws:eu-central-1", "aws:ap-southeast-2", "aws:ap-southeast-1"].freeze
 

data/lib/kennel/parts_serializer.rb

@@ -8,13 +8,9 @@ module Kennel
 
     def write(parts)
       Progress.progress "Storing" do
-        if filter.tracking_id_filter
-          write_changed(parts)
-        else
-          old = old_paths
-          used = write_changed(parts)
-          (old - used).uniq.each { |p| FileUtils.rm_rf(p) }
-        end
+        existing = existing_files_and_folders
+        used = write_changed(parts)
+        FileUtils.rm_rf(existing - used)
       end
     end
 
@@ -26,31 +22,36 @@ module Kennel
       used = []
 
       Utils.parallel(parts, max: 2) do |part|
-        path = "generated/#{part.tracking_id.tr("/", ":").sub(":", "/")}.json"
+        path = path_for_tracking_id(part.tracking_id)
 
-        used << File.dirname(path) # only 1 level of sub folders, so this is enough
+        used << File.dirname(path) # we have 1 level of sub folders, so this is enough
         used << path
 
-        payload = part.as_json.merge(api_resource: part.class.api_resource)
-        write_file_if_necessary(path, JSON.pretty_generate(payload) << "\n")
+        content = part.as_json.merge(api_resource: part.class.api_resource)
+        write_file_if_necessary(path, content)
       end
 
       used
     end
 
-    def directories_to_clean_up
-      if filter.project_filter
-        filter.project_filter.map { |project| "generated/#{project}" }
+    def existing_files_and_folders
+      if filter.tracking_id_filter
+        filter.tracking_id_filter.map { |tracking_id| path_for_tracking_id(tracking_id) }
+      elsif filter.project_filter
+        filter.project_filter.flat_map { |project| Dir["generated/#{project}/*"] }
       else
-        ["generated"]
+        Dir["generated/**/*"] # also includes folders so we clean up empty directories
       end
     end
 
-    def old_paths
-      Dir["{#{directories_to_clean_up.join(",")}}/**/*"]
+    def path_for_tracking_id(tracking_id)
+      "generated/#{tracking_id.tr("/", ":").sub(":", "/")}.json"
     end
 
     def write_file_if_necessary(path, content)
+      # NOTE: always generating is faster than JSON.load-ing and comparing
+      content = JSON.pretty_generate(content) << "\n"
+
       # 99% case
       begin
         return if File.read(path) == content

data/lib/kennel/progress.rb

@@ -4,8 +4,8 @@ require "benchmark"
 module Kennel
   class Progress
     # print what we are doing and a spinner until it is done ... then show how long it took
-    def self.progress(name, interval: 0.2, &block)
-      return progress_no_tty(name, &block) unless Kennel.err.tty?
+    def self.progress(name, interval: 0.2, plain: false, &block)
+      return progress_no_tty(name, &block) if plain || !Kennel.err.tty?
 
       Kennel.err.print "#{name} ... "
 

data/lib/kennel/syncer.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "diff/lcs"
-
 module Kennel
   class Syncer
     DELETE_ORDER = ["dashboard", "slo", "monitor", "synthetics/tests"].freeze # dashboards references monitors + slos, slos reference monitors
@@ -10,34 +8,41 @@ module Kennel
     Plan = Struct.new(:changes, keyword_init: true)
     Change = Struct.new(:type, :api_resource, :tracking_id, :id)
 
-    def initialize(api, expected, kennel:, project_filter: nil, tracking_id_filter: nil)
+    def initialize(api, expected, actual, strict_imports: true, project_filter: nil, tracking_id_filter: nil)
       @api = api
-      @kennel = kennel
+      @expected = Set.new expected # need Set to speed up deletion
+      @actual = actual
+      @strict_imports = strict_imports
       @project_filter = project_filter
       @tracking_id_filter = tracking_id_filter
-      @expected = Set.new expected # need set to speed up deletion
-      calculate_diff
-      validate_plan
+
+      @attribute_differ = AttributeDiffer.new
+
+      calculate_changes
+      validate_changes
       prevent_irreversible_partial_updates
+
+      @warnings.each { |message| Kennel.out.puts Utils.color(:yellow, "Warning: #{message}") }
     end
 
     def plan
+      Plan.new(
+        changes:
+          @create.map { |_id, e, _a| Change.new(:create, e.class.api_resource, e.tracking_id, nil) } +
+            @update.map { |id, e, _a| Change.new(:update, e.class.api_resource, e.tracking_id, id) } +
+            @delete.map { |id, _e, a| Change.new(:delete, a.fetch(:klass).api_resource, a.fetch(:tracking_id), id) }
+      )
+    end
+
+    def print_plan
       Kennel.out.puts "Plan:"
       if noop?
         Kennel.out.puts Utils.color(:green, "Nothing to do")
       else
-        @warnings.each { |message| Kennel.out.puts Utils.color(:yellow, "Warning: #{message}") }
-        print_plan "Create", @create, :green
-        print_plan "Update", @update, :yellow
-        print_plan "Delete", @delete, :red
+        print_changes "Create", @create, :green
+        print_changes "Update", @update, :yellow
+        print_changes "Delete", @delete, :red
       end
-
-      Plan.new(
-        changes:
-          @create.map { |_id, e, _a| Change.new(:create, e.class.api_resource, e.tracking_id, nil) } +
-          @update.map { |id, e, _a| Change.new(:update, e.class.api_resource, e.tracking_id, id) } +
-          @delete.map { |id, _e, a| Change.new(:delete, a.fetch(:klass).api_resource, a.fetch(:tracking_id), id) }
-      )
     end
 
     def confirm
@@ -53,7 +58,7 @@ module Kennel
         message = "#{e.class.api_resource} #{e.tracking_id}"
         Kennel.out.puts "Creating #{message}"
         reply = @api.create e.class.api_resource, e.as_json
-        cache_metadata reply, e.class
+        Utils.inline_resource_metadata reply, e.class
         id = reply.fetch(:id)
         changes << Change.new(:create, e.class.api_resource, e.tracking_id, id)
         populate_id_map [], [reply] # allow resolving ids we could previously no resolve
@@ -82,8 +87,6 @@ module Kennel
 
     private
 
-    attr_reader :kennel
-
     # loop over items until everything is resolved or crash when we get stuck
     # this solves cases like composite monitors depending on each other or monitor->monitor slo->slo monitor chains
     def each_resolved(list)
@@ -119,73 +122,67 @@ module Kennel
       @create.empty? && @update.empty? && @delete.empty?
     end
 
-    def calculate_diff
+    def calculate_changes
       @warnings = []
-      @update = []
-      @delete = []
       @id_map = IdMap.new
 
-      actual = Progress.progress("Downloading definitions") { download_definitions }
-
       Progress.progress "Diffing" do
-        populate_id_map @expected, actual
-        filter_actual! actual
+        populate_id_map @expected, @actual
+        filter_actual! @actual
         resolve_linked_tracking_ids! @expected # resolve dependencies to avoid diff
+        @expected.each(&:add_tracking_id) # avoid diff with actual, which has tracking_id
 
-        @expected.each(&:add_tracking_id) # avoid diff with actual
+        # see which expected match the actual
+        matching, unmatched_expected, unmatched_actual = partition_matched_expected
+        validate_expected_id_not_missing unmatched_expected
+        fill_details! matching # need details to diff later
 
-        lookup_map = matching_expected_lookup_map
-        items = actual.map do |a|
-          e = matching_expected(a, lookup_map)
-          if e && @expected.delete?(e)
-            [e, a]
-          else
-            [nil, a]
-          end
-        end
+        # update matching if needed
+        @update = matching.map do |e, a|
+          id = a.fetch(:id)
+          diff = e.diff(a)
+          [id, e, a, diff] if diff.any?
+        end.compact
 
-        # fill details of things we need to compare
-        details = items.map { |e, a| a if e && e.class.api_resource == "dashboard" }.compact
-        @api.fill_details! "dashboard", details
+        # delete previously managed
+        @delete = unmatched_actual.map { |a| [a.fetch(:id), nil, a] if a.fetch(:tracking_id) }.compact
 
-        # pick out things to update or delete
-        items.each do |e, a|
-          id = a.fetch(:id)
-          if e
-            diff = e.diff(a) # slow ...
-            if diff.any?
-              @update << [id, e, a, diff]
-            end
-          elsif a.fetch(:tracking_id) # was previously managed
-            @delete << [id, nil, a]
-          end
-        end
+        # unmatched expected need to be created
+        @create = unmatched_expected.map { |e| [nil, e] }
 
-        ensure_all_ids_found
-        @create = @expected.map { |e| [nil, e] }
+        # order to avoid deadlocks
         @delete.sort_by! { |_, _, a| DELETE_ORDER.index a.fetch(:klass).api_resource }
         @update.sort_by! { |_, e, _| DELETE_ORDER.index e.class.api_resource } # slo needs to come before slo alert
       end
     end
 
-    def download_definitions
-      Utils.parallel(Models::Record.subclasses) do |klass|
-        results = @api.list(klass.api_resource, with_downtimes: false) # lookup monitors without adding unnecessary downtime information
-        results = results[results.keys.first] if results.is_a?(Hash) # dashboards are nested in {dashboards: []}
-        results.each { |a| cache_metadata(a, klass) }
-      end.flatten(1)
+    def partition_matched_expected
+      lookup_map = matching_expected_lookup_map
+      unmatched_expected = @expected.dup
+      unmatched_actual = []
+      matched = []
+      @actual.each do |a|
+        e = matching_expected(a, lookup_map)
+        if e && unmatched_expected.delete?(e)
+          matched << [e, a]
+        else
+          unmatched_actual << a
+        end
+      end.compact
+      [matched, unmatched_expected, unmatched_actual]
     end
 
-    def cache_metadata(a, klass)
-      a[:klass] = klass
-      a[:tracking_id] = a.fetch(:klass).parse_tracking_id(a)
+    # fill details of things we need to compare
+    def fill_details!(details_needed)
+      details_needed = details_needed.map { |e, a| a if e && e.class.api_resource == "dashboard" }.compact
+      @api.fill_details! "dashboard", details_needed
     end
 
-    def ensure_all_ids_found
-      @expected.each do |e|
+    def validate_expected_id_not_missing(expected)
+      expected.each do |e|
         next unless id = e.id
         resource = e.class.api_resource
-        if kennel.strict_imports
+        if @strict_imports
           raise "Unable to find existing #{resource} with id #{id}\nIf the #{resource} was deleted, remove the `id: -> { #{id} }` line."
         else
           @warnings << "#{resource} #{e.tracking_id} specifies id #{id}, but no such #{resource} exists. 'id' will be ignored. Remove the `id: -> { #{id} }` line."
@@ -210,79 +207,18 @@ module Kennel
       map["#{klass.api_resource}:#{a.fetch(:id)}"] || map[a.fetch(:tracking_id)]
     end
 
-    def print_plan(step, list, color)
+    def print_changes(step, list, color)
       return if list.empty?
       list.each do |_, e, a, diff|
         klass = (e ? e.class : a.fetch(:klass))
         Kennel.out.puts Utils.color(color, "#{step} #{klass.api_resource} #{e&.tracking_id || a.fetch(:tracking_id)}")
-        print_diff(diff) if diff # only for update
+        diff&.each { |args| Kennel.out.puts @attribute_differ.format(*args) } # only for update
       end
     end
 
-    def print_diff(diff)
-      diff.each do |type, field, old, new|
-        use_diff = false
-        if type == "+"
-          temp = Utils.pretty_inspect(new)
-          new = Utils.pretty_inspect(old)
-          old = temp
-        elsif old.is_a?(String) && new.is_a?(String) && (old.include?("\n") || new.include?("\n"))
-          use_diff = true
-        else # ~ and -
-          old = Utils.pretty_inspect(old)
-          new = Utils.pretty_inspect(new)
-        end
-
-        message =
-          if use_diff
-            "  #{type}#{field}\n" +
-              diff(old, new).map { |l| "    #{l}" }.join("\n")
-          elsif (old + new).size > 100
-            "  #{type}#{field}\n" \
-              "    #{old} ->\n" \
-              "    #{new}"
-          else
-            "  #{type}#{field} #{old} -> #{new}"
-          end
-
-        Kennel.out.puts truncate_diff(message)
-      end
-    end
-
-    # display diff for multi-line strings
-    # must stay readable when color is off too
-    def diff(old, new)
-      Diff::LCS.sdiff(old.split("\n", -1), new.split("\n", -1)).flat_map do |diff|
-        case diff.action
-        when "-"
-          Utils.color(:red, "- #{diff.old_element}")
-        when "+"
-          Utils.color(:green, "+ #{diff.new_element}")
-        when "!"
-          [
-            Utils.color(:red, "- #{diff.old_element}"),
-            Utils.color(:green, "+ #{diff.new_element}")
-          ]
-        else
-          "  #{diff.old_element}"
-        end
-      end
-    end
-
-    def truncate_diff(message)
-      # min '2' because: -1 makes no sense, 0 does not work with * 2 math, 1 says '1 lines'
-      @max_diff_lines ||= [Integer(ENV.fetch("MAX_DIFF_LINES", "50")), 2].max
-      warning = Utils.color(
-        :magenta,
-        "  (Diff for this item truncated after #{@max_diff_lines} lines. " \
-          "Rerun with MAX_DIFF_LINES=#{@max_diff_lines * 2} to see more)"
-      )
-      Utils.truncate_lines(message, to: @max_diff_lines, warning: warning)
-    end
-
     # We've already validated the desired objects ('generated') in isolation.
     # Now that we have made the plan, we can perform some more validation.
-    def validate_plan
+    def validate_changes
       @update.each do |_, expected, actuals, diffs|
         expected.validate_update!(actuals, diffs)
       end
@@ -290,11 +226,10 @@ module Kennel
 
     # - do not add tracking-id when working with existing ids on a branch,
     #   so resource do not get deleted when running an update on master (for example merge->CI)
-    # - make sure the diff is clean, by kicking out the now noop-update
     # - ideally we'd never add tracking in the first place, but when adding tracking we do not know the diff yet
     def prevent_irreversible_partial_updates
-      return unless @project_filter
-      @update.select! do |_, e, _, diff|
+      return unless @project_filter # full update, so we are not on a branch
+      @update.select! do |_, e, _, diff| # ensure clean diff, by removing noop-update
         next true unless e.id # safe to add tracking when not having id
 
         diff.select! do |field_diff|
@@ -307,7 +242,7 @@ module Kennel
           actual != field_diff[3] # discard diff if now nothing changes
         end
 
-        !diff.empty?
+        diff.any?
       end
     end
 

data/lib/kennel/tasks.rb

@@ -101,8 +101,11 @@ namespace :kennel do
   end
 
   # also generate parts so users see and commit updated generated automatically
+  # (generate must run after plan to enable parallel .download+.generate inside of .plan)
   desc "show planned datadog changes (scope with PROJECT=name)"
-  task plan: :generate do
+  task plan: :environment do
+    Kennel::Tasks.kennel.preload
+    Kennel::Tasks.kennel.generate
     Kennel::Tasks.kennel.plan
   end
 

data/lib/kennel/utils.rb

@@ -5,6 +5,7 @@ module Kennel
 
     class TeeIO < IO
       def initialize(ios)
+        super(0) # called with fake file descriptor 0, so we can call super and get a proper class
         @ios = ios
       end
 
@@ -150,15 +151,9 @@ module Kennel
         end
       end
 
-      # TODO: use awesome-print or similar, but it has too many monkey-patches
-      # https://github.com/amazing-print/amazing_print/issues/36
-      def pretty_inspect(object)
-        string = object.inspect.dup
-        string.gsub!(/:([a-z_]+)=>/, "\\1: ")
-        10.times do
-          string.gsub!(/{(\S.*?\S)}/, "{ \\1 }") || break
-        end
-        string
+      def inline_resource_metadata(resource, klass)
+        resource[:klass] = klass
+        resource[:tracking_id] = klass.parse_tracking_id(resource)
       end
     end
   end

data/lib/kennel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Kennel
-  VERSION = "1.127.0"
+  VERSION = "1.129.0"
 end

data/lib/kennel.rb

@@ -10,6 +10,7 @@ require "kennel/progress"
 require "kennel/filter"
 require "kennel/parts_serializer"
 require "kennel/projects_provider"
+require "kennel/attribute_differ"
 require "kennel/syncer"
 require "kennel/id_map"
 require "kennel/api"
@@ -42,7 +43,6 @@ module Kennel
   UnresolvableIdError = Class.new(StandardError)
   DisallowedUpdateError = Class.new(StandardError)
   GenerationAbortedError = Class.new(StandardError)
-  UpdateResult = Struct.new(:plan, :update, keyword_init: true)
 
   class << self
     attr_accessor :out, :err
@@ -52,24 +52,30 @@ module Kennel
   self.err = $stderr
 
   class Engine
+    attr_accessor :strict_imports
+
     def initialize
       @strict_imports = true
     end
 
-    attr_accessor :strict_imports
+    # start generation and download in parallel to make planning faster
+    def preload
+      Utils.parallel([:generated, :definitions]) { |m| send m, plain: true }
+    end
 
     def generate
       parts = generated
-      parts_serializer.write(parts) if ENV["STORE"] != "false" # quicker when debugging
+      PartsSerializer.new(filter: filter).write(parts) if ENV["STORE"] != "false" # quicker when debugging
       parts
     end
 
     def plan
+      syncer.print_plan
       syncer.plan
     end
 
     def update
-      syncer.plan
+      syncer.print_plan
       syncer.update if syncer.confirm
     end
 
@@ -80,25 +86,25 @@ module Kennel
     end
 
     def syncer
-      @syncer ||= Syncer.new(api, generated, kennel: self, project_filter: filter.project_filter, tracking_id_filter: filter.tracking_id_filter)
+      @syncer ||= begin
+        preload
+        Syncer.new(
+          api, generated, definitions,
+          strict_imports: strict_imports,
+          project_filter: filter.project_filter,
+          tracking_id_filter: filter.tracking_id_filter
+        )
+      end
     end
 
     def api
       @api ||= Api.new
     end
 
-    def projects_provider
-      @projects_provider ||= ProjectsProvider.new
-    end
-
-    def parts_serializer
-      @parts_serializer ||= PartsSerializer.new(filter: filter)
-    end
-
-    def generated
+    def generated(**kwargs)
       @generated ||= begin
-        parts = Progress.progress "Finding parts" do
-          projects = projects_provider.projects
+        parts = Progress.progress "Finding parts", **kwargs do
+          projects = ProjectsProvider.new.projects
           projects = filter.filter_projects projects
 
           parts = Utils.parallel(projects, &:validated_parts).flatten(1)
@@ -123,5 +129,14 @@ module Kennel
         parts
       end
     end
+
+    def definitions(**kwargs)
+      @definitions ||= Progress.progress("Downloading definitions", **kwargs) do
+        Utils.parallel(Models::Record.subclasses) do |klass|
+          results = api.list(klass.api_resource, with_downtimes: false) # lookup monitors without adding unnecessary downtime information
+          results.each { |a| Utils.inline_resource_metadata(a, klass) }
+        end.flatten(1)
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kennel
 version: !ruby/object:Gem::Version
-  version: 1.127.0
+  version: 1.129.0
 platform: ruby
 authors:
 - Michael Grosser
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-12-05 00:00:00.000000000 Z
+date: 2022-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: diff-lcs
@@ -80,7 +80,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.4'
-description: 
+description:
 email: michael@grosser.it
 executables: []
 extensions: []
@@ -89,6 +89,7 @@ files:
 - Readme.md
 - lib/kennel.rb
 - lib/kennel/api.rb
+- lib/kennel/attribute_differ.rb
 - lib/kennel/file_cache.rb
 - lib/kennel/filter.rb
 - lib/kennel/github_reporter.rb
@@ -119,7 +120,7 @@ homepage: https://github.com/grosser/kennel
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -127,15 +128,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.6.0
+      version: 3.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.3.26
+signing_key:
 specification_version: 4
 summary: Keep datadog monitors/dashboards/etc in version control, avoid chaotic management
   via UI