kennel 1.125.0 → 1.126.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2464486b37bc4711843991d93789d4844d9c9aac33b40cf8b088ec4b249f39ab
-  data.tar.gz: 794318abfb10fecca670f6a8d2ad22e6f117e8425d0dc964b76163a658593d16
+  metadata.gz: d8254e03670c1874a6f9b85da5fe453200cdcd9aabe2f82dafc424a6301d8416
+  data.tar.gz: 2d1c0b444762941f79fcba893f5cddc9d0119138c5cf0fc04d6fc871f2caf7a8
 SHA512:
-  metadata.gz: 97c3664e939d5776bf4dcc2a8fc72b647af0a8df0112035cadd6105e8117dd116097e85e60838050f96959df5ea0e17d7e00ffe89420712937a32190bcce70ca
-  data.tar.gz: 4a1cf01f15ac6959c7cea683297fb41f3a74791a99b3a16c5391f76af48333d9a1b4ea9a9fc804750844feb75d5c02fae0caced96fc3b85723d128d1ad45e289
+  metadata.gz: 32c9cbe7957e0f0652180d9b743e4498f3624231e96907f2aaafe5889b9d2f5cfcac98b21dce058c7e06cd6b719ae21b91d01b478eb21028a333b6e0383732a1
+  data.tar.gz: '068566cb8e89e9c9d993ed451cd174a301ce3976c2240aa8c12c1ebf20b602b9cf245b0eb28267dc9263930ad4659163cb75fee67af1d9c1094a649129fb7e17'

data/lib/kennel/models/dashboard.rb

@@ -230,7 +230,9 @@ module Kennel
 
         # Avoid diff from datadog presets sorting.
         presets = data[:template_variable_presets]
-        invalid! "template_variable_presets must be sorted by name" if presets && presets != presets.sort_by { |p| p[:name] }
+        if presets && presets != presets.sort_by { |p| p[:name] }
+          invalid! :template_variable_presets_must_be_sorted, "template_variable_presets must be sorted by name"
+        end
       end
 
       def render_definitions(definitions)

data/lib/kennel/models/monitor.rb

@@ -214,26 +214,26 @@ module Kennel
         super
 
         if data[:name]&.start_with?(" ")
-          invalid! "name cannot start with a space"
+          invalid! :name_must_not_start_with_space, "name cannot start with a space"
         end
 
         type = data.fetch(:type)
 
         # do not allow deprecated type that will be coverted by datadog and then produce a diff
         if type == "metric alert"
-          invalid! "type 'metric alert' is deprecated, please set to a different type (e.g. 'query alert')"
+          invalid! :metric_alert_is_deprecated, "type 'metric alert' is deprecated, please set to a different type (e.g. 'query alert')"
         end
 
         # verify query includes critical value
         if query_value = data.fetch(:query)[/\s*[<>]=?\s*(\d+(\.\d+)?)\s*$/, 1]
           if Float(query_value) != Float(data.dig(:options, :thresholds, :critical))
-            invalid! "critical and value used in query must match"
+            invalid! :critical_does_not_match_query, "critical and value used in query must match"
           end
         end
 
         # verify renotify interval is valid
         unless RENOTIFY_INTERVALS.include? data.dig(:options, :renotify_interval)
-          invalid! "renotify_interval must be one of #{RENOTIFY_INTERVALS.join(", ")}"
+          invalid! :invalid_renotify_interval, "renotify_interval must be one of #{RENOTIFY_INTERVALS.join(", ")}"
         end
 
         if ["query alert", "service check"].include?(type) # TODO: most likely more types need this
@@ -243,15 +243,15 @@ module Kennel
         validate_using_links(data)
 
         if type == "service check" && !data[:query].to_s.include?(".by(")
-          invalid! "query must include a .by() at least .by(\"*\")"
+          invalid! :query_must_include_by, "query must include a .by() at least .by(\"*\")"
         end
 
         unless ALLOWED_PRIORITY_CLASSES.include?(priority.class)
-          invalid! "priority needs to be an Integer"
+          invalid! :invalid_priority, "priority needs to be an Integer"
         end
 
         if data.dig(:options, :timeout_h)&.> 24
-          invalid! "timeout_h must be <= 24"
+          invalid! :invalid_timeout_h, "timeout_h must be <= 24"
         end
       end
 
@@ -286,7 +286,7 @@ module Kennel
         forbidden = used - allowed
         return if forbidden.empty?
 
-        invalid! <<~MSG.rstrip
+        invalid! :invalid_variable_used_in_message, <<~MSG.rstrip
           Used #{forbidden.join(", ")} in the message, but can only be used with #{allowed.join(", ")}.
           Group or filter the query by #{forbidden.map { |f| f.sub(".name", "") }.join(", ")} to use it.
         MSG
@@ -298,7 +298,7 @@ module Kennel
           ids = data[:query].tr("-", "_").scan(/\b\d+\b/)
           ids.reject! { |id| ALLOWED_UNLINKED.include?([tracking_id, id]) }
           if ids.any?
-            invalid! <<~MSG.rstrip
+            invalid! :links_must_be_via_tracking_id, <<~MSG.rstrip
               Used #{ids} in the query, but should only use links in the form of %{<project id>:<monitor id>}
               If that is not possible, add `validate_using_links: ->(*){} # linked monitors are not in kennel
             MSG

data/lib/kennel/models/record.rb

@@ -211,8 +211,8 @@ module Kennel
         end
       end
 
-      def invalid!(message)
-        unfiltered_validation_errors << ValidationMessage.new(message)
+      def invalid!(tag, message)
+        unfiltered_validation_errors << ValidationMessage.new(tag || OptionalValidations::UNIGNORABLE, message)
       end
 
       def raise_with_location(error, message)

data/lib/kennel/models/slo.rb

@@ -85,7 +85,7 @@ module Kennel
         super
 
         if data[:thresholds].any? { |t| t[:warning] && t[:warning].to_f <= t[:critical].to_f }
-          invalid! "Threshold warning must be greater-than critical value"
+          invalid! :warning_must_be_gt_critical, "Threshold warning must be greater-than critical value"
         end
       end
     end

data/lib/kennel/optional_validations.rb

@@ -1,11 +1,13 @@
 # frozen_string_literal: true
 module Kennel
   module OptionalValidations
-    ValidationMessage = Struct.new(:text)
+    ValidationMessage = Struct.new(:tag, :text)
+
+    UNIGNORABLE = :unignorable
 
     def self.included(base)
-      base.settings :validate
-      base.defaults(validate: -> { true })
+      base.settings :ignored_errors
+      base.defaults(ignored_errors: -> { [] })
     end
 
     def self.valid?(parts)
@@ -15,14 +17,26 @@ module Kennel
 
       return true if parts_with_errors.empty?
 
+      example_tag = nil
+
       Kennel.err.puts
       parts_with_errors.sort_by(&:safe_tracking_id).each do |part|
         part.filtered_validation_errors.each do |err|
-          Kennel.err.puts "#{part.safe_tracking_id} #{err.text}"
+          Kennel.err.puts "#{part.safe_tracking_id} [#{err.tag.inspect}] #{err.text.gsub("\n", " ")}"
+          example_tag = err.tag unless err.tag == :unignorable
         end
       end
       Kennel.err.puts
 
+      Kennel.err.puts <<~MESSAGE if example_tag
+        If a particular error cannot be fixed, it can be marked as ignored via `ignored_errors`, e.g.:
+          Kennel::Models::Monitor.new(
+            ...,
+            ignored_errors: [#{example_tag.inspect}]
+          )
+
+      MESSAGE
+
       false
     end
 
@@ -32,6 +46,7 @@ module Kennel
       bad = Kennel::Utils.all_keys(data).grep_v(Symbol).sort.uniq
       return if bad.empty?
       invalid!(
+        :hash_keys_must_be_symbols,
         "Only use Symbols as hash keys to avoid permanent diffs when updating.\n" \
         "Change these keys to be symbols (usually 'foo' => 1 --> 'foo': 1)\n" \
         "#{bad.map(&:inspect).join("\n")}"
@@ -39,39 +54,30 @@ module Kennel
     end
 
     def filter_validation_errors
-      if validate
-        unfiltered_validation_errors
-      elsif unfiltered_validation_errors.empty?
-        msg = "`validate` is set to false, but there are no validation errors to suppress. Remove `validate: false`"
-
-        mode = ENV.fetch("UNNECESSARY_VALIDATE_FALSE") do
-          if ENV.key?("PROJECT") || ENV.key?("TRACKING_ID")
-            "fail"
-          else
-            nil
-          end
-        end
-
-        if mode == "fail"
-          [ValidationMessage.new(msg)]
-        else
-          Kennel.out.puts "#{safe_tracking_id} #{msg}" if mode == "show"
+      if unfiltered_validation_errors.empty?
+        if ignored_errors.empty?
           []
+        else
+          [ValidationMessage.new(UNIGNORABLE, "`ignored_errors` is non-empty, but there are no errors to ignore. Remove `ignored_errors`")]
         end
       else
-        mode = ENV.fetch("SUPPRESSED_ERRORS", "ignore")
-
-        if mode == "fail"
-          unfiltered_validation_errors
-        else
-          if mode == "show"
-            unfiltered_validation_errors.each do |err|
-              Kennel.out.puts "#{safe_tracking_id} `validate: false` suppressing error: #{err.text.gsub("\n", " ")}"
+        to_report =
+          if ENV["NO_IGNORED_ERRORS"]
+            # Turn off all suppressions, to see what errors are actually being suppressed
+            unfiltered_validation_errors
+          else
+            unfiltered_validation_errors.reject do |err|
+              err.tag != UNIGNORABLE && ignored_errors.include?(err.tag)
             end
           end
 
-          []
+        unused_ignores = ignored_errors - unfiltered_validation_errors.map(&:tag)
+
+        unless unused_ignores.empty?
+          to_report << ValidationMessage.new(UNIGNORABLE, "Unused ignores #{unused_ignores.map(&:inspect).sort.uniq.join(" ")}. Remove these from `ignored_errors`")
         end
+
+        to_report
       end
     end
   end

data/lib/kennel/progress.rb

@@ -4,8 +4,8 @@ require "benchmark"
 module Kennel
   class Progress
     # print what we are doing and a spinner until it is done ... then show how long it took
-    def self.progress(name, interval: 0.2, &block)
-      return progress_no_tty(name, &block) unless Kennel.err.tty?
+    def self.progress(name, interval: 0.2, plain: false, &block)
+      return progress_no_tty(name, &block) if plain || !Kennel.err.tty?
 
       Kennel.err.print "#{name} ... "
 

data/lib/kennel/syncer.rb

@@ -10,12 +10,13 @@ module Kennel
     Plan = Struct.new(:changes, keyword_init: true)
     Change = Struct.new(:type, :api_resource, :tracking_id, :id)
 
-    def initialize(api, expected, kennel:, project_filter: nil, tracking_id_filter: nil)
+    def initialize(api, expected, actual, kennel:, project_filter: nil, tracking_id_filter: nil)
       @api = api
       @kennel = kennel
       @project_filter = project_filter
       @tracking_id_filter = tracking_id_filter
       @expected = Set.new expected # need set to speed up deletion
+      @actual = actual
       calculate_diff
       validate_plan
       prevent_irreversible_partial_updates
@@ -53,7 +54,7 @@ module Kennel
         message = "#{e.class.api_resource} #{e.tracking_id}"
         Kennel.out.puts "Creating #{message}"
         reply = @api.create e.class.api_resource, e.as_json
-        cache_metadata reply, e.class
+        Utils.inline_resource_metadata reply, e.class
         id = reply.fetch(:id)
         changes << Change.new(:create, e.class.api_resource, e.tracking_id, id)
         populate_id_map [], [reply] # allow resolving ids we could previously no resolve
@@ -125,17 +126,15 @@ module Kennel
       @delete = []
       @id_map = IdMap.new
 
-      actual = Progress.progress("Downloading definitions") { download_definitions }
-
       Progress.progress "Diffing" do
-        populate_id_map @expected, actual
-        filter_actual! actual
+        populate_id_map @expected, @actual
+        filter_actual! @actual
         resolve_linked_tracking_ids! @expected # resolve dependencies to avoid diff
 
         @expected.each(&:add_tracking_id) # avoid diff with actual
 
         lookup_map = matching_expected_lookup_map
-        items = actual.map do |a|
+        items = @actual.map do |a|
           e = matching_expected(a, lookup_map)
           if e && @expected.delete?(e)
             [e, a]
@@ -168,19 +167,6 @@ module Kennel
       end
     end
 
-    def download_definitions
-      Utils.parallel(Models::Record.subclasses) do |klass|
-        results = @api.list(klass.api_resource, with_downtimes: false) # lookup monitors without adding unnecessary downtime information
-        results = results[results.keys.first] if results.is_a?(Hash) # dashboards are nested in {dashboards: []}
-        results.each { |a| cache_metadata(a, klass) }
-      end.flatten(1)
-    end
-
-    def cache_metadata(a, klass)
-      a[:klass] = klass
-      a[:tracking_id] = a.fetch(:klass).parse_tracking_id(a)
-    end
-
     def ensure_all_ids_found
       @expected.each do |e|
         next unless id = e.id

data/lib/kennel/tasks.rb

@@ -102,8 +102,9 @@ namespace :kennel do
 
   # also generate parts so users see and commit updated generated automatically
   desc "show planned datadog changes (scope with PROJECT=name)"
-  task plan: :generate do
+  task plan: :environment do
     Kennel::Tasks.kennel.plan
+    Kennel::Tasks.kennel.generate
   end
 
   desc "update datadog (scope with PROJECT=name)"

data/lib/kennel/template_variables.rb

@@ -29,8 +29,8 @@ module Kennel
       return if queries.empty?
 
       invalid!(
-        "queries #{queries.join(", ")} must use the template variables #{variables.join(", ")}\n" \
-        "If that is not possible, add `validate: -> { false } # query foo in bar does not have baz tag`"
+        :queries_must_use_template_variables,
+        "queries #{queries.join(", ")} must use the template variables #{variables.join(", ")}"
       )
     end
 

data/lib/kennel/utils.rb

@@ -160,6 +160,11 @@ module Kennel
         end
         string
       end
+
+      def inline_resource_metadata(resource, klass)
+        resource[:klass] = klass
+        resource[:tracking_id] = klass.parse_tracking_id(resource)
+      end
     end
   end
 end

data/lib/kennel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Kennel
-  VERSION = "1.125.0"
+  VERSION = "1.126.0"
 end

data/lib/kennel.rb

@@ -59,7 +59,7 @@ module Kennel
     attr_accessor :strict_imports
 
     def generate
-      parts = generated
+      parts = generated(plain: false)
       parts_serializer.write(parts) if ENV["STORE"] != "false" # quicker when debugging
       parts
     end
@@ -75,12 +75,24 @@ module Kennel
 
     private
 
+    def download_definitions
+      Progress.progress("Downloading definitions", plain: true) do
+        Utils.parallel(Models::Record.subclasses) do |klass|
+          results = api.list(klass.api_resource, with_downtimes: false) # lookup monitors without adding unnecessary downtime information
+          results.each { |a| Utils.inline_resource_metadata(a, klass) }
+        end.flatten(1)
+      end
+    end
+
     def filter
       @filter ||= Filter.new
     end
 
     def syncer
-      @syncer ||= Syncer.new(api, generated, kennel: self, project_filter: filter.project_filter, tracking_id_filter: filter.tracking_id_filter)
+      @syncer ||= begin
+        expected, actual = Utils.parallel([:generated, :download_definitions]) { |m| send m }
+        Syncer.new(api, expected, actual, kennel: self, project_filter: filter.project_filter, tracking_id_filter: filter.tracking_id_filter)
+      end
     end
 
     def api
@@ -95,9 +107,9 @@ module Kennel
       @parts_serializer ||= PartsSerializer.new(filter: filter)
     end
 
-    def generated
+    def generated(plain: true)
       @generated ||= begin
-        parts = Progress.progress "Finding parts" do
+        parts = Progress.progress "Finding parts", plain: plain do
           projects = projects_provider.projects
           projects = filter.filter_projects projects
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kennel
 version: !ruby/object:Gem::Version
-  version: 1.125.0
+  version: 1.126.0
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-11-30 00:00:00.000000000 Z
+date: 2022-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: diff-lcs