kennel 1.122.0 → 1.124.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 43b746ea91f86217a6fc43383a3368ccd654c89936c44003088054380be8c36d
-  data.tar.gz: 5fcd75471c8d0152236d032b023b4769975251047bc276f00219df3b1e60964d
+  metadata.gz: 31edba322106d5b2f1942c57640f19eff6a265ea970b5bb2cc7f4ef02d1889fd
+  data.tar.gz: f47474f8b7d745714fb49231755ed2bf41fae62cf8b5ff3817017c8cd8277713
 SHA512:
-  metadata.gz: e9ce639cdd5748f689eeb352da8ea9c14a2a4243350368574b8b3c1b52534dbf470ca3e969ad96ecd685d8df93d368e7e4d5f5c98da2af7e5cc843ccbc8053ce
-  data.tar.gz: 7039072c9129311145694b53823cab5c11263e972ae8f6c586e0b86063a8ab9d375c2ecd1069bd2812f626ce631f2940fe55ca9a018033b435df63bcbef1d355
+  metadata.gz: 3dc6b1b72e834b7b7663b577c830dfa23da87a197752b04988edd2366ac9089685ff01e19a1e3c8244ac9a41f4217c75ec34bc6198ca83cf545bf6079069d71e
+  data.tar.gz: 1dffcf6adbd06fe80117d9a5f9bc3b55fac90bc21abb4fdd4ebeb5b726ade944c799edbbcca3eabbc5a28beff6a302011ecfcce18f041b4ad2228076e164c2d6

data/lib/kennel/api.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 # encapsulates knowledge around how the api works
-# especially 1-off weirdness that should not lak into other parts of the code
+# especially 1-off weirdness that should not leak into other parts of the code
 module Kennel
   class Api
     CACHE_FILE = "tmp/cache/details"
 
-    def initialize(app_key, api_key)
-      @app_key = app_key
-      @api_key = api_key
+    def initialize(app_key = nil, api_key = nil)
+      @app_key = app_key || ENV.fetch("DATADOG_APP_KEY")
+      @api_key = api_key || ENV.fetch("DATADOG_API_KEY")
       url = Utils.path_to_url("", subdomain: "app")
       @client = Faraday.new(url: url) { |c| c.adapter :net_http_persistent }
     end

data/lib/kennel/file_cache.rb

@@ -15,11 +15,13 @@ module Kennel
     end
 
     def open
-      load_data
-      expire_old_data
-      yield self
-    ensure
-      persist
+      @data = load_data || {}
+      begin
+        expire_old_data
+        yield self
+      ensure
+        persist
+      end
     end
 
     def fetch(key, key_version)
@@ -35,12 +37,9 @@ module Kennel
     private
 
     def load_data
-      @data =
-        begin
-          Marshal.load(File.read(@file)) # rubocop:disable Security/MarshalLoad
-        rescue StandardError
-          {}
-        end
+      Marshal.load(File.read(@file)) # rubocop:disable Security/MarshalLoad
+    rescue Errno::ENOENT, TypeError, ArgumentError
+      nil
     end
 
     def persist
@@ -49,6 +48,7 @@ module Kennel
 
       Tempfile.create "kennel-file-cache", dir do |tmp|
         Marshal.dump @data, tmp
+        tmp.flush
         File.rename tmp.path, @file
       end
     end

data/lib/kennel/importer.rb

@@ -38,6 +38,8 @@ module Kennel
 
       case resource
       when "monitor"
+        raise "Import the synthetic test page and not the monitor" if data[:type] == "synthetics alert"
+
         # flatten monitor options so they are all on the base which is how Monitor builds them
         data.merge!(data.delete(:options))
         data.merge!(data.delete(:thresholds) || {})
@@ -96,7 +98,7 @@ module Kennel
     def link_composite_monitors(data)
       if data[:type] == "composite"
         data[:query].gsub!(/\d+/) do |id|
-          object = Kennel.send(:api).show("monitor", id)
+          object = @api.show("monitor", id)
           tracking_id = Kennel::Models::Monitor.parse_tracking_id(object)
           tracking_id ? "%{#{tracking_id}}" : id
         rescue StandardError # monitor not found

data/lib/kennel/models/dashboard.rb

@@ -3,7 +3,6 @@ module Kennel
   module Models
     class Dashboard < Record
       include TemplateVariables
-      include OptionalValidations
 
       READONLY_ATTRIBUTES = superclass::READONLY_ATTRIBUTES + [
         :author_handle, :author_name, :modified_at, :deleted_at, :url, :is_read_only, :notify_list, :restricted_roles
@@ -87,8 +86,7 @@ module Kennel
         tags: -> do # not inherited by default to make onboarding to using dashboard tags simple
           team = project.team
           team.tag_dashboards ? team.tags : []
-        end,
-        id: -> { nil }
+        end
       )
 
       class << self
@@ -152,29 +150,24 @@ module Kennel
         end
       end
 
-      def as_json
-        return @json if @json
+      def build_json
         all_widgets = render_definitions(definitions) + widgets
         expand_q all_widgets
         tags = tags()
         tags_as_string = (tags.empty? ? "" : " (#{tags.join(" ")})")
 
-        @json = {
+        json = super.merge(
           layout_type: layout_type,
           title: "#{title}#{tags_as_string}#{LOCK}",
           description: description,
           template_variables: render_template_variables,
           template_variable_presets: template_variable_presets,
           widgets: all_widgets
-        }
-
-        @json[:reflow_type] = reflow_type if reflow_type # setting nil breaks create with "ordered"
+        )
 
-        @json[:id] = id if id
+        json[:reflow_type] = reflow_type if reflow_type # setting nil breaks create with "ordered"
 
-        validate_json(@json) if validate
-
-        @json
+        json
       end
 
       def self.url(id)
@@ -210,9 +203,8 @@ module Kennel
       end
 
       def validate_update!(_actuals, diffs)
-        if bad_diff = diffs.find { |diff| diff[1] == "layout_type" }
-          invalid! "Datadog does not allow update of #{bad_diff[1]} (#{bad_diff[2].inspect} -> #{bad_diff[3].inspect})"
-        end
+        _, path, from, to = diffs.find { |diff| diff[1] == "layout_type" }
+        invalid_update!(path, from, to) if path
       end
 
       private

data/lib/kennel/models/monitor.rb

@@ -2,8 +2,6 @@
 module Kennel
   module Models
     class Monitor < Record
-      include OptionalValidations
-
       RENOTIFY_INTERVALS = [0, 10, 20, 30, 40, 50, 60, 90, 120, 180, 240, 300, 360, 720, 1440].freeze # minutes
       OPTIONAL_SERVICE_CHECK_THRESHOLDS = [:ok, :warning].freeze
       READONLY_ATTRIBUTES = superclass::READONLY_ATTRIBUTES + [
@@ -41,8 +39,7 @@ module Kennel
         renotify_interval: -> { project.team.renotify_interval },
         warning: -> { nil },
         ok: -> { nil },
-        id: -> { nil },
-        notify_no_data: -> { true }, # datadog sets this to false by default, but true is the safer
+        notify_no_data: -> { true }, # datadog UI sets this to false by default, but true is safer
         no_data_timeframe: -> { 60 },
         notify_audit: -> { MONITOR_OPTION_DEFAULTS.fetch(:notify_audit) },
         new_host_delay: -> { MONITOR_OPTION_DEFAULTS.fetch(:new_host_delay) },
@@ -56,9 +53,8 @@ module Kennel
         priority: -> { MONITOR_DEFAULTS.fetch(:priority) }
       )
 
-      def as_json
-        return @as_json if @as_json
-        data = {
+      def build_json
+        data = super.merge(
           name: "#{name}#{LOCK}",
           type: type,
           query: query.strip,
@@ -79,9 +75,7 @@ module Kennel
             locked: false, # setting this to true prevents any edit and breaks updates when using replace workflow
             renotify_interval: renotify_interval || 0
           }
-        }
-
-        data[:id] = id if id
+        )
 
         options = data[:options]
         if data.fetch(:type) != "composite"
@@ -105,6 +99,12 @@ module Kennel
           end
         end
 
+        # setting this via the api breaks the UI with
+        # "The no_data_timeframe option is not allowed for log alert monitors"
+        if data.fetch(:type) == "log alert"
+          options.delete :no_data_timeframe
+        end
+
         if windows = threshold_windows
           options[:threshold_windows] = windows
         end
@@ -120,9 +120,7 @@ module Kennel
           options[:renotify_statuses] = statuses
         end
 
-        validate_json(data) if validate
-
-        @as_json = data
+        data
       end
 
       def resolve_linked_tracking_ids!(id_map, **args)
@@ -140,7 +138,7 @@ module Kennel
         # ensure type does not change, but not if it's metric->query which is supported and used by importer.rb
         _, path, from, to = diffs.detect { |_, path, _, _| path == "type" }
         if path && !(from == "metric alert" && to == "query alert")
-          invalid! "Datadog does not allow update of #{path} (#{from.inspect} -> #{to.inspect})"
+          invalid_update!(path, from, to)
         end
       end
 

data/lib/kennel/models/project.rb

@@ -20,7 +20,7 @@ module Kennel
       def validated_parts
         all = parts
         unless all.is_a?(Array) && all.all? { |part| part.is_a?(Record) }
-          invalid! "#parts must return an array of Records"
+          raise "Project #{kennel_id} #parts must return an array of Records"
         end
 
         validate_parts(all)
@@ -29,11 +29,6 @@ module Kennel
 
       private
 
-      # let users know which project/resource failed when something happens during diffing where the backtrace is hidden
-      def invalid!(message)
-        raise ValidationError, "#{kennel_id} #{message}"
-      end
-
       # hook for users to add custom validations via `prepend`
       def validate_parts(parts)
       end

data/lib/kennel/models/record.rb

@@ -2,6 +2,16 @@
 module Kennel
   module Models
     class Record < Base
+      class PrepareError < StandardError
+        def initialize(tracking_id)
+          super("Error while preparing #{tracking_id}")
+        end
+      end
+
+      UnvalidatedRecordError = Class.new(StandardError)
+
+      include OptionalValidations
+
       # Apart from if you just don't like the default for some reason,
       # overriding MARKER_TEXT allows for namespacing within the same
       # Datadog account. If you run one Kennel setup with marker text
@@ -30,6 +40,8 @@ module Kennel
 
       settings :id, :kennel_id
 
+      defaults(id: nil)
+
       class << self
         def parse_any_url(url)
           subclasses.detect do |s|
@@ -72,7 +84,7 @@ module Kennel
         end
       end
 
-      attr_reader :project
+      attr_reader :project, :unfiltered_validation_errors, :filtered_validation_errors
 
       def initialize(project, *args)
         raise ArgumentError, "First argument must be a project, not #{project.class}" unless project.is_a?(Project)
@@ -96,7 +108,7 @@ module Kennel
         @tracking_id ||= begin
           id = "#{project.kennel_id}:#{kennel_id}"
           unless id.match?(ALLOWED_KENNEL_ID_REGEX) # <-> parse_tracking_id
-            raise ValidationError, "#{id} must match #{ALLOWED_KENNEL_ID_REGEX}"
+            raise "Bad kennel/tracking id: #{id.inspect} must match #{ALLOWED_KENNEL_ID_REGEX}"
           end
           id
         end
@@ -108,7 +120,7 @@ module Kennel
       def add_tracking_id
         json = as_json
         if self.class.parse_tracking_id(json)
-          invalid! "remove \"-- #{MARKER_TEXT}\" line it from #{self.class::TRACKING_FIELD} to copy a resource"
+          raise "#{safe_tracking_id} Remove \"-- #{MARKER_TEXT}\" line from #{self.class::TRACKING_FIELD} to copy a resource"
         end
         json[self.class::TRACKING_FIELD] =
           "#{json[self.class::TRACKING_FIELD]}\n" \
@@ -119,9 +131,54 @@ module Kennel
         self.class.remove_tracking_id(as_json)
       end
 
+      def build_json
+        {
+          id: id
+        }.compact
+      end
+
+      def build
+        @unfiltered_validation_errors = []
+        json = nil
+
+        begin
+          json = build_json
+          (id = json.delete(:id)) && json[:id] = id
+          validate_json(json)
+        rescue StandardError
+          if unfiltered_validation_errors.empty?
+            @unfiltered_validation_errors = nil
+            raise PrepareError, safe_tracking_id
+          end
+        end
+
+        @filtered_validation_errors = filter_validation_errors
+        @as_json = json # Only valid if filtered_validation_errors.empty?
+      end
+
+      def as_json
+        # A courtesy to those tests that still expect as_json to perform validation and raise on error
+        build if @unfiltered_validation_errors.nil?
+        raise UnvalidatedRecordError, "#{safe_tracking_id} as_json called on invalid part" unless filtered_validation_errors.empty?
+
+        @as_json
+      end
+
+      # Can raise DisallowedUpdateError
       def validate_update!(*)
       end
 
+      def invalid_update!(field, old_value, new_value)
+        raise DisallowedUpdateError, "#{safe_tracking_id} Datadog does not allow update of #{field} (#{old_value.inspect} -> #{new_value.inspect})"
+      end
+
+      # For use during error handling
+      def safe_tracking_id
+        tracking_id
+      rescue StandardError
+        "<unknown; #tracking_id crashed>"
+      end
+
       private
 
       def resolve(value, type, id_map, force:)
@@ -133,26 +190,29 @@ module Kennel
         id.is_a?(String) && id.include?(":")
       end
 
-      def resolve_link(tracking_id, type, id_map, force:)
-        if id_map.new?(type.to_s, tracking_id)
+      def resolve_link(sought_tracking_id, sought_type, id_map, force:)
+        if id_map.new?(sought_type.to_s, sought_tracking_id)
           if force
-            invalid!(
-              "#{type} #{tracking_id} was referenced but is also created by the current run.\n" \
-              "It could not be created because of a circular dependency, try creating only some of the resources"
-            )
+            raise UnresolvableIdError, <<~MESSAGE
+              #{tracking_id} #{sought_type} #{sought_tracking_id} was referenced but is also created by the current run.
+              It could not be created because of a circular dependency. Try creating only some of the resources.
+            MESSAGE
           else
             nil # will be re-resolved after the linked object was created
           end
-        elsif id = id_map.get(type.to_s, tracking_id)
+        elsif id = id_map.get(sought_type.to_s, sought_tracking_id)
           id
         else
-          invalid! "Unable to find #{type} #{tracking_id} (does not exist and is not being created by the current run)"
+          raise UnresolvableIdError, <<~MESSAGE
+            #{tracking_id} Unable to find #{sought_type} #{sought_tracking_id}
+            This is either because it doesn't exist, and isn't being created by the current run;
+            or it does exist, but is being deleted.
+          MESSAGE
         end
       end
 
-      # let users know which project/resource failed when something happens during diffing where the backtrace is hidden
       def invalid!(message)
-        raise ValidationError, "#{tracking_id} #{message}"
+        unfiltered_validation_errors << ValidationMessage.new(message)
       end
 
       def raise_with_location(error, message)

data/lib/kennel/models/slo.rb

@@ -15,7 +15,6 @@ module Kennel
       settings :type, :description, :thresholds, :query, :tags, :monitor_ids, :monitor_tags, :name, :groups
 
       defaults(
-        id: -> { nil },
         tags: -> { @project.tags },
         query: -> { DEFAULTS.fetch(:query) },
         description: -> { DEFAULTS.fetch(:description) },
@@ -24,35 +23,25 @@ module Kennel
         groups: -> { DEFAULTS.fetch(:groups) }
       )
 
-      def initialize(*)
-        super
-        if thresholds.any? { |t| t[:warning] && t[:warning].to_f <= t[:critical].to_f }
-          raise ValidationError, "Threshold warning must be greater-than critical value"
-        end
-      end
-
-      def as_json
-        return @as_json if @as_json
-        data = {
+      def build_json
+        data = super.merge(
           name: "#{name}#{LOCK}",
           description: description,
           thresholds: thresholds,
           monitor_ids: monitor_ids,
           tags: tags.uniq,
           type: type
-        }
+        )
 
         if v = query
           data[:query] = v
         end
-        if v = id
-          data[:id] = v
-        end
+
         if v = groups
           data[:groups] = v
         end
 
-        @as_json = data
+        data
       end
 
       def self.api_resource
@@ -89,6 +78,16 @@ module Kennel
 
         ignore_default(expected, actual, DEFAULTS)
       end
+
+      private
+
+      def validate_json(data)
+        super
+
+        if data[:thresholds].any? { |t| t[:warning] && t[:warning].to_f <= t[:critical].to_f }
+          invalid! "Threshold warning must be greater-than critical value"
+        end
+      end
     end
   end
 end

data/lib/kennel/models/synthetic_test.rb

@@ -11,15 +11,14 @@ module Kennel
       settings :tags, :config, :message, :subtype, :type, :name, :locations, :options
 
       defaults(
-        id: -> { nil },
         tags: -> { @project.tags },
         message: -> { "\n\n#{project.mention}" }
       )
 
-      def as_json
-        return @as_json if @as_json
+      def build_json
         locations = locations()
-        data = {
+
+        super.merge(
           message: message,
           tags: tags,
           config: config,
@@ -28,13 +27,7 @@ module Kennel
           options: options,
           name: "#{name}#{LOCK}",
           locations: locations == :all ? LOCATIONS : locations
-        }
-
-        if v = id
-          data[:id] = v
-        end
-
-        @as_json = data
+        )
       end
 
       def self.api_resource

data/lib/kennel/optional_validations.rb

@@ -1,15 +1,35 @@
 # frozen_string_literal: true
 module Kennel
   module OptionalValidations
+    ValidationMessage = Struct.new(:text)
+
     def self.included(base)
       base.settings :validate
       base.defaults(validate: -> { true })
     end
 
+    def self.valid?(parts)
+      parts_with_errors = parts.reject do |part|
+        part.filtered_validation_errors.empty?
+      end
+
+      return true if parts_with_errors.empty?
+
+      Kennel.err.puts
+      parts_with_errors.sort_by(&:safe_tracking_id).each do |part|
+        part.filtered_validation_errors.each do |err|
+          Kennel.err.puts "#{part.safe_tracking_id} #{err.text}"
+        end
+      end
+      Kennel.err.puts
+
+      false
+    end
+
     private
 
     def validate_json(data)
-      bad = Kennel::Utils.all_keys(data).grep_v(Symbol)
+      bad = Kennel::Utils.all_keys(data).grep_v(Symbol).sort.uniq
       return if bad.empty?
       invalid!(
         "Only use Symbols as hash keys to avoid permanent diffs when updating.\n" \
@@ -17,5 +37,42 @@ module Kennel
         "#{bad.map(&:inspect).join("\n")}"
       )
     end
+
+    def filter_validation_errors
+      if validate
+        unfiltered_validation_errors
+      elsif unfiltered_validation_errors.empty?
+        msg = "`validate` is set to false, but there are no validation errors to suppress. Remove `validate: false`"
+
+        mode = ENV.fetch("UNNECESSARY_VALIDATE_FALSE") do
+          if ENV.key?("PROJECT") || ENV.key?("TRACKING_ID")
+            "fail"
+          else
+            nil
+          end
+        end
+
+        if mode == "fail"
+          [ValidationMessage.new(msg)]
+        else
+          Kennel.out.puts "#{safe_tracking_id} #{msg}" if mode == "show"
+          []
+        end
+      else
+        mode = ENV.fetch("SUPPRESSED_ERRORS", "ignore")
+
+        if mode == "fail"
+          unfiltered_validation_errors
+        else
+          if mode == "show"
+            unfiltered_validation_errors.each do |err|
+              Kennel.out.puts "#{safe_tracking_id} `validate: false` suppressing error: #{err.text.gsub("\n", " ")}"
+            end
+          end
+
+          []
+        end
+      end
+    end
   end
 end

data/lib/kennel/syncer.rb

@@ -6,12 +6,11 @@ module Kennel
   class Syncer
     DELETE_ORDER = ["dashboard", "slo", "monitor", "synthetics/tests"].freeze # dashboards references monitors + slos, slos reference monitors
     LINE_UP = "\e[1A\033[K" # go up and clear
+    Plan = Struct.new(:changes, keyword_init: true)
 
-    Plan = Struct.new(:noop?, :no_change, :create, :update, :delete, keyword_init: true)
-    Update = Struct.new(:update_log, keyword_init: true)
-
-    def initialize(api, expected, project_filter: nil, tracking_id_filter: nil)
+    def initialize(api, expected, kennel:, project_filter: nil, tracking_id_filter: nil)
       @api = api
+      @kennel = kennel
       @project_filter = project_filter
       @tracking_id_filter = tracking_id_filter
       @expected = Set.new expected # need set to speed up deletion
@@ -32,11 +31,10 @@ module Kennel
       end
 
       Plan.new(
-        noop?: noop?,
-        no_change: @no_change,
-        create: @create,
-        update: @update,
-        delete: @delete
+        changes:
+          @create.map { |_id, e| [:create, e.class.api_resource, e.tracking_id, nil] } +
+          @update.map { |_id, e| [:create, e.class.api_resource, e.tracking_id, nil] } +
+          @delete.map { |_id, _e, a| [:delete, a.fetch(:klass).api_resource, a.fetch(:tracking_id), a.fetch(:id)] }
       )
     end
 
@@ -47,7 +45,7 @@ module Kennel
     end
 
     def update
-      update_log = []
+      changes = []
 
       each_resolved @create do |_, e|
         message = "#{e.class.api_resource} #{e.tracking_id}"
@@ -55,7 +53,7 @@ module Kennel
         reply = @api.create e.class.api_resource, e.as_json
         cache_metadata reply, e.class
         id = reply.fetch(:id)
-        update_log << [:create, e.class.api_resource, id]
+        changes << [:create, e.class.api_resource, e.tracking_id, id]
         populate_id_map [], [reply] # allow resolving ids we could previously no resolve
         Kennel.out.puts "#{LINE_UP}Created #{message} #{e.class.url(id)}"
       end
@@ -64,7 +62,7 @@ module Kennel
         message = "#{e.class.api_resource} #{e.tracking_id} #{e.class.url(id)}"
         Kennel.out.puts "Updating #{message}"
         @api.update e.class.api_resource, id, e.as_json
-        update_log << [:update, e.class.api_resource, id]
+        changes << [:update, e.class.api_resource, e.tracking_id, id]
         Kennel.out.puts "#{LINE_UP}Updated #{message}"
       end
 
@@ -73,15 +71,17 @@ module Kennel
         message = "#{klass.api_resource} #{a.fetch(:tracking_id)} #{id}"
         Kennel.out.puts "Deleting #{message}"
         @api.delete klass.api_resource, id
-        update_log << [:delete, klass.api_resource, id]
+        changes << [:delete, klass.api_resource, a.fetch(:tracking_id), id]
         Kennel.out.puts "#{LINE_UP}Deleted #{message}"
       end
 
-      Update.new(update_log: update_log)
+      Plan.new(changes: changes)
     end
 
     private
 
+    attr_reader :kennel
+
     # loop over items until everything is resolved or crash when we get stuck
     # this solves cases like composite monitors depending on each other or monitor->monitor slo->slo monitor chains
     def each_resolved(list)
@@ -104,11 +104,11 @@ module Kennel
     def resolved?(e)
       assert_resolved e
       true
-    rescue ValidationError
+    rescue UnresolvableIdError
       false
     end
 
-    # raises ValidationError when not resolved
+    # raises UnresolvableIdError when not resolved
     def assert_resolved(e)
       resolve_linked_tracking_ids! [e], force: true
     end
@@ -121,7 +121,6 @@ module Kennel
       @warnings = []
       @update = []
       @delete = []
-      @no_change = []
       @id_map = IdMap.new
 
       actual = Progress.progress("Downloading definitions") { download_definitions }
@@ -154,8 +153,6 @@ module Kennel
             diff = e.diff(a) # slow ...
             if diff.any?
               @update << [id, e, a, diff]
-            else
-              @no_change << [id, e, a]
             end
           elsif a.fetch(:tracking_id) # was previously managed
             @delete << [id, nil, a]
@@ -186,7 +183,7 @@ module Kennel
       @expected.each do |e|
         next unless id = e.id
         resource = e.class.api_resource
-        if Kennel.strict_imports
+        if kennel.strict_imports
           raise "Unable to find existing #{resource} with id #{id}\nIf the #{resource} was deleted, remove the `id: -> { #{id} }` line."
         else
           @warnings << "#{resource} #{e.tracking_id} specifies id #{id}, but no such #{resource} exists. 'id' will be ignored. Remove the `id: -> { #{id} }` line."
@@ -234,16 +231,19 @@ module Kennel
           new = Utils.pretty_inspect(new)
         end
 
-        if use_diff
-          Kennel.out.puts "  #{type}#{field}"
-          Kennel.out.puts(diff(old, new).map { |l| "    #{l}" })
-        elsif (old + new).size > 100
-          Kennel.out.puts "  #{type}#{field}"
-          Kennel.out.puts "    #{old} ->"
-          Kennel.out.puts "    #{new}"
-        else
-          Kennel.out.puts "  #{type}#{field} #{old} -> #{new}"
-        end
+        message =
+          if use_diff
+            "  #{type}#{field}\n" +
+              diff(old, new).map { |l| "    #{l}" }.join("\n")
+          elsif (old + new).size > 100
+            "  #{type}#{field}\n" \
+              "    #{old} ->\n" \
+              "    #{new}"
+          else
+            "  #{type}#{field} #{old} -> #{new}"
+          end
+
+        Kennel.out.puts truncate_diff(message)
       end
     end
 
@@ -267,6 +267,17 @@ module Kennel
       end
     end
 
+    def truncate_diff(message)
+      # min '2' because: -1 makes no sense, 0 does not work with * 2 math, 1 says '1 lines'
+      @max_diff_lines ||= [Integer(ENV.fetch("MAX_DIFF_LINES", "50")), 2].max
+      warning = Utils.color(
+        :magenta,
+        "  (Diff for this item truncated after #{@max_diff_lines} lines. " \
+          "Rerun with MAX_DIFF_LINES=#{@max_diff_lines * 2} to see more)"
+      )
+      Utils.truncate_lines(message, to: @max_diff_lines, warning: warning)
+    end
+
     # We've already validated the desired objects ('generated') in isolation.
     # Now that we have made the plan, we can perform some more validation.
     def validate_plan

data/lib/kennel/tasks.rb

@@ -8,6 +8,10 @@ require "json"
 module Kennel
   module Tasks
     class << self
+      def kennel
+        @kennel ||= Kennel::Engine.new
+      end
+
       def abort(message = nil)
         Kennel.err.puts message if message
         raise SystemExit.new(1), message
@@ -35,10 +39,10 @@ module Kennel
         load_environment
 
         if on_default_branch? && git_push?
-          Kennel.strict_imports = false
-          Kennel.update
+          Kennel::Tasks.kennel.strict_imports = false
+          Kennel::Tasks.kennel.update
         else
-          Kennel.plan # show plan in CI logs
+          Kennel::Tasks.kennel.plan # show plan in CI logs
         end
       end
 
@@ -66,7 +70,7 @@ namespace :kennel do
   # https://help.datadoghq.com/hc/en-us/requests/254114 for automatic validation
   desc "Verify that all used monitor  mentions are valid"
   task validate_mentions: :environment do
-    known = Kennel.send(:api)
+    known = Kennel::Api.new
       .send(:request, :get, "/monitor/notifications")
       .fetch(:handles)
       .values
@@ -93,18 +97,18 @@ namespace :kennel do
 
   desc "generate local definitions"
   task generate: :environment do
-    Kennel.generate
+    Kennel::Tasks.kennel.generate
   end
 
   # also generate parts so users see and commit updated generated automatically
   desc "show planned datadog changes (scope with PROJECT=name)"
   task plan: :generate do
-    Kennel.plan
+    Kennel::Tasks.kennel.plan
   end
 
   desc "update datadog (scope with PROJECT=name)"
   task update_datadog: :environment do
-    Kennel.update
+    Kennel::Tasks.kennel.update
   end
 
   desc "update on push to the default branch, otherwise show plan"
@@ -115,13 +119,13 @@ namespace :kennel do
   desc "show unmuted alerts filtered by TAG, for example TAG=team:foo"
   task alerts: :environment do
     tag = ENV["TAG"] || Kennel::Tasks.abort("Call with TAG=foo:bar")
-    Kennel::UnmutedAlerts.print(Kennel.send(:api), tag)
+    Kennel::UnmutedAlerts.print(Kennel::Api.new, tag)
   end
 
   desc "show monitors with no data by TAG, for example TAG=team:foo [THRESHOLD_DAYS=7] [FORMAT=json]"
   task nodata: :environment do
     tag = ENV["TAG"] || Kennel::Tasks.abort("Call with TAG=foo:bar")
-    monitors = Kennel.send(:api).list("monitor", monitor_tags: tag, group_states: "no data")
+    monitors = Kennel::Api.new.list("monitor", monitor_tags: tag, group_states: "no data")
     monitors.select! { |m| m[:overall_state] == "No Data" }
     monitors.reject! { |m| m[:tags].include? "nodata:ignore" }
     if monitors.any?
@@ -179,7 +183,7 @@ namespace :kennel do
       Kennel::Tasks.abort("Call with URL= or call with RESOURCE=#{possible_resources.join(" or ")} and ID=")
     end
 
-    Kennel.out.puts Kennel::Importer.new(Kennel.send(:api)).import(resource, id)
+    Kennel.out.puts Kennel::Importer.new(Kennel::Api.new).import(resource, id)
   end
 
   desc "Dump ALL of datadog config as raw json ... useful for grep/search [TYPE=slo|monitor|dashboard]"
@@ -190,7 +194,7 @@ namespace :kennel do
       else
         Kennel::Models::Record.api_resource_map.keys
       end
-    api = Kennel.send(:api)
+    api = Kennel::Api.new
     list = nil
     first = true
 
@@ -240,7 +244,7 @@ namespace :kennel do
     klass =
       Kennel::Models::Record.subclasses.detect { |s| s.api_resource == resource } ||
       raise("resource #{resource} not know")
-    object = Kennel.send(:api).show(resource, id)
+    object = Kennel::Api.new.show(resource, id)
     Kennel.out.puts klass.parse_tracking_id(object)
   end
 

data/lib/kennel/utils.rb

@@ -57,6 +57,12 @@ module Kennel
         "\e[#{COLORS.fetch(color)}m#{text}\e[0m"
       end
 
+      def truncate_lines(text, to:, warning:)
+        lines = text.split(/\n/, to + 1)
+        lines[-1] = warning if lines.size > to
+        lines.join("\n")
+      end
+
       def capture_stdout
         old = Kennel.out
         Kennel.out = StringIO.new

data/lib/kennel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Kennel
-  VERSION = "1.122.0"
+  VERSION = "1.124.0"
 end

data/lib/kennel.rb

@@ -5,7 +5,6 @@ require "zeitwerk"
 require "English"
 
 require "kennel/version"
-require "kennel/compatibility"
 require "kennel/utils"
 require "kennel/progress"
 require "kennel/filter"
@@ -40,21 +39,24 @@ module Teams
 end
 
 module Kennel
-  class ValidationError < RuntimeError
+  UnresolvableIdError = Class.new(StandardError)
+  DisallowedUpdateError = Class.new(StandardError)
+  GenerationAbortedError = Class.new(StandardError)
+  UpdateResult = Struct.new(:plan, :update, keyword_init: true)
+
+  class << self
+    attr_accessor :out, :err
   end
 
-  include Kennel::Compatibility
-
-  UpdateResult = Struct.new(:plan, :update, keyword_init: true)
+  self.out = $stdout
+  self.err = $stderr
 
   class Engine
     def initialize
-      @out = $stdout
-      @err = $stderr
       @strict_imports = true
     end
 
-    attr_accessor :out, :err, :strict_imports
+    attr_accessor :strict_imports
 
     def generate
       parts = generated
@@ -67,12 +69,8 @@ module Kennel
     end
 
     def update
-      the_plan = syncer.plan
-      the_update = syncer.update if syncer.confirm
-      UpdateResult.new(
-        plan: the_plan,
-        update: the_update
-      )
+      syncer.plan
+      syncer.update if syncer.confirm
     end
 
     private
@@ -82,11 +80,11 @@ module Kennel
     end
 
     def syncer
-      @syncer ||= Syncer.new(api, generated, project_filter: filter.project_filter, tracking_id_filter: filter.tracking_id_filter)
+      @syncer ||= Syncer.new(api, generated, kennel: self, project_filter: filter.project_filter, tracking_id_filter: filter.tracking_id_filter)
     end
 
     def api
-      @api ||= Api.new(ENV.fetch("DATADOG_APP_KEY"), ENV.fetch("DATADOG_API_KEY"))
+      @api ||= Api.new
     end
 
     def projects_provider
@@ -99,26 +97,30 @@ module Kennel
 
     def generated
       @generated ||= begin
-        Progress.progress "Generating" do
+        parts = Progress.progress "Finding parts" do
           projects = projects_provider.projects
           projects = filter.filter_projects projects
 
           parts = Utils.parallel(projects, &:validated_parts).flatten(1)
-          parts = filter.filter_parts parts
+          filter.filter_parts parts
+        end
 
-          parts.group_by(&:tracking_id).each do |tracking_id, same|
-            next if same.size == 1
-            raise <<~ERROR
-              #{tracking_id} is defined #{same.size} times
-              use a different `kennel_id` when defining multiple projects/monitors/dashboards to avoid this conflict
-            ERROR
-          end
+        parts.group_by(&:tracking_id).each do |tracking_id, same|
+          next if same.size == 1
+          raise <<~ERROR
+            #{tracking_id} is defined #{same.size} times
+            use a different `kennel_id` when defining multiple projects/monitors/dashboards to avoid this conflict
+          ERROR
+        end
 
+        Progress.progress "Building json" do
           # trigger json caching here so it counts into generating
-          Utils.parallel(parts, &:as_json)
-
-          parts
+          Utils.parallel(parts, &:build)
         end
+
+        OptionalValidations.valid?(parts) or raise GenerationAbortedError
+
+        parts
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kennel
 version: !ruby/object:Gem::Version
-  version: 1.122.0
+  version: 1.124.0
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-10-13 00:00:00.000000000 Z
+date: 2022-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: diff-lcs
@@ -89,7 +89,6 @@ files:
 - Readme.md
 - lib/kennel.rb
 - lib/kennel/api.rb
-- lib/kennel/compatibility.rb
 - lib/kennel/file_cache.rb
 - lib/kennel/filter.rb
 - lib/kennel/github_reporter.rb

data/lib/kennel/compatibility.rb

@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-module Kennel
-  module Compatibility
-    def self.included(into)
-      class << into
-        %I[out out= err err= strict_imports strict_imports= generate plan update].each do |sym|
-          define_method(sym) { |*args| instance.public_send(sym, *args) }
-        end
-
-        def build_default
-          Kennel::Engine.new
-        end
-
-        def instance
-          @instance ||= build_default
-        end
-
-        private
-
-        def api
-          instance.send(:api)
-        end
-      end
-    end
-  end
-end