kennel 1.86.1 → 1.89.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e2531f5f8a3ef6c978e2aeeec1db0a16f56aebe690ef6eae8baefff34c3c53fe
-  data.tar.gz: d650a926e7d730f34e3dc90833f04f693161ad6aa112fd8ffdbca88f37d37497
+  metadata.gz: ba1d58935b541b89cb78318964470bb209933a491d9b81e23785de0fb6ab2654
+  data.tar.gz: aed4b13dc2c35ced80073e592fd0d2ba2afda8dfb763d2c6baa4b1f30851cf53
 SHA512:
-  metadata.gz: 12c01b65de8be29754e78b8caa6d8d8cc3f8c8089928008917a31121d3641161eb8b519aedfd7f5a643ff9f4f1123579e8bc6e07382e26613b5dd9937f6f6c16
-  data.tar.gz: 421e6d81e4ea91f0eb152ba5230026334221e452f5dea8a4e040d5b8285db8f0913443357b128f5c00e917557ace751f264a55568e50ee86a8e304fb1f9f188e
+  metadata.gz: d3595608df6e40ec4241bf2b758f1ed1b3fde0e63dff7f5a4fa393a358cf9cb9fa33bb0c5f72899b7df54817cb9482366d9c387e3c21a33249e35e3bcc4fc8ea
+  data.tar.gz: 745ec8c762acc885a9d0f2ae23fc9f43ca05f56b8c580501075e2b9237e7199a09499053cebb45e5bfa477aa0a187cabf5899e05f7bf6c361f952347ab6cac34

data/Readme.md

@@ -52,6 +52,7 @@ end
 ```
 
 <!-- NOT IN template/Readme.md  -->
+
 ## Installation
 
  - create a new private `kennel` repo for your organization (do not fork this repo)
@@ -114,6 +115,7 @@ end
  - use [datadog monitor UI](https://app.datadoghq.com/monitors/manage) to find a monitor
  - get the `id` from the url
  - run `URL='https://app.datadoghq.com/monitors/123' bundle exec rake kennel:import` and copy the output
+ - import task also works with SLO alerts, e.g. `URL='https://app.datadoghq.com/slo/edit/123abc456def123/alerts/789' bundle exec rake kennel:import`
  - find or create a project in `projects/`
  - add the monitor to `parts: [` list, for example:
   ```Ruby
@@ -292,9 +294,12 @@ https://foo.datadog.com/monitor/123
 
 <!-- NOT IN template/Readme.md -->
 
-
 ## Development
 
+### Benchmarking
+
+Setting `FORCE_GET_CACHE=true` will cache all get requests, which makes benchmarking improvements more reliable.
+
 ### Integration testing
 
 ```Bash

data/lib/kennel/api.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
+# encapsulates knowledge around how the api works
+# especially 1-off weirdness that should not lak into other parts of the code
 module Kennel
-  # encapsulates knowledge around how the api works
   class Api
     CACHE_FILE = "tmp/cache/details"
 
@@ -11,34 +12,24 @@ module Kennel
     end
 
     def show(api_resource, id, params = {})
-      reply = request :get, "/api/v1/#{api_resource}/#{id}", params: params
-      api_resource == "slo" ? reply[:data] : reply
+      response = request :get, "/api/v1/#{api_resource}/#{id}", params: params
+      response = response.fetch(:data) if api_resource == "slo"
+      response
     end
 
     def list(api_resource, params = {})
-      if api_resource == "slo"
-        raise ArgumentError if params[:limit] || params[:offset]
-        limit = 1000
-        offset = 0
-        all = []
-
-        loop do
-          result = request :get, "/api/v1/#{api_resource}", params: params.merge(limit: limit, offset: offset)
-          data = result.fetch(:data)
-          all.concat data
-          break all if data.size < limit
-          offset += limit
-        end
-      else
-        result = request :get, "/api/v1/#{api_resource}", params: params
-        result = result.fetch(:dashboards) if api_resource == "dashboard"
-        result
+      with_pagination api_resource == "slo", params do |paginated_params|
+        response = request :get, "/api/v1/#{api_resource}", params: paginated_params
+        response = response.fetch(:dashboards) if api_resource == "dashboard"
+        response = response.fetch(:data) if api_resource == "slo"
+        response
       end
     end
 
     def create(api_resource, attributes)
-      reply = request :post, "/api/v1/#{api_resource}", body: attributes
-      api_resource == "slo" ? reply[:data].first : reply
+      response = request :post, "/api/v1/#{api_resource}", body: attributes
+      response = response.fetch(:data).first if api_resource == "slo"
+      response
     end
 
     def update(api_resource, id, attributes)
@@ -53,7 +44,6 @@ module Kennel
     end
 
     def fill_details!(api_resource, list)
-      return unless api_resource == "dashboard"
       details_cache do |cache|
         Utils.parallel(list) { |a| fill_detail!(api_resource, a, cache) }
       end
@@ -61,6 +51,21 @@ module Kennel
 
     private
 
+    def with_pagination(enabled, params)
+      return yield params unless enabled
+      raise ArgumentError if params[:limit] || params[:offset]
+      limit = 1000
+      offset = 0
+      all = []
+
+      loop do
+        response = yield params.merge(limit: limit, offset: offset)
+        all.concat response
+        return all if response.size < limit
+        offset += limit
+      end
+    end
+
     # Make diff work even though we cannot mass-fetch definitions
     def fill_detail!(api_resource, a, cache)
       args = [api_resource, a.fetch(:id)]
@@ -74,34 +79,52 @@ module Kennel
     end
 
     def request(method, path, body: nil, params: {}, ignore_404: false)
-      params = params.merge(application_key: @app_key, api_key: @api_key)
-      query = Faraday::FlatParamsEncoder.encode(params)
-      response = nil
-      tries = 2
-
-      tries.times do |i|
-        response = Utils.retry Faraday::ConnectionFailed, Faraday::TimeoutError, times: 2 do
-          @client.send(method, "#{path}?#{query}") do |request|
-            request.body = JSON.generate(body) if body
-            request.headers["Content-type"] = "application/json"
+      path = "#{path}?#{Faraday::FlatParamsEncoder.encode(params)}" if params.any?
+      with_cache ENV["FORCE_GET_CACHE"] && method == :get, path do
+        response = nil
+        tries = 2
+
+        tries.times do |i|
+          response = Utils.retry Faraday::ConnectionFailed, Faraday::TimeoutError, times: 2 do
+            @client.send(method, path) do |request|
+              request.body = JSON.generate(body) if body
+              request.headers["Content-type"] = "application/json"
+              request.headers["DD-API-KEY"] = @api_key
+              request.headers["DD-APPLICATION-KEY"] = @app_key
+            end
           end
+
+          break if i == tries - 1 || method != :get || response.status < 500
+          Kennel.err.puts "Retrying on server error #{response.status} for #{path}"
         end
 
-        break if i == tries - 1 || method != :get || response.status < 500
-        Kennel.err.puts "Retrying on server error #{response.status} for #{path}"
-      end
+        if !response.success? && (response.status != 404 || !ignore_404)
+          message = +"Error #{response.status} during #{method.upcase} #{path}\n"
+          message << "request:\n#{JSON.pretty_generate(body)}\nresponse:\n" if body
+          message << response.body
+          raise message
+        end
 
-      if !response.success? && (response.status != 404 || !ignore_404)
-        message = +"Error #{response.status} during #{method.upcase} #{path}\n"
-        message << "request:\n#{JSON.pretty_generate(body)}\nresponse:\n" if body
-        message << response.body
-        raise message
+        if response.body.empty?
+          {}
+        else
+          JSON.parse(response.body, symbolize_names: true)
+        end
       end
+    end
 
-      if response.body.empty?
-        {}
+    # allow caching all requests to speedup/benchmark logic that includes repeated requests
+    def with_cache(enabled, key)
+      return yield unless enabled
+      dir = "tmp/cache"
+      FileUtils.mkdir_p(dir) unless File.directory?(dir)
+      file = "#{dir}/#{key.delete("/?=")}" # TODO: encode nicely
+      if File.exist?(file)
+        Marshal.load(File.read(file)) # rubocop:disable Security/MarshalLoad
       else
-        JSON.parse(response.body, symbolize_names: true)
+        result = yield
+        File.write(file, Marshal.dump(result))
+        result
       end
     end
   end

data/lib/kennel/file_cache.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 # cache that reads everything from a single file
-# to avoid doing multiple disk reads while iterating all definitions
-# it also replaces updated keys and has an overall expiry to not keep deleted things forever
+# - avoids doing multiple disk reads while iterating all definitions
+# - has a global expiry to not keep deleted resources forever
 module Kennel
   class FileCache
     def initialize(file, cache_version)
@@ -22,10 +22,11 @@ module Kennel
 
     def fetch(key, key_version)
       old_value, old_version = @data[key]
-      return old_value if old_version == [key_version, @cache_version]
+      expected_version = [key_version, @cache_version]
+      return old_value if old_version == expected_version
 
       new_value = yield
-      @data[key] = [new_value, [key_version, @cache_version], @expires]
+      @data[key] = [new_value, expected_version, @expires]
       new_value
     end
 
@@ -46,8 +47,11 @@ module Kennel
       File.write(@file, Marshal.dump(@data))
     end
 
+    # keep the cache small to make loading it fast (5MB ~= 100ms)
+    # - delete expired keys
+    # - delete what would be deleted anyway when updating
     def expire_old_data
-      @data.reject! { |_, (_, _, ex)| ex < @now }
+      @data.reject! { |_, (_, (_, cv), expires)| expires < @now || cv != @cache_version }
     end
   end
 end

data/lib/kennel/importer.rb

@@ -3,7 +3,7 @@
 module Kennel
   class Importer
     TITLES = [:name, :title].freeze
-    SORT_ORDER = [*TITLES, :id, :kennel_id, :type, :tags, :query, *Syncer::TRACKING_FIELDS, :template_variables].freeze
+    SORT_ORDER = [*TITLES, :id, :kennel_id, :type, :tags, :query, *Models::Record.subclasses.map { |k| k::TRACKING_FIELDS }, :template_variables].freeze
 
     def initialize(api)
       @api = api
@@ -31,11 +31,10 @@ module Kennel
       title.tr!(Kennel::Models::Record::LOCK, "") # avoid double lock icon
 
       # calculate or reuse kennel_id
-      # TODO: this is copy-pasted from syncer, need to find a nice way to reuse it
-      tracking_field = Syncer::TRACKING_FIELDS.detect { |f| data[f] }
       data[:kennel_id] =
-        if tracking_field && data[tracking_field].sub!(/\n?-- Managed by kennel (\S+:\S+).*/, "")
-          $1.split(":").last
+        if tracking_id = model.parse_tracking_id(data)
+          model.remove_tracking_id(data)
+          tracking_id.split(":").last
         else
           Kennel::Utils.parameterize(title)
         end
@@ -66,7 +65,13 @@ module Kennel
         data[:type] = "query alert" if data[:type] == "metric alert"
       when "dashboard"
         widgets = data[:widgets]&.flat_map { |widget| widget.dig(:definition, :widgets) || [widget] }
-        widgets&.each { |widget| dry_up_query!(widget) }
+        widgets&.each do |widget|
+          convert_widget_to_compact_format!(widget)
+          dry_up_widget_metadata!(widget)
+          (widget.dig(:definition, :markers) || []).each { |m| m[:label]&.delete! " " }
+        end
+      else
+        # noop
       end
 
       data.delete(:tags) if data[:tags] == [] # do not create super + [] call
@@ -88,7 +93,7 @@ module Kennel
     private
 
     # reduce duplication in imports by using dry `q: :metadata` when possible
-    def dry_up_query!(widget)
+    def dry_up_widget_metadata!(widget)
       (widget.dig(:definition, :requests) || []).each do |request|
         next unless request.is_a?(Hash)
         next unless metadata = request[:metadata]
@@ -101,6 +106,20 @@ module Kennel
       end
     end
 
+    # new api format is very verbose, so use old dry format when possible
+    def convert_widget_to_compact_format!(widget)
+      (widget.dig(:definition, :requests) || []).each do |request|
+        next unless request.is_a?(Hash)
+        next if request[:formulas] && request[:formulas] != [{ formula: "query1" }]
+        next if request[:queries]&.size != 1
+        next if request[:queries].any? { |q| q[:data_source] != "metrics" }
+        next if widget.dig(:definition, :type) != request[:response_format]
+        request.delete(:formulas)
+        request.delete(:response_format)
+        request[:q] = request.delete(:queries).first.fetch(:query)
+      end
+    end
+
     def pretty_print(hash)
       sort_widgets hash
 

data/lib/kennel/models/base.rb

@@ -11,7 +11,7 @@ module Kennel
 
       def kennel_id
         name = self.class.name
-        if name.start_with?("Kennel::")
+        if name.start_with?("Kennel::") # core objects would always generate the same id
           raise_with_location ArgumentError, "Set :kennel_id"
         end
         @kennel_id ||= Utils.snake_case name

data/lib/kennel/models/dashboard.rb

@@ -8,12 +8,61 @@ module Kennel
       READONLY_ATTRIBUTES = superclass::READONLY_ATTRIBUTES + [
         :author_handle, :author_name, :modified_at, :url, :is_read_only, :notify_list
       ]
+      TRACKING_FIELD = :description
       REQUEST_DEFAULTS = {
         style: { line_width: "normal", palette: "dog_classic", line_type: "solid" }
       }.freeze
       WIDGET_DEFAULTS = {
-        "timeseries" => { show_legend: false, legend_size: "0" },
-        "note" => { background_color: "white", font_size: "14", show_tick: false, tick_edge: "left", tick_pos: "50%", text_align: "left" }
+        "timeseries" => {
+          legend_size: "0",
+          markers: [],
+          legend_columns: [
+            "avg",
+            "min",
+            "max",
+            "value",
+            "sum"
+          ],
+          legend_layout: "auto",
+          yaxis: {
+            include_zero: true,
+            label: "",
+            scale: "linear",
+            min: "auto",
+            max: "auto"
+          },
+          show_legend: true,
+          time: {},
+          title_align: "left",
+          title_size: "16"
+        },
+        "note" => {
+          show_tick: false,
+          tick_edge: "left",
+          tick_pos: "50%",
+          text_align: "left",
+          has_padding: true,
+          background_color: "white",
+          font_size: "14"
+        },
+        "query_value" => {
+          autoscale: true,
+          time: {},
+          title_align: "left",
+          title_size: "16"
+        },
+        "free_text" => {
+          font_size: "auto"
+        },
+        "check_status" => {
+          title_align: "left",
+          title_size: "16"
+        },
+        "slo" => {
+          global_time_target: "0",
+          title_align: "left",
+          title_size: "16"
+        }
       }.freeze
       SUPPORTED_DEFINITION_OPTIONS = [:events, :markers, :precision].freeze
 
@@ -40,53 +89,43 @@ module Kennel
         def normalize(expected, actual)
           super
 
-          ignore_default(expected, actual, DEFAULTS)
-          ignore_default(expected, actual, reflow_type: "auto") if expected[:layout_type] == "ordered"
+          ignore_default expected, actual, DEFAULTS
+          ignore_default expected, actual, reflow_type: "auto" if expected[:layout_type] == "ordered"
 
           widgets_pairs(expected, actual).each do |pair|
-            # conditional_formats ordering is randomly changed by datadog, compare a stable ordering
-            pair.each do |widgets|
-              widgets.each do |widget|
-                if formats = widget.dig(:definition, :conditional_formats)
-                  widget[:definition][:conditional_formats] = formats.sort_by(&:hash)
-                end
-              end
-            end
-
-            ignore_widget_defaults pair
-
+            pair.each { |w| sort_conditional_formats w }
+            ignore_widget_defaults(*pair)
             ignore_request_defaults(*pair)
-
-            # ids are kinda random so we always discard them
-            pair.each { |widgets| widgets.each { |w| w.delete(:id) } }
+            pair.each { |widget| widget&.delete(:id) } # ids are kinda random so we always discard them
           end
         end
 
         private
 
-        def ignore_widget_defaults(pair)
-          pair.map(&:size).max.times do |i|
-            types = pair.map { |w| w.dig(i, :definition, :type) }.uniq
-            next unless types.size == 1
-            next unless defaults = WIDGET_DEFAULTS[types.first]
-            ignore_defaults(pair[0], pair[1], defaults, nesting: :definition)
+        # conditional_formats ordering is randomly changed by datadog, compare a stable ordering
+        def sort_conditional_formats(widget)
+          if formats = widget&.dig(:definition, :conditional_formats)
+            widget[:definition][:conditional_formats] = formats.sort_by(&:hash)
           end
         end
 
+        def ignore_widget_defaults(expected, actual)
+          types = [expected&.dig(:definition, :type), actual&.dig(:definition, :type)].uniq.compact
+          return unless types.size == 1
+          return unless defaults = WIDGET_DEFAULTS[types.first]
+          ignore_default expected&.[](:definition) || {}, actual&.[](:definition) || {}, defaults
+        end
+
         # discard styles/conditional_formats/aggregator if nothing would change when we applied (both are default or nil)
         def ignore_request_defaults(expected, actual)
-          [expected.size, actual.size].max.times do |i|
-            a_r = actual.dig(i, :definition, :requests) || []
-            e_r = expected.dig(i, :definition, :requests) || []
-            ignore_defaults e_r, a_r, REQUEST_DEFAULTS
-          end
+          a_r = actual&.dig(:definition, :requests) || []
+          e_r = expected&.dig(:definition, :requests) || []
+          ignore_defaults e_r, a_r, REQUEST_DEFAULTS
         end
 
-        def ignore_defaults(expected, actual, defaults, nesting: nil)
+        def ignore_defaults(expected, actual, defaults)
           [expected.size, actual.size].max.times do |i|
-            e = expected.dig(i, *nesting) || {}
-            a = actual.dig(i, *nesting) || {}
-            ignore_default(e, a, defaults)
+            ignore_default expected[i] || {}, actual[i] || {}, defaults
           end
         end
 
@@ -99,7 +138,7 @@ module Kennel
             nested = pair.map { |d| d.dig(:widgets, i, :definition, :widgets) || [] }
             result << nested if nested.any?(&:any?)
           end
-          result
+          result.flat_map { |a, e| [a.size, e.size].max.times.map { |i| [a[i], e[i]] } }
         end
       end
 

data/lib/kennel/models/monitor.rb

@@ -9,6 +9,7 @@ module Kennel
       READONLY_ATTRIBUTES = superclass::READONLY_ATTRIBUTES + [
         :multi, :matching_downtimes, :overall_state_modified, :overall_state, :restricted_roles
       ]
+      TRACKING_FIELD = :message
 
       MONITOR_DEFAULTS = {
         priority: nil
@@ -25,6 +26,7 @@ module Kennel
         groupby_simple_monitor: false
       }.freeze
       DEFAULT_ESCALATION_MESSAGE = ["", nil].freeze
+      ALLOWED_PRIORITY_CLASSES = [NilClass, Integer].freeze
 
       settings(
         :query, :name, :message, :escalation_message, :critical, :type, :renotify_interval, :warning, :timeout_h, :evaluation_delay,
@@ -127,9 +129,15 @@ module Kennel
         Utils.path_to_url "/monitors##{id}/edit"
       end
 
-      # datadog uses / for show and # for edit as separator in it's links
       def self.parse_url(url)
-        return unless id = url[/\/monitors[\/#](\d+)/, 1]
+        # datadog uses / for show and # for edit as separator in it's links
+        id = url[/\/monitors[\/#](\d+)/, 1]
+
+        # slo alert url
+        id ||= url[/\/slo\/edit\/[a-z\d]{10,}\/alerts\/(\d+)/, 1]
+
+        return unless id
+
         Integer(id)
       end
 
@@ -217,6 +225,10 @@ module Kennel
             end
           end
         end
+
+        unless ALLOWED_PRIORITY_CLASSES.include?(priority.class)
+          invalid! "priority needs to be an Integer"
+        end
       end
     end
   end

data/lib/kennel/models/record.rb

@@ -3,8 +3,10 @@ module Kennel
   module Models
     class Record < Base
       LOCK = "\u{1F512}"
+      TRACKING_FIELDS = [:message, :description].freeze
       READONLY_ATTRIBUTES = [
-        :deleted, :id, :created, :created_at, :creator, :org_id, :modified, :modified_at, :api_resource
+        :deleted, :id, :created, :created_at, :creator, :org_id, :modified, :modified_at,
+        :klass, :tracking_id # added by syncer.rb
       ].freeze
 
       settings :id, :kennel_id
@@ -22,6 +24,18 @@ module Kennel
           subclasses.map { |s| [s.api_resource, s] }.to_h
         end
 
+        def parse_tracking_id(a)
+          a[self::TRACKING_FIELD].to_s[/-- Managed by kennel (\S+:\S+)/, 1]
+        end
+
+        # TODO: combine with parse into a single method or a single regex
+        def remove_tracking_id(a)
+          value = a[self::TRACKING_FIELD]
+          a[self::TRACKING_FIELD] =
+            value.dup.sub!(/\n?-- Managed by kennel .*/, "") ||
+            raise("did not find tracking id in #{value}")
+        end
+
         private
 
         def normalize(_expected, actual)
@@ -60,19 +74,40 @@ module Kennel
       end
 
       def tracking_id
-        "#{project.kennel_id}:#{kennel_id}"
+        @tracking_id ||= begin
+          id = "#{project.kennel_id}:#{kennel_id}"
+          raise ValidationError, "#{id} kennel_id cannot include whitespace" if id.match?(/\s/) # <-> parse_tracking_id
+          id
+        end
       end
 
       def resolve_linked_tracking_ids!(*)
       end
 
+      def add_tracking_id
+        json = as_json
+        if self.class.parse_tracking_id(json)
+          invalid! "remove \"-- Managed by kennel\" line it from #{self.class::TRACKING_FIELD} to copy a resource"
+        end
+        json[self.class::TRACKING_FIELD] =
+          "#{json[self.class::TRACKING_FIELD]}\n" \
+          "-- Managed by kennel #{tracking_id} in #{project.class.file_location}, do not modify manually".lstrip
+      end
+
+      def remove_tracking_id
+        self.class.remove_tracking_id(as_json)
+      end
+
       private
 
       def resolve_link(tracking_id, type, id_map, force:)
         id = id_map[tracking_id]
         if id == :new
           if force
-            invalid! "#{type} #{tracking_id} was referenced but is also created by the current run.\nIt could not be created because of a circular dependency, try creating only some of the resources"
+            invalid!(
+              "#{type} #{tracking_id} was referenced but is also created by the current run.\n" \
+              "It could not be created because of a circular dependency, try creating only some of the resources"
+            )
           else
             nil # will be re-resolved after the linked object was created
           end

data/lib/kennel/models/slo.rb

@@ -3,6 +3,7 @@ module Kennel
   module Models
     class Slo < Record
       READONLY_ATTRIBUTES = superclass::READONLY_ATTRIBUTES + [:type_id, :monitor_tags]
+      TRACKING_FIELD = :description
       DEFAULTS = {
         description: nil,
         query: nil,
@@ -63,7 +64,7 @@ module Kennel
       end
 
       def self.parse_url(url)
-        url[/\/slo(\?.*slo_id=|\/edit\/)([a-z\d]{10,})/, 2]
+        url[/\/slo(\?.*slo_id=|\/edit\/)([a-z\d]{10,})(&|$)/, 2]
       end
 
       def resolve_linked_tracking_ids!(id_map, **args)

data/lib/kennel/progress.rb

@@ -29,6 +29,8 @@ module Kennel
       Kennel.err.print "#{time.round(2)}s\n"
 
       result
+    ensure
+      stop = true
     end
   end
 end

data/lib/kennel/syncer.rb

@@ -1,22 +1,13 @@
 # frozen_string_literal: true
 module Kennel
   class Syncer
-    TRACKING_FIELDS = [:message, :description].freeze
     DELETE_ORDER = ["dashboard", "slo", "monitor"].freeze # dashboards references monitors + slos, slos reference monitors
+    LINE_UP = "\e[1A\033[K" # go up and clear
 
     def initialize(api, expected, project: nil)
       @api = api
       @project_filter = project
       @expected = expected
-      if @project_filter
-        original = @expected
-        @expected = @expected.select { |e| e.project.kennel_id == @project_filter }
-        if @expected.empty?
-          possible = original.map { |e| e.project.kennel_id }.uniq.sort
-          raise "#{@project_filter} does not match any projects, try any of these:\n#{possible.join("\n")}"
-        end
-      end
-      @expected.each { |e| add_tracking_id e }
       calculate_diff
       prevent_irreversible_partial_updates
     end
@@ -38,20 +29,28 @@ module Kennel
 
     def update
       each_resolved @create do |_, e|
+        message = "#{e.class.api_resource} #{e.tracking_id}"
+        Kennel.out.puts "Creating #{message}"
         reply = @api.create e.class.api_resource, e.as_json
+        cache_metadata reply, e.class
         id = reply.fetch(:id)
-        populate_id_map [reply] # allow resolving ids we could previously no resolve
-        Kennel.out.puts "Created #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.class.url(id)}"
+        populate_id_map [], [reply] # allow resolving ids we could previously no resolve
+        Kennel.out.puts "#{LINE_UP}Created #{message} #{e.class.url(id)}"
       end
 
       each_resolved @update do |id, e|
+        message = "#{e.class.api_resource} #{e.tracking_id} #{e.class.url(id)}"
+        Kennel.out.puts "Updating #{message}"
         @api.update e.class.api_resource, id, e.as_json
-        Kennel.out.puts "Updated #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.class.url(id)}"
+        Kennel.out.puts "#{LINE_UP}Updated #{message}"
       end
 
       @delete.each do |id, _, a|
-        @api.delete a.fetch(:api_resource), id
-        Kennel.out.puts "Deleted #{a.fetch(:api_resource)} #{tracking_id(a)} #{id}"
+        klass = a.fetch(:klass)
+        message = "#{klass.api_resource} #{a.fetch(:tracking_id)} #{id}"
+        Kennel.out.puts "Deleting #{message}"
+        @api.delete klass.api_resource, id
+        Kennel.out.puts "#{LINE_UP}Deleted #{message}"
       end
     end
 
@@ -99,14 +98,14 @@ module Kennel
 
       actual = Progress.progress("Downloading definitions") { download_definitions }
 
-      # resolve dependencies to avoid diff
-      populate_id_map actual
-      @expected.each { |e| @id_map[e.tracking_id] ||= :new }
-      resolve_linked_tracking_ids! @expected
+      Progress.progress "Diffing" do
+        filter_expected_by_project! @expected
+        populate_id_map @expected, actual
+        filter_actual_by_project! actual
+        resolve_linked_tracking_ids! @expected # resolve dependencies to avoid diff
 
-      filter_by_project! actual
+        @expected.each(&:add_tracking_id) # avoid diff with actual
 
-      Progress.progress "Diffing" do
         items = actual.map do |a|
           e = matching_expected(a)
           if e && @expected.delete(e)
@@ -117,9 +116,8 @@ module Kennel
         end
 
         # fill details of things we need to compare
-        detailed = Hash.new { |h, k| h[k] = [] }
-        items.each { |e, a| detailed[a[:api_resource]] << a if e }
-        detailed.each { |api_resource, actuals| @api.fill_details! api_resource, actuals }
+        details = items.map { |e, a| a if e && e.class.api_resource == "dashboard" }.compact
+        @api.fill_details! "dashboard", details
 
         # pick out things to update or delete
         items.each do |e, a|
@@ -127,26 +125,30 @@ module Kennel
           if e
             diff = e.diff(a)
             @update << [id, e, a, diff] if diff.any?
-          elsif tracking_id(a) # was previously managed
+          elsif a.fetch(:tracking_id) # was previously managed
             @delete << [id, nil, a]
           end
         end
 
         ensure_all_ids_found
         @create = @expected.map { |e| [nil, e] }
+        @delete.sort_by! { |_, _, a| DELETE_ORDER.index a.fetch(:klass).api_resource }
       end
-
-      @delete.sort_by! { |_, _, a| DELETE_ORDER.index a.fetch(:api_resource) }
     end
 
     def download_definitions
-      Utils.parallel(Models::Record.subclasses.map(&:api_resource)) do |api_resource|
-        results = @api.list(api_resource, with_downtimes: false) # lookup monitors without adding unnecessary downtime information
+      Utils.parallel(Models::Record.subclasses) do |klass|
+        results = @api.list(klass.api_resource, with_downtimes: false) # lookup monitors without adding unnecessary downtime information
         results = results[results.keys.first] if results.is_a?(Hash) # dashboards are nested in {dashboards: []}
-        results.each { |c| c[:api_resource] = api_resource } # store api resource for later diffing
+        results.each { |a| cache_metadata(a, klass) }
       end.flatten(1)
     end
 
+    def cache_metadata(a, klass)
+      a[:klass] = klass
+      a[:tracking_id] = a.fetch(:klass).parse_tracking_id(a)
+    end
+
     def ensure_all_ids_found
       @expected.each do |e|
         next unless id = e.id
@@ -158,7 +160,7 @@ module Kennel
     def matching_expected(a)
       # index list by all the thing we look up by: tracking id and actual id
       @lookup_map ||= @expected.each_with_object({}) do |e, all|
-        keys = [tracking_id(e.as_json)]
+        keys = [e.tracking_id]
         keys << "#{e.class.api_resource}:#{e.id}" if e.id
         keys.compact.each do |key|
           raise "Lookup #{key} is duplicated" if all[key]
@@ -166,14 +168,15 @@ module Kennel
         end
       end
 
-      @lookup_map["#{a.fetch(:api_resource)}:#{a.fetch(:id)}"] || @lookup_map[tracking_id(a)]
+      klass = a.fetch(:klass)
+      @lookup_map["#{klass.api_resource}:#{a.fetch(:id)}"] || @lookup_map[a.fetch(:tracking_id)]
     end
 
     def print_plan(step, list, color)
       return if list.empty?
       list.each do |_, e, a, diff|
-        api_resource = (e ? e.class.api_resource : a.fetch(:api_resource))
-        Kennel.out.puts Utils.color(color, "#{step} #{api_resource} #{e&.tracking_id || tracking_id(a)}")
+        klass = (e ? e.class : a.fetch(:klass))
+        Kennel.out.puts Utils.color(color, "#{step} #{klass.api_resource} #{e&.tracking_id || a.fetch(:tracking_id)}")
         print_diff(diff) if diff # only for update
       end
     end
@@ -199,76 +202,58 @@ module Kennel
       end
     end
 
-    # Do not add tracking-id when working with existing ids on a branch,
-    # so resource do not get deleted from running an update on master (for example merge->CI).
-    # Also make sure the diff still makes sense, by kicking out the now noop-update.
-    #
-    # Note: ideally we'd never add tracking in the first place, but at that point we do not know the diff yet
+    # - do not add tracking-id when working with existing ids on a branch,
+    #   so resource do not get deleted when running an update on master (for example merge->CI)
+    # - make sure the diff is clean, by kicking out the now noop-update
+    # - ideally we'd never add tracking in the first place, but when adding tracking we do not know the diff yet
     def prevent_irreversible_partial_updates
       return unless @project_filter
       @update.select! do |_, e, _, diff|
-        next true unless e.id # short circuit for performance
+        next true unless e.id # safe to add tracking when not having id
 
         diff.select! do |field_diff|
-          (_, field, old, new) = field_diff
-          next true unless tracking_field?(field)
+          (_, field, actual) = field_diff
+          # TODO: refactor this so TRACKING_FIELD stays record-private
+          next true if e.class::TRACKING_FIELD != field.to_sym # need to sym here because Hashdiff produces strings
+          next true if e.class.parse_tracking_id(field.to_sym => actual) # already has tracking id
 
-          if (old_tracking = tracking_value(old))
-            old_tracking == tracking_value(new) || raise("do not update! (atm unreachable)")
-          else
-            field_diff[3] = remove_tracking_id(e) # make plan output match update
-            old != field_diff[3]
-          end
+          field_diff[3] = e.remove_tracking_id # make `rake plan` output match what we are sending
+          actual != field_diff[3] # discard diff if now nothing changes
         end
 
         !diff.empty?
       end
     end
 
-    def populate_id_map(actual)
-      actual.each { |a| @id_map[tracking_id(a)] = a.fetch(:id) }
+    def populate_id_map(expected, actual)
+      actual.each do |a|
+        next unless tracking_id = a.fetch(:tracking_id)
+        @id_map[tracking_id] = a.fetch(:id)
+      end
+      expected.each { |e| @id_map[e.tracking_id] ||= :new }
     end
 
     def resolve_linked_tracking_ids!(list, force: false)
       list.each { |e| e.resolve_linked_tracking_ids!(@id_map, force: force) }
     end
 
-    def filter_by_project!(definitions)
+    def filter_actual_by_project!(actual)
       return unless @project_filter
-      definitions.select! do |a|
-        id = tracking_id(a)
-        !id || id.start_with?("#{@project_filter}:")
+      actual.select! do |a|
+        tracking_id = a.fetch(:tracking_id)
+        !tracking_id || tracking_id.start_with?("#{@project_filter}:")
       end
     end
 
-    def add_tracking_id(e)
-      json = e.as_json
-      field = tracking_field(json)
-      raise "remove \"-- Managed by kennel\" line it from #{field} to copy a resource" if tracking_value(json[field])
-      json[field] = "#{json[field]}\n-- Managed by kennel #{e.tracking_id} in #{e.project.class.file_location}, do not modify manually".lstrip
-    end
-
-    def remove_tracking_id(e)
-      json = e.as_json
-      field = tracking_field(json)
-      value = json[field]
-      json[field] = value.dup.sub!(/\n?-- Managed by kennel .*/, "") || raise("did not find tracking id in #{value}")
-    end
-
-    def tracking_id(a)
-      tracking_value a[tracking_field(a)]
-    end
-
-    def tracking_value(content)
-      content.to_s[/-- Managed by kennel (\S+:\S+)/, 1]
-    end
-
-    def tracking_field(a)
-      TRACKING_FIELDS.detect { |f| a.key?(f) }
-    end
+    def filter_expected_by_project!(expected)
+      return unless @project_filter
+      original = expected.dup
+      expected.select! { |e| e.project.kennel_id == @project_filter }
 
-    def tracking_field?(field)
-      TRACKING_FIELDS.include?(field.to_sym)
+      if expected.empty?
+        possible = original.map { |e| e.project.kennel_id }.uniq.sort
+        raise "#{@project_filter} does not match any projects, try any of these:\n#{possible.join("\n")}"
+      end
     end
   end
 end

data/lib/kennel/tasks.rb

@@ -138,7 +138,7 @@ namespace :kennel do
     resources.each do |resource|
       Kennel::Progress.progress("Downloading #{resource}") do
         list = api.list(resource)
-        api.fill_details!(resource, list)
+        api.fill_details!(resource, list) if resource == "dashboard"
       end
       list.each do |r|
         r[:api_resource] = resource

data/lib/kennel/template_variables.rb

@@ -36,7 +36,8 @@ module Kennel
 
     def widget_queries(widget)
       requests = widget.dig(:definition, :requests) || []
-      (requests.is_a?(Hash) ? requests.values : requests).map { |r| r[:q] } # hostmap widgets have hash requests
+      return requests.values.map { |r| r[:q] } if requests.is_a?(Hash) # hostmap widgets have hash requests
+      requests.flat_map { |r| r[:q] || r[:queries]&.map { |q| q[:query] } } # old format with q: or queries: [{query:}]
     end
   end
 end

data/lib/kennel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Kennel
-  VERSION = "1.86.1"
+  VERSION = "1.89.1"
 end

data/template/Readme.md

@@ -96,6 +96,7 @@ end
  - use [datadog monitor UI](https://app.datadoghq.com/monitors/manage) to find a monitor
  - get the `id` from the url
  - run `URL='https://app.datadoghq.com/monitors/123' bundle exec rake kennel:import` and copy the output
+ - import task also works with SLO alerts, e.g. `URL='https://app.datadoghq.com/slo/edit/123abc456def123/alerts/789' bundle exec rake kennel:import`
  - find or create a project in `projects/`
  - add the monitor to `parts: [` list, for example:
   ```Ruby

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kennel
 version: !ruby/object:Gem::Version
-  version: 1.86.1
+  version: 1.89.1
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-21 00:00:00.000000000 Z
+date: 2021-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday