kennel 1.85.2 → 1.88.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 80ab3a8fb786a4b556aa1318d3b1ab758a9b7d33e786023c02550ee004abd6e8
-  data.tar.gz: 0e5896d464686e750a3973feff199f25030cea119c7ccb03ddc9820a359f0c74
+  metadata.gz: 1091fcd44daba6435e7a9321610a27b8ac2510451cf97e8183d6ab744e080288
+  data.tar.gz: f137076ec36e4018eaa1c1d60dd2da187f33bdb41fc3fe5c384be4787ddd9c77
 SHA512:
-  metadata.gz: a5df094d658262ddb50a40234be1f371ebd9120ed66a0a625d68a7ec8debde55980ccc8e2c64e84afbdc16160228700c8361108b526f54ca68403e12318f8a19
-  data.tar.gz: cb924336b3e9caac886fbc60ffc0301500a6dd59f9af53c42d1c7ac13ad5891765165b28fa7da36751da64491ca8e5070e1e1ce047830e8cb3b78363e9e2da4d
+  metadata.gz: 52cd265a23705e8a8afca55f819c9fed1c2ae467e8de1cba5e696cd6191619458168fd649f26f5c00da33702e4002490e13f804335dad10e2204a96d16cf12a8
+  data.tar.gz: d2cc461399929023573e22f55bc89699d952c9338f5e415cea9328af00d212b5e87fe2061837327ec88c4b60a2a277e1f272a511e85a9de0014142c51073f60e

data/lib/kennel/importer.rb

@@ -3,7 +3,7 @@
 module Kennel
   class Importer
     TITLES = [:name, :title].freeze
-    SORT_ORDER = [*TITLES, :id, :kennel_id, :type, :tags, :query, *Syncer::TRACKING_FIELDS, :template_variables].freeze
+    SORT_ORDER = [*TITLES, :id, :kennel_id, :type, :tags, :query, *Models::Record.subclasses.map { |k| k::TRACKING_FIELDS }, :template_variables].freeze
 
     def initialize(api)
       @api = api
@@ -31,11 +31,10 @@ module Kennel
       title.tr!(Kennel::Models::Record::LOCK, "") # avoid double lock icon
 
       # calculate or reuse kennel_id
-      # TODO: this is copy-pasted from syncer, need to find a nice way to reuse it
-      tracking_field = Syncer::TRACKING_FIELDS.detect { |f| data[f] }
       data[:kennel_id] =
-        if tracking_field && data[tracking_field].sub!(/\n?-- Managed by kennel (\S+:\S+).*/, "")
-          $1.split(":").last
+        if tracking_id = model.parse_tracking_id(data)
+          model.remove_tracking_id(data)
+          tracking_id.split(":").last
         else
           Kennel::Utils.parameterize(title)
         end
@@ -66,7 +65,13 @@ module Kennel
         data[:type] = "query alert" if data[:type] == "metric alert"
       when "dashboard"
         widgets = data[:widgets]&.flat_map { |widget| widget.dig(:definition, :widgets) || [widget] }
-        widgets&.each { |widget| dry_up_query!(widget) }
+        widgets&.each do |widget|
+          convert_widget_to_compact_format!(widget)
+          dry_up_widget_metadata!(widget)
+          (widget.dig(:definition, :markers) || []).each { |m| m[:label]&.delete! " " }
+        end
+      else
+        # noop
       end
 
       data.delete(:tags) if data[:tags] == [] # do not create super + [] call
@@ -88,7 +93,7 @@ module Kennel
     private
 
     # reduce duplication in imports by using dry `q: :metadata` when possible
-    def dry_up_query!(widget)
+    def dry_up_widget_metadata!(widget)
       (widget.dig(:definition, :requests) || []).each do |request|
         next unless request.is_a?(Hash)
         next unless metadata = request[:metadata]
@@ -101,6 +106,19 @@ module Kennel
       end
     end
 
+    # new api format is very verbose, so use old dry format when possible
+    def convert_widget_to_compact_format!(widget)
+      (widget.dig(:definition, :requests) || []).each do |request|
+        next unless request.is_a?(Hash)
+        next if request[:formulas] && request[:formulas] != [{ formula: "query1" }]
+        next if request[:queries]&.size != 1
+        next if request[:queries].any? { |q| q[:data_source] != "metrics" }
+        request.delete(:formulas)
+        request[:type] = request.delete(:response_format)
+        request[:q] = request.delete(:queries).first.fetch(:query)
+      end
+    end
+
     def pretty_print(hash)
       sort_widgets hash
 

data/lib/kennel/models/dashboard.rb

@@ -8,12 +8,61 @@ module Kennel
       READONLY_ATTRIBUTES = superclass::READONLY_ATTRIBUTES + [
         :author_handle, :author_name, :modified_at, :url, :is_read_only, :notify_list
       ]
+      TRACKING_FIELD = :description
       REQUEST_DEFAULTS = {
         style: { line_width: "normal", palette: "dog_classic", line_type: "solid" }
       }.freeze
       WIDGET_DEFAULTS = {
-        "timeseries" => { show_legend: false, legend_size: "0" },
-        "note" => { background_color: "white", font_size: "14", show_tick: false, tick_edge: "left", tick_pos: "50%", text_align: "left" }
+        "timeseries" => {
+          legend_size: "0",
+          markers: [],
+          legend_columns: [
+            "avg",
+            "min",
+            "max",
+            "value",
+            "sum"
+          ],
+          legend_layout: "auto",
+          yaxis: {
+            include_zero: true,
+            label: "",
+            scale: "linear",
+            min: "auto",
+            max: "auto"
+          },
+          show_legend: true,
+          time: {},
+          title_align: "left",
+          title_size: "16"
+        },
+        "note" => {
+          show_tick: false,
+          tick_edge: "left",
+          tick_pos: "50%",
+          text_align: "left",
+          has_padding: true,
+          background_color: "white",
+          font_size: "14"
+        },
+        "query_value" => {
+          autoscale: true,
+          time: {},
+          title_align: "left",
+          title_size: "16"
+        },
+        "free_text" => {
+          font_size: "auto"
+        },
+        "check_status" => {
+          title_align: "left",
+          title_size: "16"
+        },
+        "slo" => {
+          global_time_target: "0",
+          title_align: "left",
+          title_size: "16"
+        }
       }.freeze
       SUPPORTED_DEFINITION_OPTIONS = [:events, :markers, :precision].freeze
 
@@ -40,53 +89,43 @@ module Kennel
         def normalize(expected, actual)
           super
 
-          ignore_default(expected, actual, DEFAULTS)
-          ignore_default(expected, actual, reflow_type: "auto") if expected[:layout_type] == "ordered"
+          ignore_default expected, actual, DEFAULTS
+          ignore_default expected, actual, reflow_type: "auto" if expected[:layout_type] == "ordered"
 
           widgets_pairs(expected, actual).each do |pair|
-            # conditional_formats ordering is randomly changed by datadog, compare a stable ordering
-            pair.each do |widgets|
-              widgets.each do |widget|
-                if formats = widget.dig(:definition, :conditional_formats)
-                  widget[:definition][:conditional_formats] = formats.sort_by(&:hash)
-                end
-              end
-            end
-
-            ignore_widget_defaults pair
-
+            pair.each { |w| sort_conditional_formats w }
+            ignore_widget_defaults(*pair)
             ignore_request_defaults(*pair)
-
-            # ids are kinda random so we always discard them
-            pair.each { |widgets| widgets.each { |w| w.delete(:id) } }
+            pair.each { |widget| widget&.delete(:id) } # ids are kinda random so we always discard them
           end
         end
 
         private
 
-        def ignore_widget_defaults(pair)
-          pair.map(&:size).max.times do |i|
-            types = pair.map { |w| w.dig(i, :definition, :type) }.uniq
-            next unless types.size == 1
-            next unless defaults = WIDGET_DEFAULTS[types.first]
-            ignore_defaults(pair[0], pair[1], defaults, nesting: :definition)
+        # conditional_formats ordering is randomly changed by datadog, compare a stable ordering
+        def sort_conditional_formats(widget)
+          if formats = widget&.dig(:definition, :conditional_formats)
+            widget[:definition][:conditional_formats] = formats.sort_by(&:hash)
           end
         end
 
+        def ignore_widget_defaults(expected, actual)
+          types = [expected&.dig(:definition, :type), actual&.dig(:definition, :type)].uniq.compact
+          return unless types.size == 1
+          return unless defaults = WIDGET_DEFAULTS[types.first]
+          ignore_default expected&.[](:definition) || {}, actual&.[](:definition) || {}, defaults
+        end
+
         # discard styles/conditional_formats/aggregator if nothing would change when we applied (both are default or nil)
         def ignore_request_defaults(expected, actual)
-          [expected.size, actual.size].max.times do |i|
-            a_r = actual.dig(i, :definition, :requests) || []
-            e_r = expected.dig(i, :definition, :requests) || []
-            ignore_defaults e_r, a_r, REQUEST_DEFAULTS
-          end
+          a_r = actual&.dig(:definition, :requests) || []
+          e_r = expected&.dig(:definition, :requests) || []
+          ignore_defaults e_r, a_r, REQUEST_DEFAULTS
         end
 
-        def ignore_defaults(expected, actual, defaults, nesting: nil)
+        def ignore_defaults(expected, actual, defaults)
           [expected.size, actual.size].max.times do |i|
-            e = expected.dig(i, *nesting) || {}
-            a = actual.dig(i, *nesting) || {}
-            ignore_default(e, a, defaults)
+            ignore_default expected[i] || {}, actual[i] || {}, defaults
           end
         end
 
@@ -99,7 +138,7 @@ module Kennel
             nested = pair.map { |d| d.dig(:widgets, i, :definition, :widgets) || [] }
             result << nested if nested.any?(&:any?)
           end
-          result
+          result.flat_map { |a, e| [a.size, e.size].max.times.map { |i| [a[i], e[i]] } }
         end
       end
 

data/lib/kennel/models/monitor.rb

@@ -9,6 +9,7 @@ module Kennel
       READONLY_ATTRIBUTES = superclass::READONLY_ATTRIBUTES + [
         :multi, :matching_downtimes, :overall_state_modified, :overall_state, :restricted_roles
       ]
+      TRACKING_FIELD = :message
 
       MONITOR_DEFAULTS = {
         priority: nil
@@ -25,6 +26,7 @@ module Kennel
         groupby_simple_monitor: false
       }.freeze
       DEFAULT_ESCALATION_MESSAGE = ["", nil].freeze
+      ALLOWED_PRIORITY_CLASSES = [NilClass, Integer].freeze
 
       settings(
         :query, :name, :message, :escalation_message, :critical, :type, :renotify_interval, :warning, :timeout_h, :evaluation_delay,
@@ -217,6 +219,10 @@ module Kennel
             end
           end
         end
+
+        unless ALLOWED_PRIORITY_CLASSES.include?(priority.class)
+          invalid! "priority needs to be an Integer"
+        end
       end
     end
   end

data/lib/kennel/models/record.rb

@@ -3,8 +3,10 @@ module Kennel
   module Models
     class Record < Base
       LOCK = "\u{1F512}"
+      TRACKING_FIELDS = [:message, :description].freeze
       READONLY_ATTRIBUTES = [
-        :deleted, :id, :created, :created_at, :creator, :org_id, :modified, :modified_at, :api_resource
+        :deleted, :id, :created, :created_at, :creator, :org_id, :modified, :modified_at,
+        :klass # added by syncer.rb
       ].freeze
 
       settings :id, :kennel_id
@@ -22,6 +24,18 @@ module Kennel
           subclasses.map { |s| [s.api_resource, s] }.to_h
         end
 
+        def parse_tracking_id(a)
+          a[self::TRACKING_FIELD].to_s[/-- Managed by kennel (\S+:\S+)/, 1]
+        end
+
+        # TODO: combine with parse into a single method or a single regex
+        def remove_tracking_id(a)
+          value = a[self::TRACKING_FIELD]
+          a[self::TRACKING_FIELD] =
+            value.dup.sub!(/\n?-- Managed by kennel .*/, "") ||
+            raise("did not find tracking id in #{value}")
+        end
+
         private
 
         def normalize(_expected, actual)
@@ -66,13 +80,30 @@ module Kennel
       def resolve_linked_tracking_ids!(*)
       end
 
+      def add_tracking_id
+        json = as_json
+        if self.class.parse_tracking_id(json)
+          invalid! "remove \"-- Managed by kennel\" line it from #{self.class::TRACKING_FIELD} to copy a resource"
+        end
+        json[self.class::TRACKING_FIELD] =
+          "#{json[self.class::TRACKING_FIELD]}\n" \
+          "-- Managed by kennel #{tracking_id} in #{project.class.file_location}, do not modify manually".lstrip
+      end
+
+      def remove_tracking_id
+        self.class.remove_tracking_id(as_json)
+      end
+
       private
 
       def resolve_link(tracking_id, type, id_map, force:)
         id = id_map[tracking_id]
         if id == :new
           if force
-            invalid! "#{type} #{tracking_id} was referenced but is also created by the current run.\nIt could not be created because of a circular dependency, try creating only some of the resources"
+            invalid!(
+              "#{type} #{tracking_id} was referenced but is also created by the current run.\n" \
+              "It could not be created because of a circular dependency, try creating only some of the resources"
+            )
           else
             nil # will be re-resolved after the linked object was created
           end

data/lib/kennel/models/slo.rb

@@ -3,6 +3,7 @@ module Kennel
   module Models
     class Slo < Record
       READONLY_ATTRIBUTES = superclass::READONLY_ATTRIBUTES + [:type_id, :monitor_tags]
+      TRACKING_FIELD = :description
       DEFAULTS = {
         description: nil,
         query: nil,
@@ -63,7 +64,7 @@ module Kennel
       end
 
       def self.parse_url(url)
-        url[/\/slo\?.*slo_id=([a-z\d]+)/, 1]
+        url[/\/slo(\?.*slo_id=|\/edit\/)([a-z\d]{10,})/, 2]
       end
 
       def resolve_linked_tracking_ids!(id_map, **args)

data/lib/kennel/syncer.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 module Kennel
   class Syncer
-    TRACKING_FIELDS = [:message, :description].freeze
     DELETE_ORDER = ["dashboard", "slo", "monitor"].freeze # dashboards references monitors + slos, slos reference monitors
 
     def initialize(api, expected, project: nil)
@@ -16,7 +15,7 @@ module Kennel
           raise "#{@project_filter} does not match any projects, try any of these:\n#{possible.join("\n")}"
         end
       end
-      @expected.each { |e| add_tracking_id e }
+      @expected.each(&:add_tracking_id)
       calculate_diff
       prevent_irreversible_partial_updates
     end
@@ -39,19 +38,22 @@ module Kennel
     def update
       each_resolved @create do |_, e|
         reply = @api.create e.class.api_resource, e.as_json
+        reply[:klass] = e.class # store api resource class for later use
         id = reply.fetch(:id)
         populate_id_map [reply] # allow resolving ids we could previously no resolve
-        Kennel.out.puts "Created #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.class.url(id)}"
+        Kennel.out.puts "Created #{e.class.api_resource} #{e.tracking_id} #{e.class.url(id)}"
       end
 
       each_resolved @update do |id, e|
         @api.update e.class.api_resource, id, e.as_json
-        Kennel.out.puts "Updated #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.class.url(id)}"
+        Kennel.out.puts "Updated #{e.class.api_resource} #{e.tracking_id} #{e.class.url(id)}"
       end
 
       @delete.each do |id, _, a|
-        @api.delete a.fetch(:api_resource), id
-        Kennel.out.puts "Deleted #{a.fetch(:api_resource)} #{tracking_id(a)} #{id}"
+        klass = a.fetch(:klass)
+        @api.delete klass.api_resource, id
+        tracking_id = klass.parse_tracking_id(a)
+        Kennel.out.puts "Deleted #{klass.api_resource} #{tracking_id} #{id}"
       end
     end
 
@@ -118,8 +120,8 @@ module Kennel
 
         # fill details of things we need to compare
         detailed = Hash.new { |h, k| h[k] = [] }
-        items.each { |e, a| detailed[a[:api_resource]] << a if e }
-        detailed.each { |api_resource, actuals| @api.fill_details! api_resource, actuals }
+        items.each { |e, a| detailed[a[:klass]] << a if e }
+        detailed.each { |klass, actuals| @api.fill_details! klass.api_resource, actuals }
 
         # pick out things to update or delete
         items.each do |e, a|
@@ -127,7 +129,7 @@ module Kennel
           if e
             diff = e.diff(a)
             @update << [id, e, a, diff] if diff.any?
-          elsif tracking_id(a) # was previously managed
+          elsif a.fetch(:klass).parse_tracking_id(a) # was previously managed
             @delete << [id, nil, a]
           end
         end
@@ -136,14 +138,14 @@ module Kennel
         @create = @expected.map { |e| [nil, e] }
       end
 
-      @delete.sort_by! { |_, _, a| DELETE_ORDER.index a.fetch(:api_resource) }
+      @delete.sort_by! { |_, _, a| DELETE_ORDER.index a.fetch(:klass).api_resource }
     end
 
     def download_definitions
-      Utils.parallel(Models::Record.subclasses.map(&:api_resource)) do |api_resource|
-        results = @api.list(api_resource, with_downtimes: false) # lookup monitors without adding unnecessary downtime information
+      Utils.parallel(Models::Record.subclasses) do |klass|
+        results = @api.list(klass.api_resource, with_downtimes: false) # lookup monitors without adding unnecessary downtime information
         results = results[results.keys.first] if results.is_a?(Hash) # dashboards are nested in {dashboards: []}
-        results.each { |c| c[:api_resource] = api_resource } # store api resource for later diffing
+        results.each { |c| c[:klass] = klass } # store api resource for later diffing
       end.flatten(1)
     end
 
@@ -158,7 +160,7 @@ module Kennel
     def matching_expected(a)
       # index list by all the thing we look up by: tracking id and actual id
       @lookup_map ||= @expected.each_with_object({}) do |e, all|
-        keys = [tracking_id(e.as_json)]
+        keys = [e.tracking_id]
         keys << "#{e.class.api_resource}:#{e.id}" if e.id
         keys.compact.each do |key|
           raise "Lookup #{key} is duplicated" if all[key]
@@ -166,14 +168,15 @@ module Kennel
         end
       end
 
-      @lookup_map["#{a.fetch(:api_resource)}:#{a.fetch(:id)}"] || @lookup_map[tracking_id(a)]
+      klass = a.fetch(:klass)
+      @lookup_map["#{klass.api_resource}:#{a.fetch(:id)}"] || @lookup_map[klass.parse_tracking_id(a)]
     end
 
     def print_plan(step, list, color)
       return if list.empty?
       list.each do |_, e, a, diff|
-        api_resource = (e ? e.class.api_resource : a.fetch(:api_resource))
-        Kennel.out.puts Utils.color(color, "#{step} #{api_resource} #{e&.tracking_id || tracking_id(a)}")
+        klass = (e ? e.class : a.fetch(:klass))
+        Kennel.out.puts Utils.color(color, "#{step} #{klass.api_resource} #{e&.tracking_id || klass.parse_tracking_id(a)}")
         print_diff(diff) if diff # only for update
       end
     end
@@ -199,26 +202,23 @@ module Kennel
       end
     end
 
-    # Do not add tracking-id when working with existing ids on a branch,
-    # so resource do not get deleted from running an update on master (for example merge->CI).
-    # Also make sure the diff still makes sense, by kicking out the now noop-update.
-    #
-    # Note: ideally we'd never add tracking in the first place, but at that point we do not know the diff yet
+    # - do not add tracking-id when working with existing ids on a branch,
+    #   so resource do not get deleted when running an update on master (for example merge->CI)
+    # - make sure the diff is clean, by kicking out the now noop-update
+    # - ideally we'd never add tracking in the first place, but when adding tracking we do not know the diff yet
     def prevent_irreversible_partial_updates
       return unless @project_filter
       @update.select! do |_, e, _, diff|
-        next true unless e.id # short circuit for performance
+        next true unless e.id # safe to add tracking when not having id
 
         diff.select! do |field_diff|
-          (_, field, old, new) = field_diff
-          next true unless tracking_field?(field)
+          (_, field, actual) = field_diff
+          # TODO: refactor this so TRACKING_FIELD stays record-private
+          next true if e.class::TRACKING_FIELD != field.to_sym # need to sym here because Hashdiff produces strings
+          next true if e.class.parse_tracking_id(field.to_sym => actual) # already has tracking id
 
-          if (old_tracking = tracking_value(old))
-            old_tracking == tracking_value(new) || raise("do not update! (atm unreachable)")
-          else
-            field_diff[3] = remove_tracking_id(e) # make plan output match update
-            old != field_diff[3]
-          end
+          field_diff[3] = e.remove_tracking_id # make `rake plan` output match what we are sending
+          actual != field_diff[3] # discard diff if now nothing changes
         end
 
         !diff.empty?
@@ -226,7 +226,7 @@ module Kennel
     end
 
     def populate_id_map(actual)
-      actual.each { |a| @id_map[tracking_id(a)] = a.fetch(:id) }
+      actual.each { |a| @id_map[a.fetch(:klass).parse_tracking_id(a)] = a.fetch(:id) }
     end
 
     def resolve_linked_tracking_ids!(list, force: false)
@@ -236,39 +236,9 @@ module Kennel
     def filter_by_project!(definitions)
       return unless @project_filter
       definitions.select! do |a|
-        id = tracking_id(a)
+        id = a.fetch(:klass).parse_tracking_id(a)
         !id || id.start_with?("#{@project_filter}:")
       end
     end
-
-    def add_tracking_id(e)
-      json = e.as_json
-      field = tracking_field(json)
-      raise "remove \"-- Managed by kennel\" line it from #{field} to copy a resource" if tracking_value(json[field])
-      json[field] = "#{json[field]}\n-- Managed by kennel #{e.tracking_id} in #{e.project.class.file_location}, do not modify manually".lstrip
-    end
-
-    def remove_tracking_id(e)
-      json = e.as_json
-      field = tracking_field(json)
-      value = json[field]
-      json[field] = value.dup.sub!(/\n?-- Managed by kennel .*/, "") || raise("did not find tracking id in #{value}")
-    end
-
-    def tracking_id(a)
-      tracking_value a[tracking_field(a)]
-    end
-
-    def tracking_value(content)
-      content.to_s[/-- Managed by kennel (\S+:\S+)/, 1]
-    end
-
-    def tracking_field(a)
-      TRACKING_FIELDS.detect { |f| a.key?(f) }
-    end
-
-    def tracking_field?(field)
-      TRACKING_FIELDS.include?(field.to_sym)
-    end
   end
 end

data/lib/kennel/template_variables.rb

@@ -36,7 +36,8 @@ module Kennel
 
     def widget_queries(widget)
       requests = widget.dig(:definition, :requests) || []
-      (requests.is_a?(Hash) ? requests.values : requests).map { |r| r[:q] } # hostmap widgets have hash requests
+      return requests.values.map { |r| r[:q] } if requests.is_a?(Hash) # hostmap widgets have hash requests
+      requests.flat_map { |r| r[:q] || r[:queries]&.map { |q| q[:query] } } # old format with q: or queries: [{query:}]
     end
   end
 end

data/lib/kennel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Kennel
-  VERSION = "1.85.2"
+  VERSION = "1.88.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kennel
 version: !ruby/object:Gem::Version
-  version: 1.85.2
+  version: 1.88.1
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-17 00:00:00.000000000 Z
+date: 2021-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday
@@ -94,14 +94,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.3
+rubygems_version: 3.2.16
 signing_key: 
 specification_version: 4
 summary: Keep datadog monitors/dashboards/etc in version control, avoid chaotic management