kennel 1.98.3 → 1.102.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1362ca60b0542b017fffad3e47bf1de8baf0b08950cb679a98760de51f0ef9c7
-  data.tar.gz: 7dacd36972ad7cad574d50236b6463a6be11082373b752c6f5f4530a29230a34
+  metadata.gz: 424978ab5e025d68c1b30adee8dcf9bbae531c34da1ce02abcc1a97165accb2f
+  data.tar.gz: 762d7b47d15937d0ea9b2bb7d9bb1c79d88b34828a0b455c05aa4d7498fd8f04
 SHA512:
-  metadata.gz: '069a8adb906d55515b5b21db21ca7186ab1d132b370bd9319910719df4d641ce71729f613b52e764b42a4127d0b32d0da86fdf9f01d440df870c2ebb52168730'
-  data.tar.gz: ea453b3fc2afa94e9f9d916dcdec01b9e077816ea4fcb9c886e2b0a9e36c0c2a7623b4c63b0fd5c2014fa4e04198f42dd759d6335980df899c7bf0b6c8e7b42f
+  metadata.gz: 2079300fb176ffe4a3c363c56dfb2fed0901b4f3cfce7ff6fd1c656f8981e16a94628eaa660ffee6a53cc86c9df2d7b604ca877c6c34c61eeec13cbe4b0484ee
+  data.tar.gz: 0113a4f7a4edf4813d66cb15090909e63a47945cfaa4bce22cd9fe40203f3b6dd3705c2f62652df7e09be9e4b7a7d5de1de2092074e2baa524c8392a3bb9b462

data/lib/kennel/id_map.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+module Kennel
+  class IdMap
+    NEW = :new # will be created during this run
+
+    def initialize
+      @map = Hash.new { |h, k| h[k] = {} }
+    end
+
+    def get(type, tracking_id)
+      @map[type][tracking_id]
+    end
+
+    def set(type, tracking_id, id)
+      @map[type][tracking_id] = id
+    end
+
+    def set_new(type, tracking_id)
+      @map[type][tracking_id] = NEW
+    end
+
+    def new?(type, tracking_id)
+      @map[type][tracking_id] == NEW
+    end
+  end
+end

data/lib/kennel/models/dashboard.rb

@@ -189,16 +189,17 @@ module Kennel
           when "uptime"
             if ids = definition[:monitor_ids]
               definition[:monitor_ids] = ids.map do |id|
-                tracking_id?(id) ? (resolve_link(id, :monitor, id_map, **args) || id) : id
+                resolve(id, :monitor, id_map, **args) || id
               end
             end
           when "alert_graph"
-            if (id = definition[:alert_id]) && tracking_id?(id)
-              definition[:alert_id] = (resolve_link(id, :monitor, id_map, **args) || id).to_s
+            if id = definition[:alert_id]
+              resolved = resolve(id, :monitor, id_map, **args) || id
+              definition[:alert_id] = resolved.to_s # even though it's a monitor id
             end
           when "slo"
-            if (id = definition[:slo_id]) && tracking_id?(id)
-              definition[:slo_id] = (resolve_link(id, :slo, id_map, **args) || id).to_s
+            if id = definition[:slo_id]
+              definition[:slo_id] = resolve(id, :slo, id_map, **args) || id
             end
           end
         end
@@ -206,10 +207,6 @@ module Kennel
 
       private
 
-      def tracking_id?(id)
-        id.is_a?(String) && id.include?(":")
-      end
-
       # creates queries from metadata to avoid having to keep q and expression in sync
       #
       # {q: :metadata, metadata: [{expression: "sum:bar", alias_name: "foo"}, ...], }

data/lib/kennel/models/monitor.rb

@@ -121,7 +121,7 @@ module Kennel
         when "composite", "slo alert"
           type = (as_json[:type] == "composite" ? :monitor : :slo)
           as_json[:query] = as_json[:query].gsub(/%{(.*?)}/) do
-            resolve_link($1, type, id_map, **args) || $&
+            resolve($1, type, id_map, **args) || $&
           end
         end
       end
@@ -214,27 +214,47 @@ module Kennel
         end
 
         if ["query alert", "service check"].include?(type) # TODO: most likely more types need this
-          # verify is_match/is_exact_match uses available variables
-          message = data.fetch(:message)
-          used = message.scan(/{{\s*([#^]is(?:_exact)?_match)\s*([^\s}]+)/)
-          if used.any?
-            allowed = data.fetch(:query)[/by\s*[({]([^})]+)[})]/, 1]
-              .to_s.gsub(/["']/, "").split(/\s*,\s*/)
-              .map! { |w| %("#{w}.name") }
-            used.uniq.each do |match, group|
-              next if allowed.include?(group)
-              invalid!(
-                "#{match} used with #{group}, but can only be used with #{allowed.join(", ")}. " \
-                "Group the query by #{group.sub(".name", "").tr('"', "")} or change the #{match}"
-              )
-            end
-          end
+          validate_message_variables(data)
         end
 
         unless ALLOWED_PRIORITY_CLASSES.include?(priority.class)
           invalid! "priority needs to be an Integer"
         end
       end
+
+      # verify is_match/is_exact_match and {{foo.name}} uses available variables
+      def validate_message_variables(data)
+        message = data.fetch(:message)
+
+        used =
+          message.scan(/{{\s*(?:[#^]is(?:_exact)?_match)\s*"([^\s}]+)"/) + # {{#is_match "environment.name" "production"}}
+          message.scan(/{{\s*([^}]+\.name)\s*}}/) # Pod {{pod.name}} failed
+        return if used.empty?
+        used.flatten!(1)
+        used.uniq!
+
+        # TODO
+        # - also match without by
+        # - separate parsers for query and service
+        # - service must always allow `host`, maybe others
+        return unless match = data.fetch(:query).match(/(?:{([^}]*)}\s*)?by\s*[({]([^})]+)[})]/)
+
+        allowed =
+          match[1].to_s.split(/\s*,\s*/).map { |k| k.split(":", 2)[-2] } + # {a:b} -> a TODO: does not work for service check
+          match[2].to_s.gsub(/["']/, "").split(/\s*,\s*/) # by {a} -> a
+
+        allowed.compact!
+        allowed.uniq!
+        allowed.map! { |w| "#{w.tr('"', "")}.name" }
+
+        forbidden = used - allowed
+        return if forbidden.empty?
+
+        invalid! <<~MSG.rstrip
+          Used #{forbidden.join(", ")} in the message, but can only be used with #{allowed.join(", ")}.
+          Group or filter the query by #{forbidden.map { |f| f.sub(".name", "") }.join(", ")} to use it.
+        MSG
+      end
     end
   end
 end

data/lib/kennel/models/record.rb

@@ -100,9 +100,20 @@ module Kennel
 
       private
 
+      def resolve(value, type, id_map, force:)
+        if tracking_id?(value)
+          return resolve_link(value, type, id_map, force: force)
+        end
+
+        value
+      end
+
+      def tracking_id?(id)
+        id.is_a?(String) && id.include?(":")
+      end
+
       def resolve_link(tracking_id, type, id_map, force:)
-        id = id_map[tracking_id]
-        if id == :new
+        if id_map.new?(type.to_s, tracking_id)
           if force
             invalid!(
               "#{type} #{tracking_id} was referenced but is also created by the current run.\n" \
@@ -111,7 +122,7 @@ module Kennel
           else
             nil # will be re-resolved after the linked object was created
           end
-        elsif id
+        elsif id = id_map.get(type.to_s, tracking_id)
           id
         else
           invalid! "Unable to find #{type} #{tracking_id} (does not exist and is not being created by the current run)"

data/lib/kennel/models/slo.rb

@@ -70,7 +70,7 @@ module Kennel
       def resolve_linked_tracking_ids!(id_map, **args)
         return unless as_json[:monitor_ids] # ignore_default can remove it
         as_json[:monitor_ids] = as_json[:monitor_ids].map do |id|
-          id.is_a?(String) ? (resolve_link(id, :monitor, id_map, **args) || id) : id
+          resolve(id, :monitor, id_map, **args) || id
         end
       end
 

data/lib/kennel/syncer.rb

@@ -122,7 +122,7 @@ module Kennel
     def calculate_diff
       @update = []
       @delete = []
-      @id_map = {}
+      @id_map = IdMap.new
 
       actual = Progress.progress("Downloading definitions") { download_definitions }
 
@@ -254,11 +254,19 @@ module Kennel
     end
 
     def populate_id_map(expected, actual)
+      expected.each do |e|
+        @id_map.set_new(e.class.api_resource, e.tracking_id)
+      end
+
       actual.each do |a|
         next unless tracking_id = a.fetch(:tracking_id)
-        @id_map[tracking_id] = a.fetch(:id)
+        next unless @id_map.get(a.fetch(:klass).api_resource, tracking_id)
+
+        @id_map.set(a.fetch(:klass).api_resource, tracking_id, a.fetch(:id))
+        if a[:klass].api_resource == "synthetics/tests"
+          @id_map.set(Kennel::Models::Monitor.api_resource, tracking_id, a.fetch(:monitor_id))
+        end
       end
-      expected.each { |e| @id_map[e.tracking_id] ||= :new }
     end
 
     def resolve_linked_tracking_ids!(list, force: false)

data/lib/kennel/tasks.rb

@@ -3,6 +3,7 @@ require "English"
 require "kennel"
 require "kennel/unmuted_alerts"
 require "kennel/importer"
+require "json"
 
 module Kennel
   module Tasks
@@ -89,7 +90,7 @@ namespace :kennel do
     Kennel::UnmutedAlerts.print(Kennel.send(:api), tag)
   end
 
-  desc "show monitors with no data by TAG, for example TAG=team:foo"
+  desc "show monitors with no data by TAG, for example TAG=team:foo [THRESHOLD_DAYS=7] [FORMAT=json]"
   task nodata: :environment do
     tag = ENV["TAG"] || Kennel::Tasks.abort("Call with TAG=foo:bar")
     monitors = Kennel.send(:api).list("monitor", monitor_tags: tag, group_states: "no data")
@@ -97,16 +98,45 @@ namespace :kennel do
     monitors.reject! { |m| m[:tags].include? "nodata:ignore" }
     if monitors.any?
       Kennel.err.puts <<~TEXT
-        This is a useful task to find monitors that have mis-spelled metrics or never received data at any time.
-        To ignore monitors with nodata, tag the monitor with "nodata:ignore"
+        To ignore monitors with expected nodata, tag it with "nodata:ignore"
 
       TEXT
     end
 
+    now = Time.now
     monitors.each do |m|
-      Kennel.out.puts m[:name]
-      Kennel.out.puts Kennel::Utils.path_to_url("/monitors/#{m[:id]}")
-      Kennel.out.puts
+      m[:days_in_no_data] =
+        if m[:overall_state_modified]
+          since = Date.parse(m[:overall_state_modified]).to_time
+          ((now - since) / (24 * 60 * 60)).to_i
+        else
+          999
+        end
+    end
+
+    if threshold = ENV["THRESHOLD_DAYS"]
+      monitors.select! { |m| m[:days_in_no_data] > Integer(threshold) }
+    end
+
+    monitors.each { |m| m[:url] = Kennel::Utils.path_to_url("/monitors/#{m[:id]}") }
+
+    if ENV["FORMAT"] == "json"
+      report = monitors.map do |m|
+        match = m[:message].to_s.match(/-- Managed by kennel (\S+:\S+) in (\S+), /) || []
+        m.slice(:url, :name, :tags, :days_in_no_data).merge(
+          kennel_tracking_id: match[1],
+          kennel_source: match[2]
+        )
+      end
+
+      Kennel.out.puts JSON.pretty_generate(report)
+    else
+      monitors.each do |m|
+        Kennel.out.puts m[:name]
+        Kennel.out.puts Kennel::Utils.path_to_url("/monitors/#{m[:id]}")
+        Kennel.out.puts "No data since #{m[:days_in_no_data]}d"
+        Kennel.out.puts
+      end
     end
   end
 

data/lib/kennel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Kennel
-  VERSION = "1.98.3"
+  VERSION = "1.102.0"
 end

data/lib/kennel.rb

@@ -9,6 +9,7 @@ require "kennel/version"
 require "kennel/utils"
 require "kennel/progress"
 require "kennel/syncer"
+require "kennel/id_map"
 require "kennel/api"
 require "kennel/github_reporter"
 require "kennel/subclass_tracking"
@@ -69,7 +70,8 @@ module Kennel
           path = "generated/#{part.tracking_id.tr("/", ":").sub(":", "/")}.json"
           used << File.dirname(path) # only 1 level of sub folders, so this is safe
           used << path
-          write_file_if_necessary(path, JSON.pretty_generate(part.as_json) << "\n")
+          payload = part.as_json.merge(api_resource: part.class.api_resource)
+          write_file_if_necessary(path, JSON.pretty_generate(payload) << "\n")
         end
 
         # deleting all is slow, so only delete the extras

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kennel
 version: !ruby/object:Gem::Version
-  version: 1.98.3
+  version: 1.102.0
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-04 00:00:00.000000000 Z
+date: 2021-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday
@@ -77,6 +77,7 @@ files:
 - lib/kennel/api.rb
 - lib/kennel/file_cache.rb
 - lib/kennel/github_reporter.rb
+- lib/kennel/id_map.rb
 - lib/kennel/importer.rb
 - lib/kennel/models/base.rb
 - lib/kennel/models/dashboard.rb