kennel 2.19.1 → 2.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: caaa8b242341641a57b4126d2eea6602bb2a70e445172048e6be7c4bf6ecf854
-  data.tar.gz: 7048bc44c5d705f3084fbd0630aa70dd217464eb768f870a3ced8790e0b6cdcd
+  metadata.gz: d41a632f2e488aacff6253cdb1b161d628d7522ffd4b7ce3e006455b912ca16d
+  data.tar.gz: 0ffe3c8a9ec7d828aa397460af8f06af307f14e4a9740ff0e1a1693bc6c65950
 SHA512:
-  metadata.gz: 294f05bd5277deba51fd580d0ba93a1690a24137f5f0babae710b3527e8ee89b2faa3d59cebb894ef7da1600200f0d58f0266424bc54822e587793d8b989a5e0
-  data.tar.gz: 4591dc02a72ed676301c2bed65f631096c5d0f02b682c1f1467ab26ed7d3e129a0b949d14d696bd982acedef187653a60eaece4ccfb78548fde44e78da98e00c
+  metadata.gz: 8151f10629556f4b9dbad8b11c69e032fca8ebbd00a50e888482a2bb37659ce70028a8740606a90745de3b3808967165b4cc10c1faeac2ccce707bfc57cb8811
+  data.tar.gz: 6682f0c4815aacd911464f92c94f376110e52219c465f6f482dfbb85399b0ec4ea2b5d33d78ae394f753a04ef959b6b766325bd4bbc2f0b461dd172a24ed1349

data/Readme.md

@@ -444,6 +444,9 @@ Run `rake kennel:alerts TAG=service:my-service` to see all un-muted alerts for a
 Use `KNOWN=foo@bar.com,baz@bar.com` to exempt mentions that are not returned by the API.
 Use `KNOWN_RANDOM=@sns-foo,@sns-bar` to ignore for example SNS handles that are randomly invalid in the API.
 
+### Validating planned changes
+`rake kennel:validate_plan` validates planned monitor and dashboard changes against the Datadog API.
+
 ### Grepping through all of datadog
 ```Bash
 rake kennel:dump > tmp/dump

data/lib/kennel/tasks/dump.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+namespace :kennel do
+  desc "Dump ALL of datadog config as raw json ... useful for grep/search [TYPE=slo|monitor|dashboard]"
+  task dump: :environment do
+    resources =
+      if (type = ENV["TYPE"])
+        [type]
+      else
+        Kennel::Models::Record.api_resource_map.keys
+      end
+    api = Kennel::Api.new
+    list = nil
+    first = true
+
+    Kennel.out.puts "["
+    resources.each do |resource|
+      Kennel::Progress.progress("Downloading #{resource}") do
+        list = api.list(resource)
+        api.fill_details!(resource, list) if resource == "dashboard"
+      end
+      list.each do |r|
+        r[:api_resource] = resource
+        if first
+          first = false
+        else
+          Kennel.out.puts ","
+        end
+        Kennel.out.print JSON.pretty_generate(r)
+      end
+    end
+    Kennel.out.puts "\n]"
+  end
+
+  desc "Find items from dump by pattern DUMP= PATTERN= [URLS=true]"
+  task dump_grep: :environment do
+    file = ENV.fetch("DUMP")
+    pattern = Regexp.new ENV.fetch("PATTERN")
+    items = File.read(file)[2..-2].gsub("},\n{", "}--SPLIT--{").split("--SPLIT--")
+    models = Kennel::Models::Record.api_resource_map
+    found = items.grep(pattern)
+    exit 1 if found.empty?
+    found.each do |resource|
+      if ENV["URLS"]
+        parsed = JSON.parse(resource)
+        url = models[parsed.fetch("api_resource")].url(parsed.fetch("id"))
+        title = parsed["title"] || parsed["name"]
+        Kennel.out.puts "#{url} # #{title}"
+      else
+        Kennel.out.puts resource
+      end
+    end
+  end
+end

data/lib/kennel/tasks/import.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+namespace :kennel do
+  desc "Convert existing resources to copy-pasteable definitions to import existing resources (call with URL= or call with RESOURCE= and ID=)"
+  task import: :environment do
+    if (id = ENV["ID"]) && (resource = ENV["RESOURCE"])
+      id = Integer(id) if id =~ /^\d+$/
+    elsif (url = ENV["URL"])
+      resource, id = Kennel::Models::Record.parse_any_url(url) || Kennel::Tasks.abort("Unable to parse url")
+    else
+      possible_resources = Kennel::Models::Record.subclasses.map(&:api_resource)
+      Kennel::Tasks.abort("Call with URL= or call with RESOURCE=#{possible_resources.join(" or ")} and ID=")
+    end
+
+    Kennel.out.puts Kennel::Importer.new(Kennel::Api.new).import(resource, id)
+  end
+end

data/lib/kennel/tasks/nodata.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+namespace :kennel do
+  desc "show monitors with no data by TAG, for example TAG=team:foo [THRESHOLD_DAYS=7] [FORMAT=json]"
+  task nodata: :environment do
+    tag = ENV["TAG"] || Kennel::Tasks.abort("Call with TAG=foo:bar")
+    monitors = Kennel::Api.new.list("monitor", monitor_tags: tag, group_states: "no data")
+    monitors.select! { |m| m[:overall_state] == "No Data" }
+    monitors.reject! { |m| m[:tags].include? "nodata:ignore" }
+    if monitors.any?
+      Kennel.err.puts <<~TEXT
+        To ignore monitors with expected nodata, tag it with "nodata:ignore"
+
+      TEXT
+    end
+
+    now = Time.now
+    monitors.each do |m|
+      m[:days_in_no_data] =
+        if m[:overall_state_modified]
+          since = Date.parse(m[:overall_state_modified]).to_time
+          ((now - since) / (24 * 60 * 60)).to_i
+        else
+          999
+        end
+    end
+
+    if (threshold = ENV["THRESHOLD_DAYS"])
+      monitors.select! { |m| m[:days_in_no_data] > Integer(threshold) }
+    end
+
+    monitors.each { |m| m[:url] = Kennel::Utils.path_to_url("/monitors/#{m[:id]}") }
+
+    if ENV["FORMAT"] == "json"
+      report = monitors.map do |m|
+        match = m[:message].to_s.match(/-- #{Regexp.escape(Kennel::Models::Record::MARKER_TEXT)} (\S+:\S+) in (\S+), /) || []
+        m.slice(:url, :name, :tags, :days_in_no_data).merge(
+          kennel_tracking_id: match[1],
+          kennel_source: match[2]
+        )
+      end
+
+      Kennel.out.puts JSON.pretty_generate(report)
+    else
+      monitors.each do |m|
+        Kennel.out.puts m[:name]
+        Kennel.out.puts Kennel::Utils.path_to_url("/monitors/#{m[:id]}")
+        Kennel.out.puts "No data since #{m[:days_in_no_data]}d"
+        Kennel.out.puts
+      end
+    end
+  end
+end

data/lib/kennel/tasks/tracking_id.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+namespace :kennel do
+  desc "Resolve given id to kennel tracking-id RESOURCE= ID="
+  task tracking_id: "kennel:environment" do
+    resource = ENV.fetch("RESOURCE")
+    id = ENV.fetch("ID")
+    klass =
+      Kennel::Models::Record.subclasses.detect { |s| s.api_resource == resource } ||
+      raise("resource #{resource} not know")
+    object = Kennel::Api.new.show(resource, id)
+    Kennel.out.puts klass.parse_tracking_id(object)
+  end
+end

data/lib/kennel/tasks/validate_mentions.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+namespace :kennel do
+  desc "Verify that all used monitor  mentions are valid"
+  task validate_mentions: :environment do
+    known = []
+
+    # @slack- @team- @webhook- @sns- user-emails
+    known += Kennel::Api.new.send(:request, :get, "/api/v2/notifications/handles?group_limit=99999")
+      .fetch(:data)
+      .flat_map { |d| d.dig(:attributes, :handles) }
+      .map { |v| v.fetch(:value) }
+
+    # group emails or other 1-off things we know are valid
+    manual = ENV["KNOWN"].to_s.split(",")
+    dupes = (manual & known)
+    Kennel::Tasks.abort "KNOWN=#{dupes.join(",")} values are already known and should be removed" if dupes.any?
+    known += manual
+
+    # @sns- handles are randomly invalid so we need to ignore them without checking if the ignore is needed
+    # https://help.datadoghq.com/hc/en-us/requests/2310423
+    known += ENV["KNOWN_RANDOM"].to_s.split(",")
+
+    bad = []
+    Dir["generated/**/*.json"].each do |f|
+      next unless (message = JSON.parse(File.read(f))["message"])
+      used = message
+        .scan(/(?:^|\s)(@[^\s{,'"]+)/)
+        .flatten(1)
+        .grep(/^@.*@|^@.*-/) # ignore @here etc handles ... datadog uses @foo@bar.com for emails and @foo-bar for integrations
+      (used - known).each { |v| bad << [f, v] }
+    end
+
+    if bad.any?
+      url = Kennel::Utils.path_to_url "/account/settings"
+      Kennel.err.puts "Invalid mentions found, either ignore them by adding to `KNOWN` env var or add them via #{url}"
+      bad.each { |f, v| Kennel.err.puts "Invalid mention #{v} in monitor message of #{f}" }
+      Kennel::Tasks.abort ENV["KNOWN_WARNING"]
+    end
+  end
+end

data/lib/kennel/tasks/validate_plan.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+module Kennel
+  module ValidatePlan
+    class MonitorValidator
+      COSMETIC_FIELDS = ["name", "message", "tags"].freeze
+
+      def initialize(item)
+        @item = item
+      end
+
+      def validate(api)
+        data = @item.expected.as_json
+
+        # ignore unresolved ids from yet to be created monitors
+        return nil if ["composite", "slo alert"].include?(data[:type]) && data[:query].include?("%")
+
+        api.send(:request, :post, "/api/v1/monitor/validate", body: data)
+        nil
+      rescue StandardError => e
+        "#{Kennel::Console.color(:yellow, "#{@item.api_resource} #{@item.tracking_id}:")}\n#{e.message}"
+      end
+    end
+
+    class DashboardValidator
+      COSMETIC_FIELDS = ["title", "description", "tags"].freeze
+
+      def initialize(item)
+        @item = item
+      end
+
+      # datadog does not offer a validation api for dashboards,
+      # so we insert an invalid widget at the end and see if that is the invalid widget it complains about
+      # this will break if they ever start from the back or return errors for everything that is invalid
+      #
+      # we do not need to worry about unresolved ids because:
+      # - alert_graph widgets allows kennel style ids
+      # - slo widgets allow kennel style ids
+      # - uptime widgets allow kennel style ids
+      def validate(api)
+        json = @item.expected.as_json
+        json = Marshal.load(Marshal.dump(json))
+
+        # add a semi-valid (does not fail immediately on missing definition) widget still blocks the request
+        placeholder = "invalid_metric_do_not_update"
+        json.fetch(:widgets) << {
+          definition: {
+            type: "timeseries", requests: [{
+              response_format: "timeseries",
+              queries: [{ data_source: "metrics", name: "restarts", query: placeholder }]
+            }]
+          },
+          layout: { x: 0, y: 0, height: 0, width: 0 } # needed for `layout_type: free` and valid for all
+        }
+
+        begin
+          if @item.class::TYPE == :update
+            api.update("dashboard", @item.actual.fetch(:id), json)
+          else
+            api.create("dashboard", json)
+          end
+          raise "Dashboard validation should have failed, live dashboard was update/created by accident"
+        rescue StandardError => e
+          # parse the JSON in the error message and see if there is anything except our error
+          raise "Unreadable error format: #{e.message}" unless (json = e.message[/^\{"errors":.*}$/m])
+          data =
+            begin
+              JSON.parse(json)
+            rescue JSON::ParserError
+              raise "Unreadable error format: #{json}"
+            end
+          raise "Unreadable error format: #{data}" unless (errors = data["errors"]) # uncovered
+          return if errors.size == 1 && errors.all? { |m| m.include?("unable to parse #{placeholder}") }
+          "#{@item.tracking_id}: #{e.message}"
+        end
+      end
+    end
+
+    VALIDATORS = {
+      "monitor" => MonitorValidator,
+      "dashboard" => DashboardValidator
+    }.freeze
+
+    def self.validate(plan)
+      changes = (plan.creates + plan.updates)
+
+      validators = changes.filter_map do |item|
+        next unless (validator = VALIDATORS[item.api_resource]&.new(item))
+
+        if item.class::TYPE == :update
+          # ignore if nothing can break
+          modified_fields = item.diff.map { |_, f, *| f }
+          next nil if modified_fields.all? { |f| validator.class::COSMETIC_FIELDS.include?(f) }
+        end
+
+        validator
+      end
+
+      api = Kennel::Api.new
+      errors = validators.filter_map { |v| v.validate(api) }
+      return if errors.empty?
+
+      abort "#{Kennel::Console.color(:red, "#{errors.size} validation(s) failed:")}\n#{errors.join("\n")}"
+    end
+  end
+end
+
+namespace :kennel do
+  desc "Validate planned changes against the Datadog API [PROJECT=]"
+  task "validate_plan" => "kennel:environment" do
+    kennel = Kennel::Tasks.kennel
+    kennel.preload
+    Kennel::ValidatePlan.validate(kennel.plan)
+  end
+end

data/lib/kennel/tasks.rb

@@ -5,6 +5,8 @@ require "kennel/unmuted_alerts"
 require "kennel/importer"
 require "json"
 
+Dir.children("#{__dir__}/tasks").each { |f| require_relative "tasks/#{File.basename(f, ".rb")}" }
+
 module Kennel
   module Tasks
     class << self
@@ -71,49 +73,6 @@ namespace :kennel do
     Kennel::Tasks.abort "Error during diffing" unless $CHILD_STATUS.success?
   end
 
-  # ideally do this on every run, but it's slow (~1.5s) and brittle
-  # (might not find all via the regex + might find false-positives because random emails can also be sent to)
-  # https://help.datadoghq.com/hc/en-us/requests/254114 for automatic validation
-  # /monitor/notifications has users+slack+sns but not @team- and @webhook-
-  # got a support ticket open to get sns into api/v2 too
-  desc "Verify that all used monitor  mentions are valid"
-  task validate_mentions: :environment do
-    known = []
-
-    # @slack- @team- @webhook- @sns- user-emails
-    known += Kennel::Api.new.send(:request, :get, "/api/v2/notifications/handles?group_limit=99999")
-      .fetch(:data)
-      .flat_map { |d| d.dig(:attributes, :handles) }
-      .map { |v| v.fetch(:value) }
-
-    # group emails or other 1-off things we know are valid
-    manual = ENV["KNOWN"].to_s.split(",")
-    dupes = (manual & known)
-    Kennel::Tasks.abort "KNOWN=#{dupes.join(",")} values are already known and should be removed" if dupes.any?
-    known += manual
-
-    # @sns- handles are randomly invalid so we need to ignore them without checking if the ignore is needed
-    # https://help.datadoghq.com/hc/en-us/requests/2310423
-    known += ENV["KNOWN_RANDOM"].to_s.split(",")
-
-    bad = []
-    Dir["generated/**/*.json"].each do |f|
-      next unless (message = JSON.parse(File.read(f))["message"])
-      used = message
-        .scan(/(?:^|\s)(@[^\s{,'"]+)/)
-        .flatten(1)
-        .grep(/^@.*@|^@.*-/) # ignore @here etc handles ... datadog uses @foo@bar.com for emails and @foo-bar for integrations
-      (used - known).each { |v| bad << [f, v] }
-    end
-
-    if bad.any?
-      url = Kennel::Utils.path_to_url "/account/settings"
-      Kennel.err.puts "Invalid mentions found, either ignore them by adding to `KNOWN` env var or add them via #{url}"
-      bad.each { |f, v| Kennel.err.puts "Invalid mention #{v} in monitor message of #{f}" }
-      Kennel::Tasks.abort ENV["KNOWN_WARNING"]
-    end
-  end
-
   desc "store definitions in generated/"
   task generate: :environment do
     Kennel::Tasks.kennel.generate
@@ -146,132 +105,6 @@ namespace :kennel do
     Kennel::UnmutedAlerts.print(Kennel::Api.new, tag)
   end
 
-  desc "show monitors with no data by TAG, for example TAG=team:foo [THRESHOLD_DAYS=7] [FORMAT=json]"
-  task nodata: :environment do
-    tag = ENV["TAG"] || Kennel::Tasks.abort("Call with TAG=foo:bar")
-    monitors = Kennel::Api.new.list("monitor", monitor_tags: tag, group_states: "no data")
-    monitors.select! { |m| m[:overall_state] == "No Data" }
-    monitors.reject! { |m| m[:tags].include? "nodata:ignore" }
-    if monitors.any?
-      Kennel.err.puts <<~TEXT
-        To ignore monitors with expected nodata, tag it with "nodata:ignore"
-
-      TEXT
-    end
-
-    now = Time.now
-    monitors.each do |m|
-      m[:days_in_no_data] =
-        if m[:overall_state_modified]
-          since = Date.parse(m[:overall_state_modified]).to_time
-          ((now - since) / (24 * 60 * 60)).to_i
-        else
-          999
-        end
-    end
-
-    if (threshold = ENV["THRESHOLD_DAYS"])
-      monitors.select! { |m| m[:days_in_no_data] > Integer(threshold) }
-    end
-
-    monitors.each { |m| m[:url] = Kennel::Utils.path_to_url("/monitors/#{m[:id]}") }
-
-    if ENV["FORMAT"] == "json"
-      report = monitors.map do |m|
-        match = m[:message].to_s.match(/-- #{Regexp.escape(Kennel::Models::Record::MARKER_TEXT)} (\S+:\S+) in (\S+), /) || []
-        m.slice(:url, :name, :tags, :days_in_no_data).merge(
-          kennel_tracking_id: match[1],
-          kennel_source: match[2]
-        )
-      end
-
-      Kennel.out.puts JSON.pretty_generate(report)
-    else
-      monitors.each do |m|
-        Kennel.out.puts m[:name]
-        Kennel.out.puts Kennel::Utils.path_to_url("/monitors/#{m[:id]}")
-        Kennel.out.puts "No data since #{m[:days_in_no_data]}d"
-        Kennel.out.puts
-      end
-    end
-  end
-
-  desc "Convert existing resources to copy-pasteable definitions to import existing resources (call with URL= or call with RESOURCE= and ID=)"
-  task import: :environment do
-    if (id = ENV["ID"]) && (resource = ENV["RESOURCE"])
-      id = Integer(id) if id =~ /^\d+$/ # dashboards can have alphanumeric ids
-    elsif (url = ENV["URL"])
-      resource, id = Kennel::Models::Record.parse_any_url(url) || Kennel::Tasks.abort("Unable to parse url")
-    else
-      possible_resources = Kennel::Models::Record.subclasses.map(&:api_resource)
-      Kennel::Tasks.abort("Call with URL= or call with RESOURCE=#{possible_resources.join(" or ")} and ID=")
-    end
-
-    Kennel.out.puts Kennel::Importer.new(Kennel::Api.new).import(resource, id)
-  end
-
-  desc "Dump ALL of datadog config as raw json ... useful for grep/search [TYPE=slo|monitor|dashboard]"
-  task dump: :environment do
-    resources =
-      if (type = ENV["TYPE"])
-        [type]
-      else
-        Kennel::Models::Record.api_resource_map.keys
-      end
-    api = Kennel::Api.new
-    list = nil
-    first = true
-
-    Kennel.out.puts "["
-    resources.each do |resource|
-      Kennel::Progress.progress("Downloading #{resource}") do
-        list = api.list(resource)
-        api.fill_details!(resource, list) if resource == "dashboard"
-      end
-      list.each do |r|
-        r[:api_resource] = resource
-        if first
-          first = false
-        else
-          Kennel.out.puts ","
-        end
-        Kennel.out.print JSON.pretty_generate(r)
-      end
-    end
-    Kennel.out.puts "\n]"
-  end
-
-  desc "Find items from dump by pattern DUMP= PATTERN= [URLS=true]"
-  task dump_grep: :environment do
-    file = ENV.fetch("DUMP")
-    pattern = Regexp.new ENV.fetch("PATTERN")
-    items = File.read(file)[2..-2].gsub("},\n{", "}--SPLIT--{").split("--SPLIT--")
-    models = Kennel::Models::Record.api_resource_map
-    found = items.grep(pattern)
-    exit 1 if found.empty?
-    found.each do |resource|
-      if ENV["URLS"]
-        parsed = JSON.parse(resource)
-        url = models[parsed.fetch("api_resource")].url(parsed.fetch("id"))
-        title = parsed["title"] || parsed["name"]
-        Kennel.out.puts "#{url} # #{title}"
-      else
-        Kennel.out.puts resource
-      end
-    end
-  end
-
-  desc "Resolve given id to kennel tracking-id RESOURCE= ID="
-  task tracking_id: "kennel:environment" do
-    resource = ENV.fetch("RESOURCE")
-    id = ENV.fetch("ID")
-    klass =
-      Kennel::Models::Record.subclasses.detect { |s| s.api_resource == resource } ||
-      raise("resource #{resource} not know")
-    object = Kennel::Api.new.show(resource, id)
-    Kennel.out.puts klass.parse_tracking_id(object)
-  end
-
   task :environment do
     Kennel::Tasks.load_environment
   end

data/lib/kennel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Kennel
-  VERSION = "2.19.1"
+  VERSION = "2.20.0"
 end

data/template/Readme.md

@@ -426,6 +426,9 @@ Run `rake kennel:alerts TAG=service:my-service` to see all un-muted alerts for a
 Use `KNOWN=foo@bar.com,baz@bar.com` to exempt mentions that are not returned by the API.
 Use `KNOWN_RANDOM=@sns-foo,@sns-bar` to ignore for example SNS handles that are randomly invalid in the API.
 
+### Validating planned changes
+`rake kennel:validate_plan` validates planned monitor and dashboard changes against the Datadog API.
+
 ### Grepping through all of datadog
 ```Bash
 rake kennel:dump > tmp/dump

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kennel
 version: !ruby/object:Gem::Version
-  version: 2.19.1
+  version: 2.20.0
 platform: ruby
 authors:
 - Michael Grosser
@@ -117,6 +117,12 @@ files:
 - lib/kennel/syncer/types.rb
 - lib/kennel/tags_validation.rb
 - lib/kennel/tasks.rb
+- lib/kennel/tasks/dump.rb
+- lib/kennel/tasks/import.rb
+- lib/kennel/tasks/nodata.rb
+- lib/kennel/tasks/tracking_id.rb
+- lib/kennel/tasks/validate_mentions.rb
+- lib/kennel/tasks/validate_plan.rb
 - lib/kennel/template_variables.rb
 - lib/kennel/unmuted_alerts.rb
 - lib/kennel/utils.rb