kennel 1.74.0

data/lib/kennel/tasks.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+require "English"
+require "kennel"
+require "kennel/unmuted_alerts"
+require "kennel/importer"
+
+module Kennel
+  module Tasks
+    class << self
+      def abort(message = nil)
+        Kennel.err.puts message if message
+        raise SystemExit.new(1), message
+      end
+    end
+  end
+end
+
+namespace :kennel do
+  desc "Ensure there are no uncommited changes that would be hidden from PR reviewers"
+  task no_diff: :generate do
+    result = `git status --porcelain generated/`.strip
+    Kennel::Tasks.abort "Diff found:\n#{result}\nrun `rake generate` and commit the diff to fix" unless result == ""
+    Kennel::Tasks.abort "Error during diffing" unless $CHILD_STATUS.success?
+  end
+
+  # ideally do this on every run, but it's slow (~1.5s) and brittle (might not find all + might find false-positives)
+  # https://help.datadoghq.com/hc/en-us/requests/254114 for automatic validation
+  desc "Verify that all used monitor  mentions are valid"
+  task validate_mentions: :environment do
+    known = Kennel.send(:api)
+      .send(:request, :get, "/monitor/notifications")
+      .fetch(:handles)
+      .values
+      .flatten(1)
+      .map { |v| v.fetch(:value) }
+
+    known += ENV["KNOWN"].to_s.split(",")
+
+    bad = []
+    Dir["generated/**/*.json"].each do |f|
+      next unless message = JSON.parse(File.read(f))["message"]
+      used = message.scan(/\s(@[^\s{,'"]+)/).flatten(1)
+        .grep(/^@.*@|^@.*-/) # ignore @here etc handles ... datadog uses @foo@bar.com for emails and @foo-bar for integrations
+      (used - known).each { |v| bad << [f, v] }
+    end
+
+    if bad.any?
+      url = Kennel::Utils.path_to_url "/account/settings"
+      puts "Invalid mentions found, either ignore them by adding to `KNOWN` env var or add them via #{url}"
+      bad.each { |f, v| puts "Invalid mention #{v} in monitor message of #{f}" }
+      Kennel::Tasks.abort
+    end
+  end
+
+  desc "generate local definitions"
+  task generate: :environment do
+    Kennel.generate
+  end
+
+  # also generate parts so users see and commit updated generated automatically
+  desc "show planned datadog changes (scope with PROJECT=name)"
+  task plan: :generate do
+    Kennel.plan
+  end
+
+  desc "update datadog (scope with PROJECT=name)"
+  task update_datadog: :environment do
+    Kennel.update
+  end
+
+  desc "update if this is a push to the default branch, otherwise plan"
+  task :travis do
+    on_default_branch = (ENV["TRAVIS_BRANCH"] == (ENV["DEFAULT_BRANCH"] || "master"))
+    is_push = (ENV["TRAVIS_PULL_REQUEST"] == "false")
+    task_name =
+      if on_default_branch && is_push
+        "kennel:update_datadog"
+      else
+        "kennel:plan" # show plan in travis logs
+      end
+
+    Rake::Task[task_name].invoke
+  end
+
+  desc "show unmuted alerts filtered by TAG, for example TAG=team:foo"
+  task alerts: :environment do
+    tag = ENV["TAG"] || Kennel::Tasks.abort("Call with TAG=foo:bar")
+    Kennel::UnmutedAlerts.print(Kennel.send(:api), tag)
+  end
+
+  desc "show monitors with no data by TAG, for example TAG=team:foo"
+  task nodata: :environment do
+    tag = ENV["TAG"] || Kennel::Tasks.abort("Call with TAG=foo:bar")
+    monitors = Kennel.send(:api).list("monitor", monitor_tags: tag, group_states: "no data")
+    monitors.select! { |m| m[:overall_state] == "No Data" }
+    monitors.reject! { |m| m[:tags].include? "nodata:ignore" }
+    if monitors.any?
+      Kennel.err.puts <<~TEXT
+        This is a useful task to find monitors that have mis-spelled metrics or never received data at any time.
+        To ignore monitors with nodata, tag the monitor with "nodata:ignore"
+
+      TEXT
+    end
+
+    monitors.each do |m|
+      Kennel.out.puts m[:name]
+      Kennel.out.puts Kennel::Utils.path_to_url("/monitors/#{m[:id]}")
+      Kennel.out.puts
+    end
+  end
+
+  desc "Convert existing resources to copy-pasteable definitions to import existing resources (call with URL= or call with RESOURCE= and ID=)"
+  task import: :environment do
+    if (id = ENV["ID"]) && (resource = ENV["RESOURCE"])
+      id = Integer(id) if id =~ /^\d+$/ # dashboards can have alphanumeric ids
+    elsif (url = ENV["URL"])
+      resource, id = Kennel::Models::Record.parse_any_url(url) || Kennel::Tasks.abort("Unable to parse url")
+    else
+      possible_resources = Kennel::Models::Record.subclasses.map(&:api_resource)
+      Kennel::Tasks.abort("Call with URL= or call with RESOURCE=#{possible_resources.join(" or ")} and ID=")
+    end
+
+    Kennel.out.puts Kennel::Importer.new(Kennel.send(:api)).import(resource, id)
+  end
+
+  desc "Dump ALL of datadog config as raw json ... useful for grep/search TYPE=slo|monitor|dashboard"
+  task dump: :environment do
+    Kennel.send(:api).list(ENV.fetch("TYPE")).each do |r|
+      Kennel.out.puts JSON.pretty_generate(r)
+    end
+  end
+
+  task :environment do
+    require "kennel"
+    gem "dotenv"
+    require "dotenv"
+    source = ".env"
+
+    # warn when users have things like DATADOG_TOKEN already set and it will not be loaded from .env
+    unless ENV["KENNEL_SILENCE_UPDATED_ENV"]
+      updated = Dotenv.parse(source).select { |k, v| ENV[k] && ENV[k] != v }
+      warn "Environment variables #{updated.keys.join(", ")} need to be unset to be sourced from #{source}" if updated.any?
+    end
+
+    Dotenv.load(source)
+  end
+end

data/lib/kennel/template_variables.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+module Kennel
+  module TemplateVariables
+    def self.included(base)
+      base.settings :template_variables
+      base.defaults(template_variables: -> { [] })
+    end
+
+    private
+
+    def render_template_variables
+      (template_variables || []).map do |v|
+        v.is_a?(String) ? { default: "*", prefix: v, name: v } : v
+      end
+    end
+
+    # check for queries that do not use the variables and would be misleading
+    # TODO: do the same check for apm_query and their group_by
+    def validate_template_variables(data, key)
+      variables = (data[:template_variables] || []).map { |v| "$#{v.fetch(:name)}" }
+      queries = data[key].flat_map do |widget|
+        ([widget] + (widget.dig(:definition, :widgets) || [])).flat_map { |w| widget_queries(w) }
+      end.compact
+      bad = queries.grep_v(/(#{variables.map { |v| Regexp.escape(v) }.join("|")})\b/)
+      if bad.any?
+        invalid!(
+          "queries #{bad.join(", ")} must use the template variables #{variables.join(", ")}\n" \
+          "If that is not possible, add `validate: -> { false } # query foo in bar does not have baz tag`"
+        )
+      end
+    end
+
+    def widget_queries(widget)
+      requests = widget.dig(:definition, :requests) || []
+      (requests.is_a?(Hash) ? requests.values : requests).map { |r| r[:q] } # hostmap widgets have hash requests
+    end
+  end
+end

data/lib/kennel/unmuted_alerts.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+require "kennel"
+
+# Show Alerts that are not muted and their alerting scopes
+module Kennel
+  class UnmutedAlerts
+    COLORS = {
+      "Alert" => :red,
+      "Warn" => :yellow,
+      "No Data" => :cyan
+    }.freeze
+
+    class << self
+      def print(api, tag)
+        monitors = filtered_monitors(api, tag)
+        if monitors.empty?
+          Kennel.out.puts "No unmuted alerts found"
+        else
+          monitors.each do |m|
+            Kennel.out.puts m[:name]
+            Kennel.out.puts Utils.path_to_url("/monitors/#{m[:id]}")
+            m[:state][:groups].each do |g|
+              color = COLORS[g[:status]] || :default
+              since = "\t#{time_since(g[:last_triggered_ts])}"
+              Kennel.out.puts "#{Kennel::Utils.color(color, g[:status])}\t#{g[:name]}#{since}"
+            end
+            Kennel.out.puts
+          end
+        end
+      end
+
+      private
+
+      # sort pod3 before pod11
+      def sort_groups!(monitor)
+        groups = monitor[:state][:groups].values
+        groups.sort_by! { |g| g[:name].to_s.split(",").map { |w| Utils.natural_order(w) } }
+        monitor[:state][:groups] = groups
+      end
+
+      def time_since(t)
+        diff = Time.now.to_i - Integer(t)
+        "%02d:%02d:%02d" % [diff / 3600, diff / 60 % 60, diff % 60]
+      end
+
+      def filtered_monitors(api, tag)
+        # Download all monitors with given tag
+        monitors = Progress.progress("Downloading") do
+          api.list("monitor", monitor_tags: tag, group_states: "all", with_downtimes: "true")
+        end
+
+        raise "No monitors for #{tag} found, check your spelling" if monitors.empty?
+
+        # only keep monitors that are alerting
+        monitors.reject! { |m| m[:overall_state] == "OK" }
+
+        # only keep monitors that are not completely silenced
+        monitors.reject! { |m| m[:options][:silenced].key?(:*) }
+
+        # only keep groups that are alerting
+        monitors.each { |m| m[:state][:groups].reject! { |_, g| g[:status] == "OK" || g[:status] == "Ignored" } }
+
+        # only keep alerting groups that are not silenced
+        monitors.each do |m|
+          silenced = m[:options][:silenced].keys.map { |k| k.to_s.split(",") }
+          m[:state][:groups].select! do |k, _|
+            scope = k.to_s.split(",")
+            silenced.none? { |s| (s - scope).empty? }
+          end
+        end
+
+        # only keep monitors that are not covered by a downtime
+        monitors.each do |m|
+          next unless m[:matching_downtimes]
+          downtime_groups = m[:matching_downtimes].select { |d| d[:active] }.flat_map { |d| d[:groups] }
+          m[:state][:groups].reject! do |k, _|
+            downtime_groups.include?(k.to_s)
+          end
+        end
+
+        # only keep monitors with alerting groups
+        monitors.select! { |m| m[:state][:groups].any? }
+
+        # sort group alerts
+        monitors.each { |m| sort_groups!(m) }
+      end
+    end
+  end
+end

data/lib/kennel/utils.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+module Kennel
+  module Utils
+    COLORS = { red: 31, green: 32, yellow: 33, cyan: 36, magenta: 35, default: 0 }.freeze
+
+    class TeeIO < IO
+      def initialize(ios)
+        @ios = ios
+      end
+
+      def write(string)
+        @ios.each { |io| io.write string }
+      end
+    end
+
+    class << self
+      def snake_case(string)
+        string
+          .gsub(/::/, "_") # Foo::Bar -> foo_bar
+          .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2') # FOOBar -> foo_bar
+          .gsub(/([a-z\d])([A-Z])/, '\1_\2') # fooBar -> foo_bar
+          .tr("-", "_") # foo-bar -> foo_bar
+          .downcase
+      end
+
+      # simplified version of https://apidock.com/rails/ActiveSupport/Inflector/parameterize
+      def parameterize(string)
+        string
+          .downcase
+          .gsub(/[^a-z0-9\-_]+/, "-") # remove unsupported
+          .gsub(/-{2,}/, "-") # remove duplicates
+          .gsub(/^-|-$/, "") # remove leading/trailing
+      end
+
+      def presence(value)
+        value.nil? || value.empty? ? nil : value
+      end
+
+      def ask(question)
+        Kennel.err.printf color(:red, "#{question} -  press 'y' to continue: ")
+        begin
+          STDIN.gets.chomp == "y"
+        rescue Interrupt # do not show a backtrace if user decides to Ctrl+C here
+          Kennel.err.print "\n"
+          exit 1
+        end
+      end
+
+      def color(color, text)
+        "\e[#{COLORS.fetch(color)}m#{text}\e[0m"
+      end
+
+      def strip_shell_control(text)
+        text.gsub(/\e\[\d+m(.*?)\e\[0m/, "\\1").gsub(/.#{Regexp.escape("\b")}/, "")
+      end
+
+      def capture_stdout
+        old = Kennel.out
+        Kennel.out = StringIO.new
+        yield
+        Kennel.out.string
+      ensure
+        Kennel.out = old
+      end
+
+      def capture_stderr
+        old = Kennel.err
+        Kennel.err = StringIO.new
+        yield
+        Kennel.err.string
+      ensure
+        Kennel.err = old
+      end
+
+      def tee_output
+        old_stdout = Kennel.out
+        old_stderr = Kennel.err
+        capture = StringIO.new
+        Kennel.out = TeeIO.new([capture, Kennel.out])
+        Kennel.err = TeeIO.new([capture, Kennel.err])
+        yield
+        capture.string
+      ensure
+        Kennel.out = old_stdout
+        Kennel.err = old_stderr
+      end
+
+      def capture_sh(command)
+        result = `#{command} 2>&1`
+        raise "Command failed:\n#{command}\n#{result}" unless $CHILD_STATUS.success?
+        result
+      end
+
+      def path_to_url(path)
+        if subdomain = ENV["DATADOG_SUBDOMAIN"]
+          "https://#{subdomain}.datadoghq.com#{path}"
+        else
+          path
+        end
+      end
+
+      def parallel(items, max: 10)
+        threads = [items.size, max].min
+        work = items.each_with_index.to_a
+        done = Array.new(items.size)
+        workers = Array.new(threads).map do
+          Thread.new do
+            loop do
+              item, i = work.pop
+              break unless i
+              done[i] =
+                begin
+                  yield item
+                rescue StandardError => e
+                  work.clear
+                  e
+                end
+            end
+          end
+        end
+        workers.each(&:join)
+        done.each { |d| raise d if d.is_a?(StandardError) }
+      end
+
+      def natural_order(name)
+        name.split(/(\d+)/).each_with_index.map { |x, i| i.odd? ? x.to_i : x }
+      end
+
+      def retry(*errors, times:)
+        yield
+      rescue *errors => e
+        times -= 1
+        raise if times < 0
+        Kennel.err.puts "Error #{e}, #{times} retries left"
+        retry
+      end
+
+      # https://stackoverflow.com/questions/20235206/ruby-get-all-keys-in-a-hash-including-sub-keys/53876255#53876255
+      def all_keys(items)
+        case items
+        when Hash then items.keys + items.values.flat_map { |v| all_keys(v) }
+        when Array then items.flat_map { |i| all_keys(i) }
+        else []
+        end
+      end
+
+      # TODO: use awesome-print or similar, but it has too many monkey-patches
+      # https://github.com/amazing-print/amazing_print/issues/36
+      def pretty_inspect(object)
+        string = object.inspect
+        string.gsub!(/:([a-z_]+)=>/, "\\1: ")
+        10.times do
+          string.gsub!(/{(\S.*?\S)}/, "{ \\1 }") || break
+        end
+        string
+      end
+    end
+  end
+end

data/lib/kennel/version.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+module Kennel
+  VERSION = "1.74.0"
+end

data/template/Readme.md

@@ -0,0 +1,205 @@
+# Kennel
+
+![](github/cage.jpg?raw=true)
+
+Manage datadog monitors/dashboards/slos as code
+
+ - Documented, reusable, and searchable
+ - Changes are PR reviewed and auditable
+ - Updating shows diff before applying
+ - Automated import of existing monitors/dashboards/slos
+
+![](github/screen.png?raw=true)
+
+## Structure
+
+ - `projects/` monitors/dashboards/etc scoped by project
+ - `teams/` team definitions
+ - `parts/` monitors/dashboards/etc that are used by multiple projects
+ - `generated/` projects as json, to show current state and proposed changes in PRs
+
+## Workflows
+
+### Setup
+ - clone the repo
+ - `gem install bundler && bundle install`
+ - `cp .env.example .env`
+ - open [Datadog API Settings](https://app.datadoghq.com/account/settings#api)
+ - copy any `API Key` and add it to `.env` as `DATADOG_API_KEY`
+ - find or create (check last page) your personal "Application Key" and add it to `.env` as `DATADOG_APP_KEY=`
+ - change the `DATADOG_SUBDOMAIN=app` in `.env` to your companies subdomain if you have one
+ - verify it works by running `rake plan`, it might show some diff, but should not crash
+
+### Adding a team
+
+ - `mention` is used for all team monitors via `super()`
+ - `renotify_interval` is used for all team monitors (defaults to `0` / off)
+ - `tags` is used for all team monitors/dashboards (defaults to `team:<team-name>`)
+
+```Ruby
+# teams/my_team.rb
+module Teams
+  class MyTeam < Kennel::Models::Team
+    defaults(
+      mention: -> { "@slack-my-team" }
+    )
+  end
+end
+```
+
+### Adding a new monitor
+ - use [datadog monitor UI](https://app.datadoghq.com/monitors#create) to create a monitor
+ - see below
+
+### Updating an existing monitor
+ - use [datadog monitor UI](https://app.datadoghq.com/monitors/manage) to find a monitor
+ - get the `id` from the url
+ - run `URL='https://app.datadoghq.com/monitors/123' bundle exec rake kennel:import` and copy the output
+ - find or create a project in `projects/`
+ - add the monitor to `parts: [` list, for example:
+  ```Ruby
+  # projects/my_project.rb
+  class MyProject < Kennel::Models::Project
+    defaults(
+      team: -> { Teams::MyTeam.new }, # use existing team or create new one in teams/
+      parts: -> {
+        [
+          Kennel::Models::Monitor.new(
+            self,
+            id: -> { 123456 }, # id from datadog url, not necessary when creating a new monitor
+            type: -> { "query alert" },
+            kennel_id: -> { "load-too-high" }, # make up a unique name
+            name: -> { "Foobar Load too high" }, # nice descriptive name that will show up in alerts and emails
+            message: -> {
+              # Explain what behavior to expect and how to fix the cause
+              # Use #{super()} to add team notifications.
+              <<~TEXT
+                Foobar will be slow and that could cause Barfoo to go down.
+                Add capacity or debug why it is suddenly slow.
+                #{super()}
+              TEXT
+            },
+            query: -> { "avg(last_5m):avg:system.load.5{hostgroup:api} by {pod} > #{critical}" }, # replace actual value with #{critical} to keep them in sync
+            critical: -> { 20 }
+          )
+        ]
+      }
+    )
+  end
+  ```
+ - run `PROJECT=my_project bundle exec rake plan`, an Update to the existing monitor should be shown (not Create / Delete)
+ - alternatively: `bundle exec rake generate` to only locally update the generated `json` files
+ - review changes then `git commit`
+ - make a PR ... get reviewed ... merge
+ - datadog is updated by travis
+
+### Adding a new dashboard
+ - go to [datadog dashboard UI](https://app.datadoghq.com/dashboard/lists) and click on _New Dashboard_ to create a dashboard
+ - see below
+
+### Updating an existing dashboard
+ - go to [datadog dashboard UI](https://app.datadoghq.com/dashboard/lists) and click on _New Dashboard_ to find a dashboard
+ - get the `id` from the url
+ - run `URL='https://app.datadoghq.com/dashboard/bet-foo-bar' bundle exec rake kennel:import` and copy the output
+ - find or create a project in `projects/`
+ - add a dashboard to `parts: [` list, for example:
+  ```Ruby
+  class MyProject < Kennel::Models::Project
+    defaults(
+      team: -> { Teams::MyTeam.new }, # use existing team or create new one in teams/
+      parts: -> {
+        [
+          Kennel::Models::Dashboard.new(
+            self,
+            id: -> { "abc-def-ghi" }, # id from datadog url, not needed when creating a new dashboard
+            title: -> { "My Dashboard" },
+            description: -> { "Overview of foobar" },
+            template_variables: -> { ["environment"] }, # see https://docs.datadoghq.com/api/?lang=ruby#timeboards
+            kennel_id: -> { "overview-dashboard" }, # make up a unique name
+            layout_type: -> { "ordered" },
+            definitions: -> {
+              [ # An array or arrays, each one is a graph in the dashboard, alternatively a hash for finer control
+                [
+                  # title, viz, type, query, edit an existing graph and see the json definition
+                  "Graph name", "timeseries", "area", "sum:mystats.foobar{$environment}"
+                ],
+                [
+                  # queries can be an Array as well, this will generate multiple requests
+                  # for a single graph
+                  "Graph name", "timeseries", "area", ["sum:mystats.foobar{$environment}", "sum:mystats.success{$environment}"],
+                  # add events too ...
+                  events: [{q: "tags:foobar,deploy", tags_execution: "and"}]
+                ]
+              ]
+            }
+          )
+        ]
+      }
+    )
+  end
+ ```
+
+### Skipping validations
+
+Some validations might be too strict for your usecase or just wrong, please [open an issue](https://github.com/grosser/kennel/issues) and
+to unblock use the `validate: -> { false }` option.
+
+### Linking with kennel_ids
+
+To link to existing monitors via their kennel_id
+
+ - Screens `uptime` widgets can use `monitor: {id: "foo:bar"}`
+ - Screens `alert_graph` widgets can use `alert_id: "foo:bar"`
+ - Monitors `composite` can use `query: -> { "%{foo:bar} || %{foo:baz}" }`
+
+### Debugging changes locally
+
+ - rebase on updated `master` to not undo other changes
+ - figure out project name by converting the class name to snake-case
+ - run `PROJECT=foo bundle exec rake kennel:update_datadog` to test changes for a single project
+
+### Listing un-muted alerts
+
+Run `rake kennel:alerts TAG=service:my-service` to see all un-muted alerts for a given datadog monitor tag.
+
+### Validating mentions work
+
+`rake kennel:validate_mentions` should run as part of CI
+
+### Grepping through all of datadog
+
+`TYPE=monitor rake kennel:dump`
+
+### Find all monitors with No-Data
+
+`rake kennel:nodata TAG=team:foo`
+
+## Examples
+
+### Reusable monitors/dashes/etc
+
+Add to `parts/<folder>`.
+
+```Ruby
+module Monitors
+  class LoadTooHigh < Kennel::Models::Monitor
+    defaults(
+      name: -> { "#{project.name} load too high" },
+      message: -> { "Shut it down!" },
+      type: -> { "query alert" },
+      query: -> { "avg(last_5m):avg:system.load.5{hostgroup:#{project.kennel_id}} by {pod} > #{critical}" }
+    )
+  end
+end
+```
+
+Reuse it in multiple projects.
+
+```Ruby
+class Database < Kennel::Models::Project
+  defaults(
+    team: -> { Kennel::Models::Team.new(mention: -> { '@slack-foo' }, kennel_id: -> { 'foo' }) },
+    parts: -> { [Monitors::LoadTooHigh.new(self, critical: -> { 13 })] }
+  )
+end
+```