kennel 1.75.0

data/lib/kennel/template_variables.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+module Kennel
+  module TemplateVariables
+    def self.included(base)
+      base.settings :template_variables
+      base.defaults(template_variables: -> { [] })
+    end
+
+    private
+
+    def render_template_variables
+      (template_variables || []).map do |v|
+        v.is_a?(String) ? { default: "*", prefix: v, name: v } : v
+      end
+    end
+
+    # check for queries that do not use the variables and would be misleading
+    # TODO: do the same check for apm_query and their group_by
+    def validate_template_variables(data, key)
+      variables = (data[:template_variables] || []).map { |v| "$#{v.fetch(:name)}" }
+      queries = data[key].flat_map do |widget|
+        ([widget] + (widget.dig(:definition, :widgets) || [])).flat_map { |w| widget_queries(w) }
+      end.compact
+      bad = queries.grep_v(/(#{variables.map { |v| Regexp.escape(v) }.join("|")})\b/)
+      if bad.any?
+        invalid!(
+          "queries #{bad.join(", ")} must use the template variables #{variables.join(", ")}\n" \
+          "If that is not possible, add `validate: -> { false } # query foo in bar does not have baz tag`"
+        )
+      end
+    end
+
+    def widget_queries(widget)
+      requests = widget.dig(:definition, :requests) || []
+      (requests.is_a?(Hash) ? requests.values : requests).map { |r| r[:q] } # hostmap widgets have hash requests
+    end
+  end
+end

data/lib/kennel/unmuted_alerts.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+require "kennel"
+
+# Show Alerts that are not muted and their alerting scopes
+module Kennel
+  class UnmutedAlerts
+    COLORS = {
+      "Alert" => :red,
+      "Warn" => :yellow,
+      "No Data" => :cyan
+    }.freeze
+
+    class << self
+      def print(api, tag)
+        monitors = filtered_monitors(api, tag)
+        if monitors.empty?
+          Kennel.out.puts "No unmuted alerts found"
+        else
+          monitors.each do |m|
+            Kennel.out.puts m[:name]
+            Kennel.out.puts Utils.path_to_url("/monitors/#{m[:id]}")
+            m[:state][:groups].each do |g|
+              color = COLORS[g[:status]] || :default
+              since = "\t#{time_since(g[:last_triggered_ts])}"
+              Kennel.out.puts "#{Kennel::Utils.color(color, g[:status])}\t#{g[:name]}#{since}"
+            end
+            Kennel.out.puts
+          end
+        end
+      end
+
+      private
+
+      # sort pod3 before pod11
+      def sort_groups!(monitor)
+        groups = monitor[:state][:groups].values
+        groups.sort_by! { |g| g[:name].to_s.split(",").map { |w| Utils.natural_order(w) } }
+        monitor[:state][:groups] = groups
+      end
+
+      def time_since(t)
+        diff = Time.now.to_i - Integer(t)
+        "%02d:%02d:%02d" % [diff / 3600, diff / 60 % 60, diff % 60]
+      end
+
+      def filtered_monitors(api, tag)
+        # Download all monitors with given tag
+        monitors = Progress.progress("Downloading") do
+          api.list("monitor", monitor_tags: tag, group_states: "all", with_downtimes: "true")
+        end
+
+        raise "No monitors for #{tag} found, check your spelling" if monitors.empty?
+
+        # only keep monitors that are alerting
+        monitors.reject! { |m| m[:overall_state] == "OK" }
+
+        # only keep monitors that are not completely silenced
+        monitors.reject! { |m| m[:options][:silenced].key?(:*) }
+
+        # only keep groups that are alerting
+        monitors.each { |m| m[:state][:groups].reject! { |_, g| g[:status] == "OK" || g[:status] == "Ignored" } }
+
+        # only keep alerting groups that are not silenced
+        monitors.each do |m|
+          silenced = m[:options][:silenced].keys.map { |k| k.to_s.split(",") }
+          m[:state][:groups].select! do |k, _|
+            scope = k.to_s.split(",")
+            silenced.none? { |s| (s - scope).empty? }
+          end
+        end
+
+        # only keep monitors that are not covered by a downtime
+        monitors.each do |m|
+          next unless m[:matching_downtimes]
+          downtime_groups = m[:matching_downtimes].select { |d| d[:active] }.flat_map { |d| d[:groups] }
+          m[:state][:groups].reject! do |k, _|
+            downtime_groups.include?(k.to_s)
+          end
+        end
+
+        # only keep monitors with alerting groups
+        monitors.select! { |m| m[:state][:groups].any? }
+
+        # sort group alerts
+        monitors.each { |m| sort_groups!(m) }
+      end
+    end
+  end
+end

data/lib/kennel/utils.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+module Kennel
+  module Utils
+    COLORS = { red: 31, green: 32, yellow: 33, cyan: 36, magenta: 35, default: 0 }.freeze
+
+    class TeeIO < IO
+      def initialize(ios)
+        @ios = ios
+      end
+
+      def write(string)
+        @ios.each { |io| io.write string }
+      end
+    end
+
+    class << self
+      def snake_case(string)
+        string
+          .gsub(/::/, "_") # Foo::Bar -> foo_bar
+          .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2') # FOOBar -> foo_bar
+          .gsub(/([a-z\d])([A-Z])/, '\1_\2') # fooBar -> foo_bar
+          .tr("-", "_") # foo-bar -> foo_bar
+          .downcase
+      end
+
+      # simplified version of https://apidock.com/rails/ActiveSupport/Inflector/parameterize
+      def parameterize(string)
+        string
+          .downcase
+          .gsub(/[^a-z0-9\-_]+/, "-") # remove unsupported
+          .gsub(/-{2,}/, "-") # remove duplicates
+          .gsub(/^-|-$/, "") # remove leading/trailing
+      end
+
+      def presence(value)
+        value.nil? || value.empty? ? nil : value
+      end
+
+      def ask(question)
+        Kennel.err.printf color(:red, "#{question} -  press 'y' to continue: ")
+        begin
+          STDIN.gets.chomp == "y"
+        rescue Interrupt # do not show a backtrace if user decides to Ctrl+C here
+          Kennel.err.print "\n"
+          exit 1
+        end
+      end
+
+      def color(color, text)
+        "\e[#{COLORS.fetch(color)}m#{text}\e[0m"
+      end
+
+      def strip_shell_control(text)
+        text.gsub(/\e\[\d+m(.*?)\e\[0m/, "\\1").gsub(/.#{Regexp.escape("\b")}/, "")
+      end
+
+      def capture_stdout
+        old = Kennel.out
+        Kennel.out = StringIO.new
+        yield
+        Kennel.out.string
+      ensure
+        Kennel.out = old
+      end
+
+      def capture_stderr
+        old = Kennel.err
+        Kennel.err = StringIO.new
+        yield
+        Kennel.err.string
+      ensure
+        Kennel.err = old
+      end
+
+      def tee_output
+        old_stdout = Kennel.out
+        old_stderr = Kennel.err
+        capture = StringIO.new
+        Kennel.out = TeeIO.new([capture, Kennel.out])
+        Kennel.err = TeeIO.new([capture, Kennel.err])
+        yield
+        capture.string
+      ensure
+        Kennel.out = old_stdout
+        Kennel.err = old_stderr
+      end
+
+      def capture_sh(command)
+        result = `#{command} 2>&1`
+        raise "Command failed:\n#{command}\n#{result}" unless $CHILD_STATUS.success?
+        result
+      end
+
+      def path_to_url(path)
+        if subdomain = ENV["DATADOG_SUBDOMAIN"]
+          "https://#{subdomain}.datadoghq.com#{path}"
+        else
+          path
+        end
+      end
+
+      def parallel(items, max: 10)
+        threads = [items.size, max].min
+        work = items.each_with_index.to_a
+        done = Array.new(items.size)
+        workers = Array.new(threads).map do
+          Thread.new do
+            loop do
+              item, i = work.pop
+              break unless i
+              done[i] =
+                begin
+                  yield item
+                rescue StandardError => e
+                  work.clear
+                  e
+                end
+            end
+          end
+        end
+        workers.each(&:join)
+        done.each { |d| raise d if d.is_a?(StandardError) }
+      end
+
+      def natural_order(name)
+        name.split(/(\d+)/).each_with_index.map { |x, i| i.odd? ? x.to_i : x }
+      end
+
+      def retry(*errors, times:)
+        yield
+      rescue *errors => e
+        times -= 1
+        raise if times < 0
+        Kennel.err.puts "Error #{e}, #{times} retries left"
+        retry
+      end
+
+      # https://stackoverflow.com/questions/20235206/ruby-get-all-keys-in-a-hash-including-sub-keys/53876255#53876255
+      def all_keys(items)
+        case items
+        when Hash then items.keys + items.values.flat_map { |v| all_keys(v) }
+        when Array then items.flat_map { |i| all_keys(i) }
+        else []
+        end
+      end
+
+      # TODO: use awesome-print or similar, but it has too many monkey-patches
+      # https://github.com/amazing-print/amazing_print/issues/36
+      def pretty_inspect(object)
+        string = object.inspect
+        string.gsub!(/:([a-z_]+)=>/, "\\1: ")
+        10.times do
+          string.gsub!(/{(\S.*?\S)}/, "{ \\1 }") || break
+        end
+        string
+      end
+    end
+  end
+end

data/lib/kennel/version.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+module Kennel
+  VERSION = "1.75.0"
+end

data/template/Readme.md

@@ -0,0 +1,247 @@
+![](github/cage.jpg?raw=true)
+
+Manage Datadog Monitors / Dashboards / Slos as code
+
+ - DRY, searchable, audited, documented
+ - Changes are PR reviewed and applied on merge
+ - Updating shows diff before applying
+ - Automated import of existing resources
+ - Resources are grouped into projects that belong to teams and inherit tags
+ - No copy-pasting of ids to create new resources
+ - Automated cleanup when removing code
+ - [Helpers](#helpers) for automating common tasks
+ 
+### Applying changes
+
+![](github/screen.png?raw=true)
+
+### Example code
+
+```Ruby
+# teams/foo.rb
+module Teams
+  class Foo < Kennel::Models::Team
+    defaults(mention: -> { "@slack-my-team" })
+  end
+end
+
+# projects/bar.rb
+class Bar < Kennel::Models::Project
+  defaults(
+    team: -> { Teams::Foo.new }, # use mention and tags from the team
+    parts: -> {
+      [
+        Kennel::Models::Monitor.new(
+          self, # the current project
+          type: -> { "query alert" },
+          kennel_id: -> { "load-too-high" }, # pick a unique name
+          name: -> { "Foobar Load too high" }, # nice descriptive name that will show up in alerts and emails
+          message: -> {
+            <<~TEXT
+              This is bad!
+              #{super()} # inserts mention from team
+            TEXT
+          },
+          query: -> { "avg(last_5m):avg:system.load.5{hostgroup:api} by {pod} > #{critical}" },
+          critical: -> { 20 }
+        )
+      ]
+    }
+  )
+end
+```
+
+
+## Structure
+
+ - `projects/` monitors/dashboards/etc scoped by project
+ - `teams/` team definitions
+ - `parts/` monitors/dashboards/etc that are used by multiple projects
+ - `generated/` projects as json, to show current state and proposed changes in PRs
+
+## Workflows
+
+### Setup
+ - clone the repo
+ - `gem install bundler && bundle install`
+ - `cp .env.example .env`
+ - open [Datadog API Settings](https://app.datadoghq.com/account/settings#api)
+ - copy any `API Key` and add it to `.env` as `DATADOG_API_KEY`
+ - find or create (check last page) your personal "Application Key" and add it to `.env` as `DATADOG_APP_KEY=`
+ - change the `DATADOG_SUBDOMAIN=app` in `.env` to your companies subdomain if you have one
+ - verify it works by running `rake plan`, it might show some diff, but should not crash
+
+### Adding a team
+
+ - `mention` is used for all team monitors via `super()`
+ - `renotify_interval` is used for all team monitors (defaults to `0` / off)
+ - `tags` is used for all team monitors/dashboards (defaults to `team:<team-name>`)
+
+```Ruby
+# teams/my_team.rb
+module Teams
+  class MyTeam < Kennel::Models::Team
+    defaults(
+      mention: -> { "@slack-my-team" }
+    )
+  end
+end
+```
+
+### Adding a new monitor
+ - use [datadog monitor UI](https://app.datadoghq.com/monitors#create) to create a monitor
+ - see below
+
+### Updating an existing monitor
+ - use [datadog monitor UI](https://app.datadoghq.com/monitors/manage) to find a monitor
+ - get the `id` from the url
+ - run `URL='https://app.datadoghq.com/monitors/123' bundle exec rake kennel:import` and copy the output
+ - find or create a project in `projects/`
+ - add the monitor to `parts: [` list, for example:
+  ```Ruby
+  # projects/my_project.rb
+  class MyProject < Kennel::Models::Project
+    defaults(
+      team: -> { Teams::MyTeam.new }, # use existing team or create new one in teams/
+      parts: -> {
+        [
+          Kennel::Models::Monitor.new(
+            self,
+            id: -> { 123456 }, # id from datadog url, not necessary when creating a new monitor
+            type: -> { "query alert" },
+            kennel_id: -> { "load-too-high" }, # make up a unique name
+            name: -> { "Foobar Load too high" }, # nice descriptive name that will show up in alerts and emails
+            message: -> {
+              # Explain what behavior to expect and how to fix the cause
+              # Use #{super()} to add team notifications.
+              <<~TEXT
+                Foobar will be slow and that could cause Barfoo to go down.
+                Add capacity or debug why it is suddenly slow.
+                #{super()}
+              TEXT
+            },
+            query: -> { "avg(last_5m):avg:system.load.5{hostgroup:api} by {pod} > #{critical}" }, # replace actual value with #{critical} to keep them in sync
+            critical: -> { 20 }
+          )
+        ]
+      }
+    )
+  end
+  ```
+ - run `PROJECT=my_project bundle exec rake plan`, an Update to the existing monitor should be shown (not Create / Delete)
+ - alternatively: `bundle exec rake generate` to only locally update the generated `json` files
+ - review changes then `git commit`
+ - make a PR ... get reviewed ... merge
+ - datadog is updated by CI
+
+### Adding a new dashboard
+ - go to [datadog dashboard UI](https://app.datadoghq.com/dashboard/lists) and click on _New Dashboard_ to create a dashboard
+ - see below
+
+### Updating an existing dashboard
+ - go to [datadog dashboard UI](https://app.datadoghq.com/dashboard/lists) and click on _New Dashboard_ to find a dashboard
+ - get the `id` from the url
+ - run `URL='https://app.datadoghq.com/dashboard/bet-foo-bar' bundle exec rake kennel:import` and copy the output
+ - find or create a project in `projects/`
+ - add a dashboard to `parts: [` list, for example:
+  ```Ruby
+  class MyProject < Kennel::Models::Project
+    defaults(
+      team: -> { Teams::MyTeam.new }, # use existing team or create new one in teams/
+      parts: -> {
+        [
+          Kennel::Models::Dashboard.new(
+            self,
+            id: -> { "abc-def-ghi" }, # id from datadog url, not needed when creating a new dashboard
+            title: -> { "My Dashboard" },
+            description: -> { "Overview of foobar" },
+            template_variables: -> { ["environment"] }, # see https://docs.datadoghq.com/api/?lang=ruby#timeboards
+            kennel_id: -> { "overview-dashboard" }, # make up a unique name
+            layout_type: -> { "ordered" },
+            definitions: -> {
+              [ # An array or arrays, each one is a graph in the dashboard, alternatively a hash for finer control
+                [
+                  # title, viz, type, query, edit an existing graph and see the json definition
+                  "Graph name", "timeseries", "area", "sum:mystats.foobar{$environment}"
+                ],
+                [
+                  # queries can be an Array as well, this will generate multiple requests
+                  # for a single graph
+                  "Graph name", "timeseries", "area", ["sum:mystats.foobar{$environment}", "sum:mystats.success{$environment}"],
+                  # add events too ...
+                  events: [{q: "tags:foobar,deploy", tags_execution: "and"}]
+                ]
+              ]
+            }
+          )
+        ]
+      }
+    )
+  end
+ ```
+
+### Skipping validations
+
+Some validations might be too strict for your usecase or just wrong, please [open an issue](https://github.com/grosser/kennel/issues) and
+to unblock use the `validate: -> { false }` option.
+
+### Linking with kennel_ids
+
+To link to existing monitors via their kennel_id
+
+ - Screens `uptime` widgets can use `monitor: {id: "foo:bar"}`
+ - Screens `alert_graph` widgets can use `alert_id: "foo:bar"`
+ - Monitors `composite` can use `query: -> { "%{foo:bar} || %{foo:baz}" }`
+
+### Debugging changes locally
+
+ - rebase on updated `master` to not undo other changes
+ - figure out project name by converting the class name to snake-case
+ - run `PROJECT=foo bundle exec rake kennel:update_datadog` to test changes for a single project
+
+### Reuse
+
+Add to `parts/<folder>`.
+
+```Ruby
+module Monitors
+  class LoadTooHigh < Kennel::Models::Monitor
+    defaults(
+      name: -> { "#{project.name} load too high" },
+      message: -> { "Shut it down!" },
+      type: -> { "query alert" },
+      query: -> { "avg(last_5m):avg:system.load.5{hostgroup:#{project.kennel_id}} by {pod} > #{critical}" }
+    )
+  end
+end
+```
+
+Reuse it in multiple projects.
+
+```Ruby
+class Database < Kennel::Models::Project
+  defaults(
+    team: -> { Kennel::Models::Team.new(mention: -> { '@slack-foo' }, kennel_id: -> { 'foo' }) },
+    parts: -> { [Monitors::LoadTooHigh.new(self, critical: -> { 13 })] }
+  )
+end
+```
+
+## Helpers
+
+### Listing un-muted alerts
+
+Run `rake kennel:alerts TAG=service:my-service` to see all un-muted alerts for a given datadog monitor tag.
+
+### Validating mentions work
+
+`rake kennel:validate_mentions` should run as part of CI
+
+### Grepping through all of datadog
+
+`TYPE=monitor rake kennel:dump`
+
+### Find all monitors with No-Data
+
+`rake kennel:nodata TAG=team:foo`
+