kennel 1.74.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 769ede5638522cc0c56394f0fde86207b88aad4ade1ff05e0360018972e52a55
+  data.tar.gz: 467dfab6d84ec29c1efbba1f7ab1df2fad3fd5767521a677c88b38c8bb8f2e19
+SHA512:
+  metadata.gz: 64f8dbae6ba9ea0d51be0c20a955a261ca026c760e77cf05e945b0a2514e6ef99527f8a00a9dc39abc053235fd76474435fb13d610e6e1ed1ce4e92d99e81ea9
+  data.tar.gz: 695f7f97b7355291d66b72423ac3c1adcadf804ced559aa35222c431c92198beca00fa625ebbad70ab105173ec8ca0b11eb4ec13a3dd19c04945fa9709821552

data/Readme.md

@@ -0,0 +1,244 @@
+# Kennel
+
+![](template/github/cage.jpg?raw=true)
+
+Manage datadog monitors/dashboards/slos as code
+
+ - Documented, reusable, and searchable
+ - Changes are PR reviewed and auditable
+ - Updating shows diff before applying
+ - Automated import of existing monitors/dashboards/slos
+
+![](template/github/screen.png?raw=true)
+<!-- NOT IN template/Readme.md  -->
+## Install
+
+ - create a new private `kennel` repo for your organization (do not fork this repo)
+ - use the template folder as starting point:
+    ```Bash
+    git clone git@github.com:your-org/kennel.git
+    git clone git@github.com:grosser/kennel.git seed
+    mv seed/template/* kennel/
+    cd kennel && git add . && git commit -m 'initial'
+    ```
+ - add a basic projects and teams so others can copy-paste to get started
+ - setup travis build for your repo
+ - uncomment `.travis.yml` section for automated PR planing and datadog updates on merge
+ - follow `Setup` in your repos Readme.md
+<!-- NOT IN -->
+
+## Structure
+
+ - `projects/` monitors/dashboards/etc scoped by project
+ - `teams/` team definitions
+ - `parts/` monitors/dashboards/etc that are used by multiple projects
+ - `generated/` projects as json, to show current state and proposed changes in PRs
+
+## Workflows
+
+<!-- ONLY IN template/Readme.md
+### Setup
+ - clone the repo
+ - `gem install bundler && bundle install`
+ - `cp .env.example .env`
+ - open [Datadog API Settings](https://app.datadoghq.com/account/settings#api)
+ - copy any `API Key` and add it to `.env` as `DATADOG_API_KEY`
+ - find or create (check last page) your personal "Application Key" and add it to `.env` as `DATADOG_APP_KEY=`
+ - change the `DATADOG_SUBDOMAIN=app` in `.env` to your companies subdomain if you have one
+ - verify it works by running `rake plan`, it might show some diff, but should not crash
+-->
+
+### Adding a team
+
+ - `mention` is used for all team monitors via `super()`
+ - `renotify_interval` is used for all team monitors (defaults to `0` / off)
+ - `tags` is used for all team monitors/dashboards (defaults to `team:<team-name>`)
+
+```Ruby
+# teams/my_team.rb
+module Teams
+  class MyTeam < Kennel::Models::Team
+    defaults(
+      mention: -> { "@slack-my-team" }
+    )
+  end
+end
+```
+
+### Adding a new monitor
+ - use [datadog monitor UI](https://app.datadoghq.com/monitors#create) to create a monitor
+ - see below
+
+### Updating an existing monitor
+ - use [datadog monitor UI](https://app.datadoghq.com/monitors/manage) to find a monitor
+ - get the `id` from the url
+ - run `URL='https://app.datadoghq.com/monitors/123' bundle exec rake kennel:import` and copy the output
+ - find or create a project in `projects/`
+ - add the monitor to `parts: [` list, for example:
+  ```Ruby
+  # projects/my_project.rb
+  class MyProject < Kennel::Models::Project
+    defaults(
+      team: -> { Teams::MyTeam.new }, # use existing team or create new one in teams/
+      parts: -> {
+        [
+          Kennel::Models::Monitor.new(
+            self,
+            id: -> { 123456 }, # id from datadog url, not necessary when creating a new monitor
+            type: -> { "query alert" },
+            kennel_id: -> { "load-too-high" }, # make up a unique name
+            name: -> { "Foobar Load too high" }, # nice descriptive name that will show up in alerts and emails
+            message: -> {
+              # Explain what behavior to expect and how to fix the cause
+              # Use #{super()} to add team notifications.
+              <<~TEXT
+                Foobar will be slow and that could cause Barfoo to go down.
+                Add capacity or debug why it is suddenly slow.
+                #{super()}
+              TEXT
+            },
+            query: -> { "avg(last_5m):avg:system.load.5{hostgroup:api} by {pod} > #{critical}" }, # replace actual value with #{critical} to keep them in sync
+            critical: -> { 20 }
+          )
+        ]
+      }
+    )
+  end
+  ```
+ - run `PROJECT=my_project bundle exec rake plan`, an Update to the existing monitor should be shown (not Create / Delete)
+ - alternatively: `bundle exec rake generate` to only locally update the generated `json` files
+ - review changes then `git commit`
+ - make a PR ... get reviewed ... merge
+ - datadog is updated by travis
+
+### Adding a new dashboard
+ - go to [datadog dashboard UI](https://app.datadoghq.com/dashboard/lists) and click on _New Dashboard_ to create a dashboard
+ - see below
+
+### Updating an existing dashboard
+ - go to [datadog dashboard UI](https://app.datadoghq.com/dashboard/lists) and click on _New Dashboard_ to find a dashboard
+ - get the `id` from the url
+ - run `URL='https://app.datadoghq.com/dashboard/bet-foo-bar' bundle exec rake kennel:import` and copy the output
+ - find or create a project in `projects/`
+ - add a dashboard to `parts: [` list, for example:
+  ```Ruby
+  class MyProject < Kennel::Models::Project
+    defaults(
+      team: -> { Teams::MyTeam.new }, # use existing team or create new one in teams/
+      parts: -> {
+        [
+          Kennel::Models::Dashboard.new(
+            self,
+            id: -> { "abc-def-ghi" }, # id from datadog url, not needed when creating a new dashboard
+            title: -> { "My Dashboard" },
+            description: -> { "Overview of foobar" },
+            template_variables: -> { ["environment"] }, # see https://docs.datadoghq.com/api/?lang=ruby#timeboards
+            kennel_id: -> { "overview-dashboard" }, # make up a unique name
+            layout_type: -> { "ordered" },
+            definitions: -> {
+              [ # An array or arrays, each one is a graph in the dashboard, alternatively a hash for finer control
+                [
+                  # title, viz, type, query, edit an existing graph and see the json definition
+                  "Graph name", "timeseries", "area", "sum:mystats.foobar{$environment}"
+                ],
+                [
+                  # queries can be an Array as well, this will generate multiple requests
+                  # for a single graph
+                  "Graph name", "timeseries", "area", ["sum:mystats.foobar{$environment}", "sum:mystats.success{$environment}"],
+                  # add events too ...
+                  events: [{q: "tags:foobar,deploy", tags_execution: "and"}]
+                ]
+              ]
+            }
+          )
+        ]
+      }
+    )
+  end
+ ```
+
+### Skipping validations
+
+Some validations might be too strict for your usecase or just wrong, please [open an issue](https://github.com/grosser/kennel/issues) and
+to unblock use the `validate: -> { false }` option.
+
+### Linking with kennel_ids
+
+To link to existing monitors via their kennel_id
+
+ - Screens `uptime` widgets can use `monitor: {id: "foo:bar"}`
+ - Screens `alert_graph` widgets can use `alert_id: "foo:bar"`
+ - Monitors `composite` can use `query: -> { "%{foo:bar} || %{foo:baz}" }`
+
+### Debugging changes locally
+
+ - rebase on updated `master` to not undo other changes
+ - figure out project name by converting the class name to snake-case
+ - run `PROJECT=foo bundle exec rake kennel:update_datadog` to test changes for a single project
+
+### Listing un-muted alerts
+
+Run `rake kennel:alerts TAG=service:my-service` to see all un-muted alerts for a given datadog monitor tag.
+
+### Validating mentions work
+
+`rake kennel:validate_mentions` should run as part of CI
+
+### Grepping through all of datadog
+
+`TYPE=monitor rake kennel:dump`
+
+### Find all monitors with No-Data
+
+`rake kennel:nodata TAG=team:foo`
+
+## Examples
+
+### Reusable monitors/dashes/etc
+
+Add to `parts/<folder>`.
+
+```Ruby
+module Monitors
+  class LoadTooHigh < Kennel::Models::Monitor
+    defaults(
+      name: -> { "#{project.name} load too high" },
+      message: -> { "Shut it down!" },
+      type: -> { "query alert" },
+      query: -> { "avg(last_5m):avg:system.load.5{hostgroup:#{project.kennel_id}} by {pod} > #{critical}" }
+    )
+  end
+end
+```
+
+Reuse it in multiple projects.
+
+```Ruby
+class Database < Kennel::Models::Project
+  defaults(
+    team: -> { Kennel::Models::Team.new(mention: -> { '@slack-foo' }, kennel_id: -> { 'foo' }) },
+    parts: -> { [Monitors::LoadTooHigh.new(self, critical: -> { 13 })] }
+  )
+end
+```
+<!-- NOT IN template/Readme.md -->
+
+### Integration testing
+
+```Bash
+rake play
+cd template
+rake plan
+```
+
+Then make changes to play around, do not commit changes and make sure to revert with a `rake kennel:update_datadog` after deleting everything.
+
+To make changes via the UI, make a new free datadog account and use it's credentaisl instead.
+
+Author
+======
+[Michael Grosser](http://grosser.it)<br/>
+michael@grosser.it<br/>
+License: MIT<br/>
+[![Build Status](https://travis-ci.org/grosser/kennel.png)](https://travis-ci.org/grosser/kennel)
+<!-- NOT IN -->

data/lib/kennel.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+require "faraday"
+require "json"
+require "English"
+
+require "kennel/version"
+require "kennel/utils"
+require "kennel/progress"
+require "kennel/syncer"
+require "kennel/api"
+require "kennel/github_reporter"
+require "kennel/subclass_tracking"
+require "kennel/settings_as_methods"
+require "kennel/file_cache"
+require "kennel/template_variables"
+require "kennel/optional_validations"
+require "kennel/unmuted_alerts"
+
+require "kennel/models/base"
+require "kennel/models/record"
+
+# records
+require "kennel/models/dashboard"
+require "kennel/models/monitor"
+require "kennel/models/slo"
+
+# settings
+require "kennel/models/project"
+require "kennel/models/team"
+
+module Kennel
+  class ValidationError < RuntimeError
+  end
+
+  @out = $stdout
+  @err = $stderr
+
+  class << self
+    attr_accessor :out, :err
+
+    def generate
+      FileUtils.rm_rf("generated")
+      generated.each do |part|
+        path = "generated/#{part.tracking_id.sub(":", "/")}.json"
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, JSON.pretty_generate(part.as_json) << "\n")
+      end
+    end
+
+    def plan
+      syncer.plan
+    end
+
+    def update
+      syncer.plan
+      syncer.update if syncer.confirm
+    end
+
+    private
+
+    def syncer
+      @syncer ||= Syncer.new(api, generated, project: ENV["PROJECT"])
+    end
+
+    def api
+      @api ||= Api.new(ENV.fetch("DATADOG_APP_KEY"), ENV.fetch("DATADOG_API_KEY"))
+    end
+
+    def generated
+      @generated ||= begin
+        Progress.progress "Generating" do
+          load_all
+          parts = Models::Project.recursive_subclasses.flat_map do |project_class|
+            project_class.new.validated_parts
+          end
+          parts.map(&:tracking_id).group_by { |id| id }.select do |id, same|
+            raise "#{id} is defined #{same.size} times" if same.size != 1
+          end
+          parts
+        end
+      end
+    end
+
+    def load_all
+      ["teams", "parts", "projects"].each do |folder|
+        Dir["#{folder}/**/*.rb"].sort.each { |f| require "./#{f}" }
+      end
+    end
+  end
+end

data/lib/kennel/api.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+module Kennel
+  class Api
+    def initialize(app_key, api_key)
+      @app_key = app_key
+      @api_key = api_key
+      @client = Faraday.new(url: "https://app.datadoghq.com") { |c| c.adapter :net_http_persistent }
+    end
+
+    def show(api_resource, id, params = {})
+      reply = request :get, "/api/v1/#{api_resource}/#{id}", params: params
+      api_resource == "slo" ? reply[:data] : reply
+    end
+
+    def list(api_resource, params = {})
+      if api_resource == "slo"
+        raise ArgumentError if params[:limit] || params[:offset]
+        limit = 1000
+        offset = 0
+        all = []
+
+        loop do
+          result = request :get, "/api/v1/#{api_resource}", params: params.merge(limit: limit, offset: offset)
+          data = result.fetch(:data)
+          all.concat data
+          break all if data.size < limit
+          offset += limit
+        end
+      else
+        result = request :get, "/api/v1/#{api_resource}", params: params
+        result = result.fetch(:dashboards) if api_resource == "dashboard"
+        result
+      end
+    end
+
+    def create(api_resource, attributes)
+      reply = request :post, "/api/v1/#{api_resource}", body: attributes
+      api_resource == "slo" ? reply[:data].first : reply
+    end
+
+    def update(api_resource, id, attributes)
+      request :put, "/api/v1/#{api_resource}/#{id}", body: attributes
+    end
+
+    def delete(api_resource, id)
+      request :delete, "/api/v1/#{api_resource}/#{id}", ignore_404: true
+    end
+
+    private
+
+    def request(method, path, body: nil, params: {}, ignore_404: false)
+      params = params.merge(application_key: @app_key, api_key: @api_key)
+      query = Faraday::FlatParamsEncoder.encode(params)
+      response = nil
+      tries = 2
+
+      tries.times do |i|
+        response = Utils.retry Faraday::ConnectionFailed, Faraday::TimeoutError, times: 2 do
+          @client.send(method, "#{path}?#{query}") do |request|
+            request.body = JSON.generate(body) if body
+            request.headers["Content-type"] = "application/json"
+          end
+        end
+
+        break if i == tries - 1 || method != :get || response.status < 500
+        Kennel.err.puts "Retrying on server error #{response.status} for #{path}"
+      end
+
+      if !response.success? && (response.status != 404 || !ignore_404)
+        message = +"Error #{response.status} during #{method.upcase} #{path}\n"
+        message << "request:\n#{JSON.pretty_generate(body)}\nresponse:\n" if body
+        message << response.body
+        raise message
+      end
+
+      if response.body.empty?
+        {}
+      else
+        JSON.parse(response.body, symbolize_names: true)
+      end
+    end
+  end
+end

data/lib/kennel/file_cache.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# cache that reads everything from a single file
+# to avoid doing multiple disk reads while iterating all definitions
+# it also replaces updated keys and has an overall expiry to not keep deleted things forever
+module Kennel
+  class FileCache
+    def initialize(file, cache_version)
+      @file = file
+      @cache_version = cache_version
+      @now = Time.now.to_i
+      @expires = @now + (30 * 24 * 60 * 60) # 1 month
+    end
+
+    def open
+      load_data
+      expire_old_data
+      yield self
+    ensure
+      persist
+    end
+
+    def fetch(key, key_version)
+      old_value, old_version = @data[key]
+      return old_value if old_version == [key_version, @cache_version]
+
+      new_value = yield
+      @data[key] = [new_value, [key_version, @cache_version], @expires]
+      new_value
+    end
+
+    private
+
+    def load_data
+      @data =
+        begin
+          Marshal.load(File.read(@file)) # rubocop:disable Security/MarshalLoad
+        rescue StandardError
+          {}
+        end
+    end
+
+    def persist
+      dir = File.dirname(@file)
+      FileUtils.mkdir_p(dir) unless File.directory?(dir)
+      File.write(@file, Marshal.dump(@data))
+    end
+
+    def expire_old_data
+      @data.reject! { |_, (_, _, ex)| ex < @now }
+    end
+  end
+end