kennel 1.78.4 → 1.81.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 23d5816783075cabd0b6a51addfe89bd0d21f4935a547fd1bfbd0c754d6f3b18
-  data.tar.gz: 73df009d112f1af7d1500428e700ac6cdea3d99cbb36a12dd3ceef926b2f5fd4
+  metadata.gz: 5f9d3050dbb7a4e94cd22e64618142bedffd6fdfc5b5ee8e20b621b90f1d8f27
+  data.tar.gz: 640a86b74e46e27ec7f1b09dd555f0ba8f99e4fc1f687e5230bb00d12636ae1d
 SHA512:
-  metadata.gz: 04a237a134852bd15c4ce7eb6a309c0f8526c656f646269ccfeaf8f148013ea6c8328bcfbaffaee53c228af1e6b79c021ff9cade7afacb8a301f4e418302dbd0
-  data.tar.gz: 3e51d0361af539cd3aca894f51d8e28cf5ef613722c1f3a7f7b5f3f88fb74e43ab535034144d246cc0f079d5d06e9e62f488bb76257390cae4a7f0f14636984e
+  metadata.gz: 80dbfff63550b6f5c3127cb8ecc97db6b3db38876469bcb9d50fce00a7e1b78e2f473dd3c3abaec081cb21037f53be72021d907c4285352b30dd3b6991f6c622
+  data.tar.gz: 02a3122ec70a951dc01f919c4d13877de88c6266048d006770a62adadd19587d74a603e8281699f878be0a20227a7086e37727c751cf5833e815fc795e3f9ee3

data/Readme.md

@@ -84,7 +84,7 @@ end
  - `cp .env.example .env`
  - open [Datadog API Settings](https://app.datadoghq.com/account/settings#api)
  - create a `API Key` or get an existing one from an admin, then add it to `.env` as `DATADOG_API_KEY`
- - find or create (check last page) your personal "Application Key" and add it to `.env` as `DATADOG_APP_KEY=`
+ - open [Datadog API Settings](https://app.datadoghq.com/access/application-keys) and create a new key, then add it to `.env` as `DATADOG_APP_KEY=`
  - change the `DATADOG_SUBDOMAIN=app` in `.env` to your companies subdomain if you have one
  - verify it works by running `rake plan`, it might show some diff, but should not crash
 -->
@@ -212,14 +212,20 @@ removing the `id` will cause kennel to create a new resource in datadog.
 Some validations might be too strict for your usecase or just wrong, please [open an issue](https://github.com/grosser/kennel/issues) and
 to unblock use the `validate: -> { false }` option.
 
-### Linking with kennel_ids
+### Linking resources with kennel_id
 
-To link to existing monitors via their kennel_id `projects kennel_id` + `:` + `monitors kennel id`
+Link resources with their kennel_id in the format `project kennel_id` + `:` + `resource kennel_id`,
+this should be used to create dependent resources like monitor + slos,
+so they can be created in a single update and can be re-created if any of them is deleted.
 
- - Screens `uptime` widgets can use `monitor: {id: "foo:bar"}`
- - Screens `alert_graph` widgets can use `alert_id: "foo:bar"`
- - Monitors `composite` can use `query: -> { "%{foo:bar} || %{foo:baz}" }`
- - Slos can use `monitor_ids: -> ["foo:bar"]` 
+|Resource|Type|Syntax|
+|---|---|---|
+|Dashboard|uptime|`monitor: {id: "foo:bar"}`|
+|Dashboard|alert_graph|`alert_id: "foo:bar"`|
+|Dashboard|slo|`slo_id: "foo:bar"`|
+|Monitor|composite|`query: -> { "%{foo:bar} && %{foo:baz}" }`|
+|Monitor|slo alert|`query: -> { "error_budget(\"%{foo:bar}\").over(\"7d\") > 123.0" }`|
+|Slo|monitor|`monitor_ids: -> ["foo:bar"]`|
 
 ### Debugging changes locally
 
@@ -296,5 +302,5 @@ Author
 [Michael Grosser](http://grosser.it)<br/>
 michael@grosser.it<br/>
 License: MIT<br/>
-[![Build Status](https://travis-ci.org/grosser/kennel.png)](https://travis-ci.org/grosser/kennel)
+![CI](https://github.com/grosser/kennel/workflows/CI/badge.svg)
 <!-- NOT IN -->

data/lib/kennel.rb

@@ -39,12 +39,7 @@ module Kennel
     attr_accessor :out, :err
 
     def generate
-      FileUtils.rm_rf("generated")
-      generated.each do |part|
-        path = "generated/#{part.tracking_id.sub(":", "/")}.json"
-        FileUtils.mkdir_p(File.dirname(path))
-        File.write(path, JSON.pretty_generate(part.as_json) << "\n")
-      end
+      store generated
     end
 
     def plan
@@ -58,6 +53,35 @@ module Kennel
 
     private
 
+    def store(parts)
+      Progress.progress "Storing" do
+        old = Dir["generated/**/*"]
+        used = []
+
+        Utils.parallel(parts, max: 2) do |part|
+          path = "generated/#{part.tracking_id.tr("/", ":").sub(":", "/")}.json"
+          used << File.dirname(path) # only 1 level of sub folders, so this is safe
+          used << path
+          write_file_if_necessary(path, JSON.pretty_generate(part.as_json) << "\n")
+        end
+
+        # deleting all is slow, so only delete the extras
+        (old - used).each { |p| FileUtils.rm_rf(p) }
+      end
+    end
+
+    def write_file_if_necessary(path, content)
+      # 99% case
+      begin
+        return if File.read(path) == content
+      rescue Errno::ENOENT
+        FileUtils.mkdir_p(File.dirname(path))
+      end
+
+      # slow 1% case
+      File.write(path, content)
+    end
+
     def syncer
       @syncer ||= Syncer.new(api, generated, project: ENV["PROJECT"])
     end
@@ -73,8 +97,12 @@ module Kennel
           parts = Models::Project.recursive_subclasses.flat_map do |project_class|
             project_class.new.validated_parts
           end
-          parts.map(&:tracking_id).group_by { |id| id }.select do |id, same|
-            raise "#{id} is defined #{same.size} times" if same.size != 1
+          parts.group_by(&:tracking_id).each do |tracking_id, same|
+            next if same.size == 1
+            raise <<~ERROR
+              #{tracking_id} is defined #{same.size} times
+              use a different `kennel_id` when defining multiple projects/monitors/dashboards to avoid this conflict
+            ERROR
           end
           parts
         end

data/lib/kennel/models/dashboard.rb

@@ -140,16 +140,16 @@ module Kennel
           when "uptime"
             if ids = definition[:monitor_ids]
               definition[:monitor_ids] = ids.map do |id|
-                tracking_id?(id) ? resolve_link(id, :monitor, id_map, **args) : id
+                tracking_id?(id) ? (resolve_link(id, :monitor, id_map, **args) || id) : id
               end
             end
           when "alert_graph"
             if (id = definition[:alert_id]) && tracking_id?(id)
-              definition[:alert_id] = resolve_link(id, :monitor, id_map, **args).to_s
+              definition[:alert_id] = (resolve_link(id, :monitor, id_map, **args) || id).to_s
             end
           when "slo"
             if (id = definition[:slo_id]) && tracking_id?(id)
-              definition[:slo_id] = resolve_link(id, :slo, id_map, **args).to_s
+              definition[:slo_id] = (resolve_link(id, :slo, id_map, **args) || id).to_s
             end
           end
         end
@@ -186,22 +186,26 @@ module Kennel
       end
 
       def render_definitions(definitions)
-        definitions.map do |title, type, display_type, queries, options = {}, ignored = nil|
-          # validate inputs
-          if ignored || (!title || !type || !queries || !options.is_a?(Hash))
-            raise ArgumentError, "Expected exactly 5 arguments for each definition (title, type, display_type, queries, options)"
-          end
-          if (SUPPORTED_DEFINITION_OPTIONS | options.keys) != SUPPORTED_DEFINITION_OPTIONS
-            raise ArgumentError, "Supported options are: #{SUPPORTED_DEFINITION_OPTIONS.map(&:inspect).join(", ")}"
-          end
+        definitions.map do |title, type, display_type, queries, options = {}, too_many_args = nil|
+          if title.is_a?(Hash) && !type
+            title # user gave a full widget, just use it
+          else
+            # validate inputs
+            if too_many_args || (!title || !type || !queries || !options.is_a?(Hash))
+              raise ArgumentError, "Expected exactly 5 arguments for each definition (title, type, display_type, queries, options)"
+            end
+            if (SUPPORTED_DEFINITION_OPTIONS | options.keys) != SUPPORTED_DEFINITION_OPTIONS
+              raise ArgumentError, "Supported options are: #{SUPPORTED_DEFINITION_OPTIONS.map(&:inspect).join(", ")}"
+            end
 
-          # build definition
-          requests = Array(queries).map do |q|
-            request = { q: q }
-            request[:display_type] = display_type if display_type
-            request
+            # build definition
+            requests = Array(queries).map do |q|
+              request = { q: q }
+              request[:display_type] = display_type if display_type
+              request
+            end
+            { definition: { title: title, type: type, requests: requests, **options } }
           end
-          { definition: { title: title, type: type, requests: requests, **options } }
         end
       end
     end

data/lib/kennel/models/monitor.rb

@@ -12,6 +12,10 @@ module Kennel
         :multi, :matching_downtimes, :overall_state_modified, :overall_state, :restricted_roles
       ]
 
+      MONITOR_DEFAULTS = {
+        priority: nil
+      }.freeze
+
       # defaults that datadog uses when options are not sent, so safe to leave out if our values match their defaults
       MONITOR_OPTION_DEFAULTS = {
         evaluation_delay: nil,
@@ -27,7 +31,7 @@ module Kennel
       settings(
         :query, :name, :message, :escalation_message, :critical, :type, :renotify_interval, :warning, :timeout_h, :evaluation_delay,
         :ok, :no_data_timeframe, :notify_no_data, :notify_audit, :tags, :critical_recovery, :warning_recovery, :require_full_window,
-        :threshold_windows, :new_host_delay
+        :threshold_windows, :new_host_delay, :priority
       )
 
       defaults(
@@ -46,7 +50,8 @@ module Kennel
         evaluation_delay: -> { MONITOR_OPTION_DEFAULTS.fetch(:evaluation_delay) },
         critical_recovery: -> { nil },
         warning_recovery: -> { nil },
-        threshold_windows: -> { nil }
+        threshold_windows: -> { nil },
+        priority: -> { MONITOR_DEFAULTS.fetch(:priority) }
       )
 
       def as_json
@@ -57,6 +62,7 @@ module Kennel
           query: query.strip,
           message: message.strip,
           tags: tags.uniq,
+          priority: priority,
           options: {
             timeout_h: timeout_h,
             notify_no_data: notify_no_data,
@@ -106,9 +112,11 @@ module Kennel
       end
 
       def resolve_linked_tracking_ids!(id_map, **args)
-        if as_json[:type] == "composite"
-          as_json[:query] = as_json[:query].gsub(/%\{(.*?)\}/) do
-            resolve_link($1, :monitor, id_map, **args)
+        case as_json[:type]
+        when "composite", "slo alert"
+          type = (as_json[:type] == "composite" ? :monitor : :slo)
+          as_json[:query] = as_json[:query].gsub(/%{(.*?)}/) do
+            resolve_link($1, type, id_map, **args) || $&
           end
         end
       end
@@ -129,6 +137,9 @@ module Kennel
 
       def self.normalize(expected, actual)
         super
+
+        ignore_default(expected, actual, MONITOR_DEFAULTS)
+
         options = actual.fetch(:options)
         options.delete(:silenced) # we do not manage silenced, so ignore it when diffing
 

data/lib/kennel/models/record.rb

@@ -65,19 +65,18 @@ module Kennel
 
       private
 
-      def resolve_link(id, type, id_map, force:)
-        value = id_map[id]
-        if value == :new
+      def resolve_link(tracking_id, type, id_map, force:)
+        id = id_map[tracking_id]
+        if id == :new
           if force
-            # TODO: remove the need for this by sorting monitors by missing resolutions
-            invalid! "#{id} needs to already exist, try again"
+            invalid! "#{type} #{tracking_id} was referenced but is also created by the current run.\nIt could not be created because of a circular dependency, try creating only some of the resources"
           else
-            id # will be re-resolved by syncer after the linked object was created
+            nil # will be re-resolved after the linked object was created
           end
-        elsif value
-          value
+        elsif id
+          id
         else
-          invalid! "Unable to find #{type} #{id} (does not exist and is not being created by the current run)"
+          invalid! "Unable to find #{type} #{tracking_id} (does not exist and is not being created by the current run)"
         end
       end
 

data/lib/kennel/models/slo.rb

@@ -69,7 +69,7 @@ module Kennel
       def resolve_linked_tracking_ids!(id_map, **args)
         return unless as_json[:monitor_ids] # ignore_default can remove it
         as_json[:monitor_ids] = as_json[:monitor_ids].map do |id|
-          id.is_a?(String) ? resolve_link(id, :monitor, id_map, **args) : id
+          id.is_a?(String) ? (resolve_link(id, :monitor, id_map, **args) || id) : id
         end
       end
 

data/lib/kennel/syncer.rb

@@ -38,23 +38,14 @@ module Kennel
     end
 
     def update
-      changed = (@create + @update).map { |_, e| e }
-
-      @create.each do |_, e|
-        e.resolve_linked_tracking_ids!({}, force: true)
-
+      each_resolved @create do |_, e|
         reply = @api.create e.class.api_resource, e.as_json
         id = reply.fetch(:id)
-
-        # resolve ids we could previously no resolve
-        changed.delete e
-        resolve_linked_tracking_ids! from: [reply], to: changed
-
+        populate_id_map [reply] # allow resolving ids we could previously no resolve
         Kennel.out.puts "Created #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.url(id)}"
       end
 
-      @update.each do |id, e|
-        e.resolve_linked_tracking_ids!({}, force: true)
+      each_resolved @update do |id, e|
         @api.update e.class.api_resource, id, e.as_json
         Kennel.out.puts "Updated #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.url(id)}"
       end
@@ -67,6 +58,37 @@ module Kennel
 
     private
 
+    # loop over items until everything is resolved or crash when we get stuck
+    # this solves cases like composite monitors depending on each other or monitor->monitor slo->slo monitor chains
+    def each_resolved(list)
+      list = list.dup
+      loop do
+        return if list.empty?
+        list.reject! do |id, e|
+          if resolved?(e)
+            yield id, e
+            true
+          else
+            false
+          end
+        end ||
+          assert_resolved(list[0][1]) # resolve something or show a circular dependency error
+      end
+    end
+
+    # TODO: optimize by storing an instance variable if already resolved
+    def resolved?(e)
+      assert_resolved e
+      true
+    rescue ValidationError
+      false
+    end
+
+    # raises ValidationError when not resolved
+    def assert_resolved(e)
+      resolve_linked_tracking_ids! [e], force: true
+    end
+
     def noop?
       @create.empty? && @update.empty? && @delete.empty?
     end
@@ -74,9 +96,15 @@ module Kennel
     def calculate_diff
       @update = []
       @delete = []
+      @id_map = {}
 
       actual = Progress.progress("Downloading definitions") { download_definitions }
-      resolve_linked_tracking_ids! from: actual, to: @expected
+
+      # resolve dependencies to avoid diff
+      populate_id_map actual
+      @expected.each { |e| @id_map[e.tracking_id] ||= :new }
+      resolve_linked_tracking_ids! @expected
+
       filter_by_project! actual
 
       Progress.progress "Diffing" do
@@ -107,7 +135,6 @@ module Kennel
 
         ensure_all_ids_found
         @create = @expected.map { |e| [nil, e] }
-        @create.sort_by! { |_, e| -DELETE_ORDER.index(e.class.api_resource) }
       end
 
       @delete.sort_by! { |_, _, a| DELETE_ORDER.index a.fetch(:api_resource) }
@@ -214,10 +241,12 @@ module Kennel
       end
     end
 
-    def resolve_linked_tracking_ids!(from:, to:)
-      map = from.each_with_object({}) { |a, lookup| lookup[tracking_id(a)] = a.fetch(:id) }
-      to.each { |e| map[e.tracking_id] ||= :new }
-      to.each { |e| e.resolve_linked_tracking_ids!(map, force: false) }
+    def populate_id_map(actual)
+      actual.each { |a| @id_map[tracking_id(a)] = a.fetch(:id) }
+    end
+
+    def resolve_linked_tracking_ids!(list, force: false)
+      list.each { |e| e.resolve_linked_tracking_ids!(@id_map, force: force) }
     end
 
     def filter_by_project!(definitions)

data/lib/kennel/utils.rb

@@ -23,6 +23,11 @@ module Kennel
           .downcase
       end
 
+      # for child projects, not used internally
+      def title_case(string)
+        string.split(/[\s_]/).map(&:capitalize) * " "
+      end
+
       # simplified version of https://apidock.com/rails/ActiveSupport/Inflector/parameterize
       def parameterize(string)
         string

data/lib/kennel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Kennel
-  VERSION = "1.78.4"
+  VERSION = "1.81.2"
 end

data/template/Readme.md

@@ -67,7 +67,7 @@ end
  - `cp .env.example .env`
  - open [Datadog API Settings](https://app.datadoghq.com/account/settings#api)
  - create a `API Key` or get an existing one from an admin, then add it to `.env` as `DATADOG_API_KEY`
- - find or create (check last page) your personal "Application Key" and add it to `.env` as `DATADOG_APP_KEY=`
+ - open [Datadog API Settings](https://app.datadoghq.com/access/application-keys) and create a new key, then add it to `.env` as `DATADOG_APP_KEY=`
  - change the `DATADOG_SUBDOMAIN=app` in `.env` to your companies subdomain if you have one
  - verify it works by running `rake plan`, it might show some diff, but should not crash
 
@@ -194,14 +194,20 @@ removing the `id` will cause kennel to create a new resource in datadog.
 Some validations might be too strict for your usecase or just wrong, please [open an issue](https://github.com/grosser/kennel/issues) and
 to unblock use the `validate: -> { false }` option.
 
-### Linking with kennel_ids
+### Linking resources with kennel_id
 
-To link to existing monitors via their kennel_id `projects kennel_id` + `:` + `monitors kennel id`
+Link resources with their kennel_id in the format `project kennel_id` + `:` + `resource kennel_id`,
+this should be used to create dependent resources like monitor + slos,
+so they can be created in a single update and can be re-created if any of them is deleted.
 
- - Screens `uptime` widgets can use `monitor: {id: "foo:bar"}`
- - Screens `alert_graph` widgets can use `alert_id: "foo:bar"`
- - Monitors `composite` can use `query: -> { "%{foo:bar} || %{foo:baz}" }`
- - Slos can use `monitor_ids: -> ["foo:bar"]` 
+|Resource|Type|Syntax|
+|---|---|---|
+|Dashboard|uptime|`monitor: {id: "foo:bar"}`|
+|Dashboard|alert_graph|`alert_id: "foo:bar"`|
+|Dashboard|slo|`slo_id: "foo:bar"`|
+|Monitor|composite|`query: -> { "%{foo:bar} && %{foo:baz}" }`|
+|Monitor|slo alert|`query: -> { "error_budget(\"%{foo:bar}\").over(\"7d\") > 123.0" }`|
+|Slo|monitor|`monitor_ids: -> ["foo:bar"]`|
 
 ### Debugging changes locally
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kennel
 version: !ruby/object:Gem::Version
-  version: 1.78.4
+  version: 1.81.2
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-17 00:00:00.000000000 Z
+date: 2021-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday