kennel 1.78.3 → 1.81.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1606ccb73f8c7b4e4a0ac2d6c4cf7473d320ece51d38bab7d7b574366086476d
-  data.tar.gz: 1ce0a1b1edd186c0acaf392ca5dd668419ccf891388e197cdbb40b611ec6cfc4
+  metadata.gz: 4ecb9c89748e6eeee268d0d744622b536242af1c1d7f615bf77fc5963e1630d8
+  data.tar.gz: e6cae4da66e0947163d1f441bce2afb19f1818dac0ed9418143b3f2a2480b220
 SHA512:
-  metadata.gz: f39e476825364d1d6eca54cefab1173008894c02bcb749457c2c2b3d8a0995553b6bfbc44f281e61f9e031f316d1a8c06bf81f2bfaefea618258bf536bfcb555
-  data.tar.gz: 8588a48b7a5886b04d4ba9030e1cd6b27fc7ba3271150edab3447d7aae977e3b47100218b5c1ac2597d4176f969194c326a28434d825311b69710ec14bcb8ef5
+  metadata.gz: 7a3342551a926a55c580571524cadbb32bf7b02b77a914f32e1edf2a07b6772091373286c36871d9651a57899134364d8c9c5894a8ef1e46f72ad091ca241990
+  data.tar.gz: 6eb25f647a3a7d47c0858ca03f0649e151f6437d711701ea24f1dd3b742b5adbfbd3f0318a531076b2236063dda194bbdec72237eaf3f1159f447ac86a95ee68

data/Readme.md

@@ -84,7 +84,7 @@ end
  - `cp .env.example .env`
  - open [Datadog API Settings](https://app.datadoghq.com/account/settings#api)
  - create a `API Key` or get an existing one from an admin, then add it to `.env` as `DATADOG_API_KEY`
- - find or create (check last page) your personal "Application Key" and add it to `.env` as `DATADOG_APP_KEY=`
+ - open [Datadog API Settings](https://app.datadoghq.com/access/application-keys) and create a new key, then add it to `.env` as `DATADOG_APP_KEY=`
  - change the `DATADOG_SUBDOMAIN=app` in `.env` to your companies subdomain if you have one
  - verify it works by running `rake plan`, it might show some diff, but should not crash
 -->
@@ -212,14 +212,20 @@ removing the `id` will cause kennel to create a new resource in datadog.
 Some validations might be too strict for your usecase or just wrong, please [open an issue](https://github.com/grosser/kennel/issues) and
 to unblock use the `validate: -> { false }` option.
 
-### Linking with kennel_ids
+### Linking resources with kennel_id
 
-To link to existing monitors via their kennel_id `projects kennel_id` + `:` + `monitors kennel id`
+Link resources with their kennel_id in the format `project kennel_id` + `:` + `resource kennel_id`,
+this should be used to create dependent resources like monitor + slos,
+so they can be created in a single update and can be re-created if any of them is deleted. 
 
- - Screens `uptime` widgets can use `monitor: {id: "foo:bar"}`
- - Screens `alert_graph` widgets can use `alert_id: "foo:bar"`
- - Monitors `composite` can use `query: -> { "%{foo:bar} || %{foo:baz}" }`
- - Slos can use `monitor_ids: -> ["foo:bar"]` 
+|Resource|Type|Syntax|
+|---|---|---|
+|Dashboard|uptime|`monitor: {id: "foo:bar"}`|
+|Dashboard|alert_graph|`alert_id: "foo:bar"`|
+|Dashboard|slo|`slo_id: "foo:bar"`|
+|Monitor|composite|`query: -> { "%{foo:bar} && %{foo:baz}" }`|
+|Monitor|slo alert|`query: -> { "error_budget(\"%{foo:bar}\").over(\"7d\") > 123.0" }`|
+|Slo|monitor|`monitor_ids: -> ["foo:bar"]`|
 
 ### Debugging changes locally
 
@@ -296,5 +302,5 @@ Author
 [Michael Grosser](http://grosser.it)<br/>
 michael@grosser.it<br/>
 License: MIT<br/>
-[![Build Status](https://travis-ci.org/grosser/kennel.png)](https://travis-ci.org/grosser/kennel)
+![CI](https://github.com/grosser/kennel/workflows/CI/badge.svg)
 <!-- NOT IN -->

data/lib/kennel.rb

@@ -39,12 +39,7 @@ module Kennel
     attr_accessor :out, :err
 
     def generate
-      FileUtils.rm_rf("generated")
-      generated.each do |part|
-        path = "generated/#{part.tracking_id.sub(":", "/")}.json"
-        FileUtils.mkdir_p(File.dirname(path))
-        File.write(path, JSON.pretty_generate(part.as_json) << "\n")
-      end
+      store generated
     end
 
     def plan
@@ -58,6 +53,35 @@ module Kennel
 
     private
 
+    def store(parts)
+      Progress.progress "Storing" do
+        old = Dir["generated/**/*"]
+        used = []
+
+        Utils.parallel(parts, max: 2) do |part|
+          path = "generated/#{part.tracking_id.tr("/", ":").sub(":", "/")}.json"
+          used << File.dirname(path) # only 1 level of sub folders, so this is safe
+          used << path
+          write_file_if_necessary(path, JSON.pretty_generate(part.as_json) << "\n")
+        end
+
+        # deleting all is slow, so only delete the extras
+        (old - used).each { |p| FileUtils.rm_rf(p) }
+      end
+    end
+
+    def write_file_if_necessary(path, content)
+      # 99% case
+      begin
+        return if File.read(path) == content
+      rescue Errno::ENOENT
+        FileUtils.mkdir_p(File.dirname(path))
+      end
+
+      # slow 1% case
+      File.write(path, content)
+    end
+
     def syncer
       @syncer ||= Syncer.new(api, generated, project: ENV["PROJECT"])
     end
@@ -73,8 +97,12 @@ module Kennel
           parts = Models::Project.recursive_subclasses.flat_map do |project_class|
             project_class.new.validated_parts
           end
-          parts.map(&:tracking_id).group_by { |id| id }.select do |id, same|
-            raise "#{id} is defined #{same.size} times" if same.size != 1
+          parts.group_by(&:tracking_id).each do |tracking_id, same|
+            next if same.size == 1
+            raise <<~ERROR
+              #{tracking_id} is defined #{same.size} times
+              use a different `kennel_id` when defining multiple projects/monitors/dashboards to avoid this conflict
+            ERROR
           end
           parts
         end

data/lib/kennel/models/dashboard.rb

@@ -105,7 +105,7 @@ module Kennel
 
       def as_json
         return @json if @json
-        all_widgets = render_definitions + widgets
+        all_widgets = render_definitions(definitions) + widgets
         expand_q all_widgets
 
         @json = {
@@ -140,16 +140,16 @@ module Kennel
           when "uptime"
             if ids = definition[:monitor_ids]
               definition[:monitor_ids] = ids.map do |id|
-                tracking_id?(id) ? resolve_link(id, :monitor, id_map, **args) : id
+                tracking_id?(id) ? (resolve_link(id, :monitor, id_map, **args) || id) : id
               end
             end
           when "alert_graph"
             if (id = definition[:alert_id]) && tracking_id?(id)
-              definition[:alert_id] = resolve_link(id, :monitor, id_map, **args).to_s
+              definition[:alert_id] = (resolve_link(id, :monitor, id_map, **args) || id).to_s
             end
           when "slo"
             if (id = definition[:slo_id]) && tracking_id?(id)
-              definition[:slo_id] = resolve_link(id, :slo, id_map, **args).to_s
+              definition[:slo_id] = (resolve_link(id, :slo, id_map, **args) || id).to_s
             end
           end
         end
@@ -185,7 +185,7 @@ module Kennel
         invalid! "template_variable_presets must be sorted by name" if presets && presets != presets.sort_by { |p| p[:name] }
       end
 
-      def render_definitions
+      def render_definitions(definitions)
         definitions.map do |title, type, display_type, queries, options = {}, ignored = nil|
           # validate inputs
           if ignored || (!title || !type || !queries || !options.is_a?(Hash))

data/lib/kennel/models/monitor.rb

@@ -12,6 +12,10 @@ module Kennel
         :multi, :matching_downtimes, :overall_state_modified, :overall_state, :restricted_roles
       ]
 
+      MONITOR_DEFAULTS = {
+        priority: nil
+      }.freeze
+
       # defaults that datadog uses when options are not sent, so safe to leave out if our values match their defaults
       MONITOR_OPTION_DEFAULTS = {
         evaluation_delay: nil,
@@ -27,7 +31,7 @@ module Kennel
       settings(
         :query, :name, :message, :escalation_message, :critical, :type, :renotify_interval, :warning, :timeout_h, :evaluation_delay,
         :ok, :no_data_timeframe, :notify_no_data, :notify_audit, :tags, :critical_recovery, :warning_recovery, :require_full_window,
-        :threshold_windows, :new_host_delay
+        :threshold_windows, :new_host_delay, :priority
       )
 
       defaults(
@@ -46,7 +50,8 @@ module Kennel
         evaluation_delay: -> { MONITOR_OPTION_DEFAULTS.fetch(:evaluation_delay) },
         critical_recovery: -> { nil },
         warning_recovery: -> { nil },
-        threshold_windows: -> { nil }
+        threshold_windows: -> { nil },
+        priority: -> { MONITOR_DEFAULTS.fetch(:priority) }
       )
 
       def as_json
@@ -57,6 +62,7 @@ module Kennel
           query: query.strip,
           message: message.strip,
           tags: tags.uniq,
+          priority: priority,
           options: {
             timeout_h: timeout_h,
             notify_no_data: notify_no_data,
@@ -106,9 +112,11 @@ module Kennel
       end
 
       def resolve_linked_tracking_ids!(id_map, **args)
-        if as_json[:type] == "composite"
-          as_json[:query] = as_json[:query].gsub(/%\{(.*?)\}/) do
-            resolve_link($1, :monitor, id_map, **args)
+        case as_json[:type]
+        when "composite", "slo alert"
+          type = (as_json[:type] == "composite" ? :monitor : :slo)
+          as_json[:query] = as_json[:query].gsub(/%{(.*?)}/) do
+            resolve_link($1, type, id_map, **args) || $&
           end
         end
       end
@@ -129,6 +137,9 @@ module Kennel
 
       def self.normalize(expected, actual)
         super
+
+        ignore_default(expected, actual, MONITOR_DEFAULTS)
+
         options = actual.fetch(:options)
         options.delete(:silenced) # we do not manage silenced, so ignore it when diffing
 

data/lib/kennel/models/record.rb

@@ -65,19 +65,18 @@ module Kennel
 
       private
 
-      def resolve_link(id, type, id_map, force:)
-        value = id_map[id]
-        if value == :new
+      def resolve_link(tracking_id, type, id_map, force:)
+        id = id_map[tracking_id]
+        if id == :new
           if force
-            # TODO: remove the need for this by sorting monitors by missing resolutions
-            invalid! "#{id} needs to already exist, try again"
+            invalid! "#{type} #{tracking_id} was referenced but is also created by the current run.\nIt could not be created because of a circular dependency, try creating only some of the resources"
           else
-            id # will be re-resolved by syncer after the linked object was created
+            nil # will be re-resolved after the linked object was created
           end
-        elsif value
-          value
+        elsif id
+          id
         else
-          invalid! "Unable to find #{type} #{id} (does not exist and is not being created by the current run)"
+          invalid! "Unable to find #{type} #{tracking_id} (does not exist and is not being created by the current run)"
         end
       end
 

data/lib/kennel/models/slo.rb

@@ -69,7 +69,7 @@ module Kennel
       def resolve_linked_tracking_ids!(id_map, **args)
         return unless as_json[:monitor_ids] # ignore_default can remove it
         as_json[:monitor_ids] = as_json[:monitor_ids].map do |id|
-          id.is_a?(String) ? resolve_link(id, :monitor, id_map, **args) : id
+          id.is_a?(String) ? (resolve_link(id, :monitor, id_map, **args) || id) : id
         end
       end
 

data/lib/kennel/syncer.rb

@@ -38,23 +38,14 @@ module Kennel
     end
 
     def update
-      changed = (@create + @update).map { |_, e| e }
-
-      @create.each do |_, e|
-        e.resolve_linked_tracking_ids!({}, force: true)
-
+      each_resolved @create do |_, e|
         reply = @api.create e.class.api_resource, e.as_json
         id = reply.fetch(:id)
-
-        # resolve ids we could previously no resolve
-        changed.delete e
-        resolve_linked_tracking_ids! from: [reply], to: changed
-
+        populate_id_map [reply] # allow resolving ids we could previously no resolve
         Kennel.out.puts "Created #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.url(id)}"
       end
 
-      @update.each do |id, e|
-        e.resolve_linked_tracking_ids!({}, force: true)
+      each_resolved @update do |id, e|
         @api.update e.class.api_resource, id, e.as_json
         Kennel.out.puts "Updated #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.url(id)}"
       end
@@ -67,6 +58,37 @@ module Kennel
 
     private
 
+    # loop over items until everything is resolved or crash when we get stuck
+    # this solves cases like composite monitors depending on each other or monitor->monitor slo->slo monitor chains
+    def each_resolved(list)
+      list = list.dup
+      loop do
+        return if list.empty?
+        list.reject! do |id, e|
+          if resolved?(e)
+            yield id, e
+            true
+          else
+            false
+          end
+        end ||
+          assert_resolved(list[0][1]) # resolve something or show a circular dependency error
+      end
+    end
+
+    # TODO: optimize by storing an instance variable if already resolved
+    def resolved?(e)
+      assert_resolved e
+      true
+    rescue ValidationError
+      false
+    end
+
+    # raises ValidationError when not resolved
+    def assert_resolved(e)
+      resolve_linked_tracking_ids! [e], force: true
+    end
+
     def noop?
       @create.empty? && @update.empty? && @delete.empty?
     end
@@ -74,9 +96,15 @@ module Kennel
     def calculate_diff
       @update = []
       @delete = []
+      @id_map = {}
 
       actual = Progress.progress("Downloading definitions") { download_definitions }
-      resolve_linked_tracking_ids! from: actual, to: @expected
+
+      # resolve dependencies to avoid diff
+      populate_id_map actual
+      @expected.each { |e| @id_map[e.tracking_id] ||= :new }
+      resolve_linked_tracking_ids! @expected
+
       filter_by_project! actual
 
       Progress.progress "Diffing" do
@@ -107,7 +135,6 @@ module Kennel
 
         ensure_all_ids_found
         @create = @expected.map { |e| [nil, e] }
-        @create.sort_by! { |_, e| -DELETE_ORDER.index(e.class.api_resource) }
       end
 
       @delete.sort_by! { |_, _, a| DELETE_ORDER.index a.fetch(:api_resource) }
@@ -214,10 +241,12 @@ module Kennel
       end
     end
 
-    def resolve_linked_tracking_ids!(from:, to:)
-      map = from.each_with_object({}) { |a, lookup| lookup[tracking_id(a)] = a.fetch(:id) }
-      to.each { |e| map[e.tracking_id] ||= :new }
-      to.each { |e| e.resolve_linked_tracking_ids!(map, force: false) }
+    def populate_id_map(actual)
+      actual.each { |a| @id_map[tracking_id(a)] = a.fetch(:id) }
+    end
+
+    def resolve_linked_tracking_ids!(list, force: false)
+      list.each { |e| e.resolve_linked_tracking_ids!(@id_map, force: force) }
     end
 
     def filter_by_project!(definitions)

data/lib/kennel/utils.rb

@@ -23,6 +23,11 @@ module Kennel
           .downcase
       end
 
+      # for child projects, not used internally
+      def title_case(string)
+        string.split(/[\s_]/).map(&:capitalize) * " "
+      end
+
       # simplified version of https://apidock.com/rails/ActiveSupport/Inflector/parameterize
       def parameterize(string)
         string

data/lib/kennel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Kennel
-  VERSION = "1.78.3"
+  VERSION = "1.81.1"
 end

data/template/Readme.md

@@ -67,7 +67,7 @@ end
  - `cp .env.example .env`
  - open [Datadog API Settings](https://app.datadoghq.com/account/settings#api)
  - create a `API Key` or get an existing one from an admin, then add it to `.env` as `DATADOG_API_KEY`
- - find or create (check last page) your personal "Application Key" and add it to `.env` as `DATADOG_APP_KEY=`
+ - open [Datadog API Settings](https://app.datadoghq.com/access/application-keys) and create a new key, then add it to `.env` as `DATADOG_APP_KEY=`
  - change the `DATADOG_SUBDOMAIN=app` in `.env` to your companies subdomain if you have one
  - verify it works by running `rake plan`, it might show some diff, but should not crash
 
@@ -196,12 +196,17 @@ to unblock use the `validate: -> { false }` option.
 
 ### Linking with kennel_ids
 
-To link to existing monitors via their kennel_id `projects kennel_id` + `:` + `monitors kennel id`
-
- - Screens `uptime` widgets can use `monitor: {id: "foo:bar"}`
- - Screens `alert_graph` widgets can use `alert_id: "foo:bar"`
- - Monitors `composite` can use `query: -> { "%{foo:bar} || %{foo:baz}" }`
- - Slos can use `monitor_ids: -> ["foo:bar"]` 
+Link resources via their kennel_id `projects kennel_id` + `:` + `monitors kennel id`,
+this should be used to create dependent resources like monitor + slos. 
+
+|Resource|Type|Syntax|
+|---|---|---|
+|Dashboard|uptime|`monitor: {id: "foo:bar"}`|
+|Dashboard|alert_graph|`alert_id: "foo:bar"`|
+|Dashboard|slo|`slo_id: "foo:bar"`|
+|Monitor|composite|`query: -> { "%{foo:bar} && %{foo:baz}" }`|
+|Monitor|slo alert|`query: -> { "error_budget(\"%{foo:bar}\").over(\"7d\") > 123.0" }`|
+|Slo|monitor|`monitor_ids: -> ["foo:bar"]`|
 
 ### Debugging changes locally
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kennel
 version: !ruby/object:Gem::Version
-  version: 1.78.3
+  version: 1.81.1
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-09 00:00:00.000000000 Z
+date: 2021-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday