kennel 1.80.0 → 1.83.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0cc9d71dff0d4c4bf8a29d9c58c1b62efbb739f3ebdc0d63fb7bd1a3e1cfec7c
-  data.tar.gz: 312fbda2e1577d071ba7cad7a6c5571b6b3913517cc3e5e9b92f7858637f96dc
+  metadata.gz: 5c3c766383bf7f533e74b935a6faf2272e552bb7c974fa15261c70aa9d31e800
+  data.tar.gz: 339c1dcad92fc215d2a26b7091bbe57dc98d3f6ed5a01666e1ee683162bf20e0
 SHA512:
-  metadata.gz: 8d07c6d405178e7114c1ebaf67ce3f92a09a411208d20353bf7d07f453093e322b98f2e8e2f501cf902a601fafa1b352b5fa19bff9d54f1d042e87fc7ef81277
-  data.tar.gz: 9d681c94f1d61bbc60a098ea0b6fe12e3efc5b6e3c771ec49743db8e6edaae1c83a64ae322a9f5759fc677465b199842f0401c8a38973d1bbaa14157f53f94da
+  metadata.gz: 97abfc77652ab1afb6192c97f038dc6caa33a2f8aa505c62c6782fef1359b431d93cf293c737d9f2ed03c9bb4d22d0474ac916586aa7ed15863900dc8eb1791d
+  data.tar.gz: 13e247e4ab000410155b9f4835686168a2017c1ee2fd8b4e0f6619c54f0b055581b1acc855d12ac06b0ed3920782722fbffa1d303ea7aa04ed92104ab1ec6908

data/Readme.md

@@ -212,15 +212,20 @@ removing the `id` will cause kennel to create a new resource in datadog.
 Some validations might be too strict for your usecase or just wrong, please [open an issue](https://github.com/grosser/kennel/issues) and
 to unblock use the `validate: -> { false }` option.
 
-### Linking with kennel_ids
+### Linking resources with kennel_id
 
-To link to existing monitors via their kennel_id `projects kennel_id` + `:` + `monitors kennel id`
+Link resources with their kennel_id in the format `project kennel_id` + `:` + `resource kennel_id`,
+this should be used to create dependent resources like monitor + slos,
+so they can be created in a single update and can be re-created if any of them is deleted.
 
- - Screens `uptime` widgets can use `monitor: {id: "foo:bar"}`
- - Screens `alert_graph` widgets can use `alert_id: "foo:bar"`
- - Monitors `composite` can use `query: -> { "%{foo:bar} || %{foo:baz}" }`
- - Monitors `slo alert` can use `query: -> { "error_budget(\"%{foo:bar}\").over(\"7d\") > 123.0" }`
- - Slos can use `monitor_ids: -> ["foo:bar"]` 
+|Resource|Type|Syntax|
+|---|---|---|
+|Dashboard|uptime|`monitor: {id: "foo:bar"}`|
+|Dashboard|alert_graph|`alert_id: "foo:bar"`|
+|Dashboard|slo|`slo_id: "foo:bar"`|
+|Monitor|composite|`query: -> { "%{foo:bar} && %{foo:baz}" }`|
+|Monitor|slo alert|`query: -> { "error_budget(\"%{foo:bar}\").over(\"7d\") > 123.0" }`|
+|Slo|monitor|`monitor_ids: -> ["foo:bar"]`|
 
 ### Debugging changes locally
 
@@ -268,9 +273,19 @@ Run `rake kennel:alerts TAG=service:my-service` to see all un-muted alerts for a
 
 ### Grepping through all of datadog
 
-`rake kennel:dump`
+```Bash
+rake kennel:dump > tmp/dump
+cat tmp/dump | grep foo
+```
 focus on a single type: `TYPE=monitors`
 
+Show full resources or just their urls by pattern:
+```Bash
+rake kennel:dump_grep DUMP=tmp/dump PATTERN=foo URLS=true
+https://foo.datadog.com/dasboard/123
+https://foo.datadog.com/monitor/123
+```
+
 ### Find all monitors with No-Data
 
 `rake kennel:nodata TAG=team:foo`

data/lib/kennel/api.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 module Kennel
+  # encapsulates knowledge around how the api works
   class Api
+    CACHE_FILE = "tmp/cache/details"
+
     def initialize(app_key, api_key)
       @app_key = app_key
       @api_key = api_key
@@ -49,8 +52,27 @@ module Kennel
       request :delete, "/api/v1/#{api_resource}/#{id}", params: { force: "true" }, ignore_404: true
     end
 
+    def fill_details!(api_resource, list)
+      return unless api_resource == "dashboard"
+      details_cache do |cache|
+        Utils.parallel(list) { |a| fill_detail!(api_resource, a, cache) }
+      end
+    end
+
     private
 
+    # Make diff work even though we cannot mass-fetch definitions
+    def fill_detail!(api_resource, a, cache)
+      args = [api_resource, a.fetch(:id)]
+      full = cache.fetch(args, a.fetch(:modified_at)) { show(*args) }
+      a.merge!(full)
+    end
+
+    def details_cache(&block)
+      cache = FileCache.new CACHE_FILE, Kennel::VERSION
+      cache.open(&block)
+    end
+
     def request(method, path, body: nil, params: {}, ignore_404: false)
       params = params.merge(application_key: @app_key, api_key: @api_key)
       query = Faraday::FlatParamsEncoder.encode(params)

data/lib/kennel/models/dashboard.rb

@@ -5,7 +5,6 @@ module Kennel
       include TemplateVariables
       include OptionalValidations
 
-      API_LIST_INCOMPLETE = true
       DASHBOARD_DEFAULTS = { template_variables: [] }.freeze
       READONLY_ATTRIBUTES = superclass::READONLY_ATTRIBUTES + [
         :author_handle, :author_name, :modified_at, :url, :is_read_only, :notify_list
@@ -124,7 +123,7 @@ module Kennel
         @json
       end
 
-      def url(id)
+      def self.url(id)
         Utils.path_to_url "/dashboard/#{id}"
       end
 
@@ -140,16 +139,16 @@ module Kennel
           when "uptime"
             if ids = definition[:monitor_ids]
               definition[:monitor_ids] = ids.map do |id|
-                tracking_id?(id) ? resolve_link(id, :monitor, id_map, **args) : id
+                tracking_id?(id) ? (resolve_link(id, :monitor, id_map, **args) || id) : id
               end
             end
           when "alert_graph"
             if (id = definition[:alert_id]) && tracking_id?(id)
-              definition[:alert_id] = resolve_link(id, :monitor, id_map, **args).to_s
+              definition[:alert_id] = (resolve_link(id, :monitor, id_map, **args) || id).to_s
             end
           when "slo"
             if (id = definition[:slo_id]) && tracking_id?(id)
-              definition[:slo_id] = resolve_link(id, :slo, id_map, **args).to_s
+              definition[:slo_id] = (resolve_link(id, :slo, id_map, **args) || id).to_s
             end
           end
         end
@@ -186,22 +185,26 @@ module Kennel
       end
 
       def render_definitions(definitions)
-        definitions.map do |title, type, display_type, queries, options = {}, ignored = nil|
-          # validate inputs
-          if ignored || (!title || !type || !queries || !options.is_a?(Hash))
-            raise ArgumentError, "Expected exactly 5 arguments for each definition (title, type, display_type, queries, options)"
-          end
-          if (SUPPORTED_DEFINITION_OPTIONS | options.keys) != SUPPORTED_DEFINITION_OPTIONS
-            raise ArgumentError, "Supported options are: #{SUPPORTED_DEFINITION_OPTIONS.map(&:inspect).join(", ")}"
-          end
+        definitions.map do |title, type, display_type, queries, options = {}, too_many_args = nil|
+          if title.is_a?(Hash) && !type
+            title # user gave a full widget, just use it
+          else
+            # validate inputs
+            if too_many_args || (!title || !type || !queries || !options.is_a?(Hash))
+              raise ArgumentError, "Expected exactly 5 arguments for each definition (title, type, display_type, queries, options)"
+            end
+            if (SUPPORTED_DEFINITION_OPTIONS | options.keys) != SUPPORTED_DEFINITION_OPTIONS
+              raise ArgumentError, "Supported options are: #{SUPPORTED_DEFINITION_OPTIONS.map(&:inspect).join(", ")}"
+            end
 
-          # build definition
-          requests = Array(queries).map do |q|
-            request = { q: q }
-            request[:display_type] = display_type if display_type
-            request
+            # build definition
+            requests = Array(queries).map do |q|
+              request = { q: q }
+              request[:display_type] = display_type if display_type
+              request
+            end
+            { definition: { title: title, type: type, requests: requests, **options } }
           end
-          { definition: { title: title, type: type, requests: requests, **options } }
         end
       end
     end

data/lib/kennel/models/monitor.rb

@@ -116,7 +116,7 @@ module Kennel
         when "composite", "slo alert"
           type = (as_json[:type] == "composite" ? :monitor : :slo)
           as_json[:query] = as_json[:query].gsub(/%{(.*?)}/) do
-            resolve_link($1, type, id_map, **args)
+            resolve_link($1, type, id_map, **args) || $&
           end
         end
       end
@@ -125,7 +125,7 @@ module Kennel
         "monitor"
       end
 
-      def url(id)
+      def self.url(id)
         Utils.path_to_url "/monitors##{id}/edit"
       end
 

data/lib/kennel/models/record.rb

@@ -6,7 +6,6 @@ module Kennel
       READONLY_ATTRIBUTES = [
         :deleted, :id, :created, :created_at, :creator, :org_id, :modified, :modified_at, :api_resource
       ].freeze
-      API_LIST_INCOMPLETE = false
 
       settings :id, :kennel_id
 
@@ -19,6 +18,10 @@ module Kennel
           end
         end
 
+        def api_resource_map
+          subclasses.map { |s| [s.api_resource, s] }.to_h
+        end
+
         private
 
         def normalize(_expected, actual)
@@ -65,19 +68,18 @@ module Kennel
 
       private
 
-      def resolve_link(id, type, id_map, force:)
-        value = id_map[id]
-        if value == :new
+      def resolve_link(tracking_id, type, id_map, force:)
+        id = id_map[tracking_id]
+        if id == :new
           if force
-            # TODO: remove the need for this by sorting monitors by missing resolutions
-            invalid! "#{id} needs to already exist, try again"
+            invalid! "#{type} #{tracking_id} was referenced but is also created by the current run.\nIt could not be created because of a circular dependency, try creating only some of the resources"
           else
-            id # will be re-resolved by syncer after the linked object was created
+            nil # will be re-resolved after the linked object was created
           end
-        elsif value
-          value
+        elsif id
+          id
         else
-          invalid! "Unable to find #{type} #{id} (does not exist and is not being created by the current run)"
+          invalid! "Unable to find #{type} #{tracking_id} (does not exist and is not being created by the current run)"
         end
       end
 

data/lib/kennel/models/slo.rb

@@ -58,7 +58,7 @@ module Kennel
         "slo"
       end
 
-      def url(id)
+      def self.url(id)
         Utils.path_to_url "/slo?slo_id=#{id}"
       end
 
@@ -69,7 +69,7 @@ module Kennel
       def resolve_linked_tracking_ids!(id_map, **args)
         return unless as_json[:monitor_ids] # ignore_default can remove it
         as_json[:monitor_ids] = as_json[:monitor_ids].map do |id|
-          id.is_a?(String) ? resolve_link(id, :monitor, id_map, **args) : id
+          id.is_a?(String) ? (resolve_link(id, :monitor, id_map, **args) || id) : id
         end
       end
 

data/lib/kennel/syncer.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 module Kennel
   class Syncer
-    CACHE_FILE = "tmp/cache/details" # keep in sync with .travis.yml caching
     TRACKING_FIELDS = [:message, :description].freeze
     DELETE_ORDER = ["dashboard", "slo", "monitor"].freeze # dashboards references monitors + slos, slos reference monitors
 
@@ -38,25 +37,16 @@ module Kennel
     end
 
     def update
-      changed = (@create + @update).map { |_, e| e }
-
-      @create.each do |_, e|
-        e.resolve_linked_tracking_ids!({}, force: true)
-
+      each_resolved @create do |_, e|
         reply = @api.create e.class.api_resource, e.as_json
         id = reply.fetch(:id)
-
-        # resolve ids we could previously no resolve
-        changed.delete e
-        resolve_linked_tracking_ids! from: [reply], to: changed
-
-        Kennel.out.puts "Created #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.url(id)}"
+        populate_id_map [reply] # allow resolving ids we could previously no resolve
+        Kennel.out.puts "Created #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.class.url(id)}"
       end
 
-      @update.each do |id, e|
-        e.resolve_linked_tracking_ids!({}, force: true)
+      each_resolved @update do |id, e|
         @api.update e.class.api_resource, id, e.as_json
-        Kennel.out.puts "Updated #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.url(id)}"
+        Kennel.out.puts "Updated #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.class.url(id)}"
       end
 
       @delete.each do |id, _, a|
@@ -67,6 +57,37 @@ module Kennel
 
     private
 
+    # loop over items until everything is resolved or crash when we get stuck
+    # this solves cases like composite monitors depending on each other or monitor->monitor slo->slo monitor chains
+    def each_resolved(list)
+      list = list.dup
+      loop do
+        return if list.empty?
+        list.reject! do |id, e|
+          if resolved?(e)
+            yield id, e
+            true
+          else
+            false
+          end
+        end ||
+          assert_resolved(list[0][1]) # resolve something or show a circular dependency error
+      end
+    end
+
+    # TODO: optimize by storing an instance variable if already resolved
+    def resolved?(e)
+      assert_resolved e
+      true
+    rescue ValidationError
+      false
+    end
+
+    # raises ValidationError when not resolved
+    def assert_resolved(e)
+      resolve_linked_tracking_ids! [e], force: true
+    end
+
     def noop?
       @create.empty? && @update.empty? && @delete.empty?
     end
@@ -74,9 +95,15 @@ module Kennel
     def calculate_diff
       @update = []
       @delete = []
+      @id_map = {}
 
       actual = Progress.progress("Downloading definitions") { download_definitions }
-      resolve_linked_tracking_ids! from: actual, to: @expected
+
+      # resolve dependencies to avoid diff
+      populate_id_map actual
+      @expected.each { |e| @id_map[e.tracking_id] ||= :new }
+      resolve_linked_tracking_ids! @expected
+
       filter_by_project! actual
 
       Progress.progress "Diffing" do
@@ -89,10 +116,10 @@ module Kennel
           end
         end
 
-        details_cache do |cache|
-          # fill details of things we need to compare (only do this part in parallel for safety & balancing)
-          Utils.parallel(items.select { |e, _| e && e.class::API_LIST_INCOMPLETE }) { |_, a| fill_details(a, cache) }
-        end
+        # fill details of things we need to compare
+        detailed = Hash.new { |h, k| h[k] = [] }
+        items.each { |e, a| detailed[a[:api_resource]] << a if e }
+        detailed.each { |api_resource, actuals| @api.fill_details! api_resource, actuals }
 
         # pick out things to update or delete
         items.each do |e, a|
@@ -107,27 +134,11 @@ module Kennel
 
         ensure_all_ids_found
         @create = @expected.map { |e| [nil, e] }
-        @create.sort_by! { |_, e| -DELETE_ORDER.index(e.class.api_resource) }
       end
 
       @delete.sort_by! { |_, _, a| DELETE_ORDER.index a.fetch(:api_resource) }
     end
 
-    # Make diff work even though we cannot mass-fetch definitions
-    def fill_details(a, cache)
-      resource = a.fetch(:api_resource)
-      args = [resource, a.fetch(:id)]
-      full = cache.fetch(args, a[:modified] || a.fetch(:modified_at)) do
-        @api.show(*args)
-      end
-      a.merge!(full)
-    end
-
-    def details_cache(&block)
-      cache = FileCache.new CACHE_FILE, Kennel::VERSION
-      cache.open(&block)
-    end
-
     def download_definitions
       Utils.parallel(Models::Record.subclasses.map(&:api_resource)) do |api_resource|
         results = @api.list(api_resource, with_downtimes: false) # lookup monitors without adding unnecessary downtime information
@@ -214,10 +225,12 @@ module Kennel
       end
     end
 
-    def resolve_linked_tracking_ids!(from:, to:)
-      map = from.each_with_object({}) { |a, lookup| lookup[tracking_id(a)] = a.fetch(:id) }
-      to.each { |e| map[e.tracking_id] ||= :new }
-      to.each { |e| e.resolve_linked_tracking_ids!(map, force: false) }
+    def populate_id_map(actual)
+      actual.each { |a| @id_map[tracking_id(a)] = a.fetch(:id) }
+    end
+
+    def resolve_linked_tracking_ids!(list, force: false)
+      list.each { |e| e.resolve_linked_tracking_ids!(@id_map, force: force) }
     end
 
     def filter_by_project!(definitions)

data/lib/kennel/tasks.rb

@@ -130,15 +130,41 @@ namespace :kennel do
       if type = ENV["TYPE"]
         [type]
       else
-        Kennel::Models::Record.subclasses.map(&:api_resource)
+        Kennel::Models::Record.api_resource_map.keys
       end
+    api = Kennel.send(:api)
+    list = nil
+
     resources.each do |resource|
-      Kennel.send(:api).list(resource).each do |r|
+      Kennel::Progress.progress("Downloading #{resource}") do
+        list = api.list(resource)
+        api.fill_details!(resource, list)
+      end
+      list.each do |r|
+        r[:api_resource] = resource
         Kennel.out.puts JSON.pretty_generate(r)
       end
     end
   end
 
+  desc "Find items from dump by pattern DUMP= PATTERN= [URLS=true]"
+  task dump_grep: :environment do
+    file = ENV.fetch("DUMP")
+    pattern = Regexp.new ENV.fetch("PATTERN")
+    items = File.read(file).gsub("}\n{", "}--SPLIT--{").split("--SPLIT--")
+    models = Kennel::Models::Record.api_resource_map
+    found = items.grep(pattern)
+    exit 1 if found.empty?
+    found.each do |resource|
+      if ENV["URLS"]
+        parsed = JSON.parse(resource)
+        Kennel.out.puts models[parsed.fetch("api_resource")].url(parsed.fetch("id"))
+      else
+        Kennel.out.puts resource
+      end
+    end
+  end
+
   task :environment do
     require "kennel"
     gem "dotenv"

data/lib/kennel/utils.rb

@@ -23,6 +23,11 @@ module Kennel
           .downcase
       end
 
+      # for child projects, not used internally
+      def title_case(string)
+        string.split(/[\s_]/).map(&:capitalize) * " "
+      end
+
       # simplified version of https://apidock.com/rails/ActiveSupport/Inflector/parameterize
       def parameterize(string)
         string

data/lib/kennel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Kennel
-  VERSION = "1.80.0"
+  VERSION = "1.83.0"
 end

data/template/Readme.md

@@ -194,15 +194,20 @@ removing the `id` will cause kennel to create a new resource in datadog.
 Some validations might be too strict for your usecase or just wrong, please [open an issue](https://github.com/grosser/kennel/issues) and
 to unblock use the `validate: -> { false }` option.
 
-### Linking with kennel_ids
+### Linking resources with kennel_id
 
-To link to existing monitors via their kennel_id `projects kennel_id` + `:` + `monitors kennel id`
+Link resources with their kennel_id in the format `project kennel_id` + `:` + `resource kennel_id`,
+this should be used to create dependent resources like monitor + slos,
+so they can be created in a single update and can be re-created if any of them is deleted.
 
- - Screens `uptime` widgets can use `monitor: {id: "foo:bar"}`
- - Screens `alert_graph` widgets can use `alert_id: "foo:bar"`
- - Monitors `composite` can use `query: -> { "%{foo:bar} || %{foo:baz}" }`
- - Monitors `slo alert` can use `query: -> { "error_budget(\"%{foo:bar}\").over(\"7d\") > 123.0" }`
- - Slos can use `monitor_ids: -> ["foo:bar"]` 
+|Resource|Type|Syntax|
+|---|---|---|
+|Dashboard|uptime|`monitor: {id: "foo:bar"}`|
+|Dashboard|alert_graph|`alert_id: "foo:bar"`|
+|Dashboard|slo|`slo_id: "foo:bar"`|
+|Monitor|composite|`query: -> { "%{foo:bar} && %{foo:baz}" }`|
+|Monitor|slo alert|`query: -> { "error_budget(\"%{foo:bar}\").over(\"7d\") > 123.0" }`|
+|Slo|monitor|`monitor_ids: -> ["foo:bar"]`|
 
 ### Debugging changes locally
 
@@ -250,9 +255,19 @@ Run `rake kennel:alerts TAG=service:my-service` to see all un-muted alerts for a
 
 ### Grepping through all of datadog
 
-`rake kennel:dump`
+```Bash
+rake kennel:dump > tmp/dump
+cat tmp/dump | grep foo
+```
 focus on a single type: `TYPE=monitors`
 
+Show full resources or just their urls by pattern:
+```Bash
+rake kennel:dump_grep DUMP=tmp/dump PATTERN=foo URLS=true
+https://foo.datadog.com/dasboard/123
+https://foo.datadog.com/monitor/123
+```
+
 ### Find all monitors with No-Data
 
 `rake kennel:nodata TAG=team:foo`

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kennel
 version: !ruby/object:Gem::Version
-  version: 1.80.0
+  version: 1.83.0
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-07 00:00:00.000000000 Z
+date: 2021-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday