kennel 1.79.0 → 1.82.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e2a8b643d03deb142461d127a97ae6ad45144cffffb3bae2925cc60cf6a1d683
-  data.tar.gz: 1baaa314a376a99317163a871e5ec6df10dcd33a2cdd3cbc36d2bed9283fe288
+  metadata.gz: 97ae92e1ab137731096bbedde401059fc2b8d411ff667df489056f05a3cf42aa
+  data.tar.gz: c48683c4888e16da817885a44a876638086c32e8e370aaf16553c31b7fde0136
 SHA512:
-  metadata.gz: e4a2266131f58c1d833172e95e3b87b80b49f120675b27baf5e70bd0c5cf191a1dafeb0374b407ff983b5d5e793479d79fdfd548921f161e2644df1dbb86abb8
-  data.tar.gz: d791193329518eb1c277a24543abe1c3d4b62621af0046bd58a3347cdab3faa74da06dcc639dcd8c4d12ac0c76536652d91ada35db63b3d2fb71c7d618afa6b8
+  metadata.gz: 89f7a44c794130ecd4a4238ace5846d050412fd923cc5ae358cb469370c3dd380569bffe991b2251bc80baf4e183bd765591295b043ffc51fd88c5593b723f21
+  data.tar.gz: 4a9eece67d3f0e0cf8dd56a21440ce585397d531c4aa82d1d4c3d43db08169b57dd1e982ba74e68326ae61ed8278ac4d5fed7eefe8f6979dcf5a3c3a3e98ceb0

data/Readme.md

@@ -84,7 +84,7 @@ end
  - `cp .env.example .env`
  - open [Datadog API Settings](https://app.datadoghq.com/account/settings#api)
  - create a `API Key` or get an existing one from an admin, then add it to `.env` as `DATADOG_API_KEY`
- - find or create (check last page) your personal "Application Key" and add it to `.env` as `DATADOG_APP_KEY=`
+ - open [Datadog API Settings](https://app.datadoghq.com/access/application-keys) and create a new key, then add it to `.env` as `DATADOG_APP_KEY=`
  - change the `DATADOG_SUBDOMAIN=app` in `.env` to your companies subdomain if you have one
  - verify it works by running `rake plan`, it might show some diff, but should not crash
 -->
@@ -212,14 +212,20 @@ removing the `id` will cause kennel to create a new resource in datadog.
 Some validations might be too strict for your usecase or just wrong, please [open an issue](https://github.com/grosser/kennel/issues) and
 to unblock use the `validate: -> { false }` option.
 
-### Linking with kennel_ids
+### Linking resources with kennel_id
 
-To link to existing monitors via their kennel_id `projects kennel_id` + `:` + `monitors kennel id`
+Link resources with their kennel_id in the format `project kennel_id` + `:` + `resource kennel_id`,
+this should be used to create dependent resources like monitor + slos,
+so they can be created in a single update and can be re-created if any of them is deleted.
 
- - Screens `uptime` widgets can use `monitor: {id: "foo:bar"}`
- - Screens `alert_graph` widgets can use `alert_id: "foo:bar"`
- - Monitors `composite` can use `query: -> { "%{foo:bar} || %{foo:baz}" }`
- - Slos can use `monitor_ids: -> ["foo:bar"]` 
+|Resource|Type|Syntax|
+|---|---|---|
+|Dashboard|uptime|`monitor: {id: "foo:bar"}`|
+|Dashboard|alert_graph|`alert_id: "foo:bar"`|
+|Dashboard|slo|`slo_id: "foo:bar"`|
+|Monitor|composite|`query: -> { "%{foo:bar} && %{foo:baz}" }`|
+|Monitor|slo alert|`query: -> { "error_budget(\"%{foo:bar}\").over(\"7d\") > 123.0" }`|
+|Slo|monitor|`monitor_ids: -> ["foo:bar"]`|
 
 ### Debugging changes locally
 

data/lib/kennel.rb

@@ -39,12 +39,7 @@ module Kennel
     attr_accessor :out, :err
 
     def generate
-      FileUtils.rm_rf("generated")
-      generated.each do |part|
-        path = "generated/#{part.tracking_id.sub(":", "/")}.json"
-        FileUtils.mkdir_p(File.dirname(path))
-        File.write(path, JSON.pretty_generate(part.as_json) << "\n")
-      end
+      store generated
     end
 
     def plan
@@ -58,6 +53,35 @@ module Kennel
 
     private
 
+    def store(parts)
+      Progress.progress "Storing" do
+        old = Dir["generated/**/*"]
+        used = []
+
+        Utils.parallel(parts, max: 2) do |part|
+          path = "generated/#{part.tracking_id.tr("/", ":").sub(":", "/")}.json"
+          used << File.dirname(path) # only 1 level of sub folders, so this is safe
+          used << path
+          write_file_if_necessary(path, JSON.pretty_generate(part.as_json) << "\n")
+        end
+
+        # deleting all is slow, so only delete the extras
+        (old - used).each { |p| FileUtils.rm_rf(p) }
+      end
+    end
+
+    def write_file_if_necessary(path, content)
+      # 99% case
+      begin
+        return if File.read(path) == content
+      rescue Errno::ENOENT
+        FileUtils.mkdir_p(File.dirname(path))
+      end
+
+      # slow 1% case
+      File.write(path, content)
+    end
+
     def syncer
       @syncer ||= Syncer.new(api, generated, project: ENV["PROJECT"])
     end
@@ -73,8 +97,12 @@ module Kennel
           parts = Models::Project.recursive_subclasses.flat_map do |project_class|
             project_class.new.validated_parts
           end
-          parts.map(&:tracking_id).group_by { |id| id }.select do |id, same|
-            raise "#{id} is defined #{same.size} times" if same.size != 1
+          parts.group_by(&:tracking_id).each do |tracking_id, same|
+            next if same.size == 1
+            raise <<~ERROR
+              #{tracking_id} is defined #{same.size} times
+              use a different `kennel_id` when defining multiple projects/monitors/dashboards to avoid this conflict
+            ERROR
           end
           parts
         end

data/lib/kennel/api.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 module Kennel
+  # encapsulates knowledge around how the api works
   class Api
+    CACHE_FILE = "tmp/cache/details"
+
     def initialize(app_key, api_key)
       @app_key = app_key
       @api_key = api_key
@@ -49,8 +52,27 @@ module Kennel
       request :delete, "/api/v1/#{api_resource}/#{id}", params: { force: "true" }, ignore_404: true
     end
 
+    def fill_details!(api_resource, list)
+      return unless api_resource == "dashboard"
+      details_cache do |cache|
+        Utils.parallel(list) { |a| fill_detail!(api_resource, a, cache) }
+      end
+    end
+
     private
 
+    # Make diff work even though we cannot mass-fetch definitions
+    def fill_detail!(api_resource, a, cache)
+      args = [api_resource, a.fetch(:id)]
+      full = cache.fetch(args, a.fetch(:modified_at)) { show(*args) }
+      a.merge!(full)
+    end
+
+    def details_cache(&block)
+      cache = FileCache.new CACHE_FILE, Kennel::VERSION
+      cache.open(&block)
+    end
+
     def request(method, path, body: nil, params: {}, ignore_404: false)
       params = params.merge(application_key: @app_key, api_key: @api_key)
       query = Faraday::FlatParamsEncoder.encode(params)

data/lib/kennel/models/dashboard.rb

@@ -5,7 +5,6 @@ module Kennel
       include TemplateVariables
       include OptionalValidations
 
-      API_LIST_INCOMPLETE = true
       DASHBOARD_DEFAULTS = { template_variables: [] }.freeze
       READONLY_ATTRIBUTES = superclass::READONLY_ATTRIBUTES + [
         :author_handle, :author_name, :modified_at, :url, :is_read_only, :notify_list
@@ -140,16 +139,16 @@ module Kennel
           when "uptime"
             if ids = definition[:monitor_ids]
               definition[:monitor_ids] = ids.map do |id|
-                tracking_id?(id) ? resolve_link(id, :monitor, id_map, **args) : id
+                tracking_id?(id) ? (resolve_link(id, :monitor, id_map, **args) || id) : id
               end
             end
           when "alert_graph"
             if (id = definition[:alert_id]) && tracking_id?(id)
-              definition[:alert_id] = resolve_link(id, :monitor, id_map, **args).to_s
+              definition[:alert_id] = (resolve_link(id, :monitor, id_map, **args) || id).to_s
             end
           when "slo"
             if (id = definition[:slo_id]) && tracking_id?(id)
-              definition[:slo_id] = resolve_link(id, :slo, id_map, **args).to_s
+              definition[:slo_id] = (resolve_link(id, :slo, id_map, **args) || id).to_s
             end
           end
         end
@@ -186,22 +185,26 @@ module Kennel
       end
 
       def render_definitions(definitions)
-        definitions.map do |title, type, display_type, queries, options = {}, ignored = nil|
-          # validate inputs
-          if ignored || (!title || !type || !queries || !options.is_a?(Hash))
-            raise ArgumentError, "Expected exactly 5 arguments for each definition (title, type, display_type, queries, options)"
-          end
-          if (SUPPORTED_DEFINITION_OPTIONS | options.keys) != SUPPORTED_DEFINITION_OPTIONS
-            raise ArgumentError, "Supported options are: #{SUPPORTED_DEFINITION_OPTIONS.map(&:inspect).join(", ")}"
-          end
+        definitions.map do |title, type, display_type, queries, options = {}, too_many_args = nil|
+          if title.is_a?(Hash) && !type
+            title # user gave a full widget, just use it
+          else
+            # validate inputs
+            if too_many_args || (!title || !type || !queries || !options.is_a?(Hash))
+              raise ArgumentError, "Expected exactly 5 arguments for each definition (title, type, display_type, queries, options)"
+            end
+            if (SUPPORTED_DEFINITION_OPTIONS | options.keys) != SUPPORTED_DEFINITION_OPTIONS
+              raise ArgumentError, "Supported options are: #{SUPPORTED_DEFINITION_OPTIONS.map(&:inspect).join(", ")}"
+            end
 
-          # build definition
-          requests = Array(queries).map do |q|
-            request = { q: q }
-            request[:display_type] = display_type if display_type
-            request
+            # build definition
+            requests = Array(queries).map do |q|
+              request = { q: q }
+              request[:display_type] = display_type if display_type
+              request
+            end
+            { definition: { title: title, type: type, requests: requests, **options } }
           end
-          { definition: { title: title, type: type, requests: requests, **options } }
         end
       end
     end

data/lib/kennel/models/monitor.rb

@@ -112,9 +112,11 @@ module Kennel
       end
 
       def resolve_linked_tracking_ids!(id_map, **args)
-        if as_json[:type] == "composite"
-          as_json[:query] = as_json[:query].gsub(/%\{(.*?)\}/) do
-            resolve_link($1, :monitor, id_map, **args)
+        case as_json[:type]
+        when "composite", "slo alert"
+          type = (as_json[:type] == "composite" ? :monitor : :slo)
+          as_json[:query] = as_json[:query].gsub(/%{(.*?)}/) do
+            resolve_link($1, type, id_map, **args) || $&
           end
         end
       end

data/lib/kennel/models/record.rb

@@ -6,7 +6,6 @@ module Kennel
       READONLY_ATTRIBUTES = [
         :deleted, :id, :created, :created_at, :creator, :org_id, :modified, :modified_at, :api_resource
       ].freeze
-      API_LIST_INCOMPLETE = false
 
       settings :id, :kennel_id
 
@@ -65,19 +64,18 @@ module Kennel
 
       private
 
-      def resolve_link(id, type, id_map, force:)
-        value = id_map[id]
-        if value == :new
+      def resolve_link(tracking_id, type, id_map, force:)
+        id = id_map[tracking_id]
+        if id == :new
           if force
-            # TODO: remove the need for this by sorting monitors by missing resolutions
-            invalid! "#{id} needs to already exist, try again"
+            invalid! "#{type} #{tracking_id} was referenced but is also created by the current run.\nIt could not be created because of a circular dependency, try creating only some of the resources"
           else
-            id # will be re-resolved by syncer after the linked object was created
+            nil # will be re-resolved after the linked object was created
           end
-        elsif value
-          value
+        elsif id
+          id
         else
-          invalid! "Unable to find #{type} #{id} (does not exist and is not being created by the current run)"
+          invalid! "Unable to find #{type} #{tracking_id} (does not exist and is not being created by the current run)"
         end
       end
 

data/lib/kennel/models/slo.rb

@@ -69,7 +69,7 @@ module Kennel
       def resolve_linked_tracking_ids!(id_map, **args)
         return unless as_json[:monitor_ids] # ignore_default can remove it
         as_json[:monitor_ids] = as_json[:monitor_ids].map do |id|
-          id.is_a?(String) ? resolve_link(id, :monitor, id_map, **args) : id
+          id.is_a?(String) ? (resolve_link(id, :monitor, id_map, **args) || id) : id
         end
       end
 

data/lib/kennel/syncer.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 module Kennel
   class Syncer
-    CACHE_FILE = "tmp/cache/details" # keep in sync with .travis.yml caching
     TRACKING_FIELDS = [:message, :description].freeze
     DELETE_ORDER = ["dashboard", "slo", "monitor"].freeze # dashboards references monitors + slos, slos reference monitors
 
@@ -38,23 +37,14 @@ module Kennel
     end
 
     def update
-      changed = (@create + @update).map { |_, e| e }
-
-      @create.each do |_, e|
-        e.resolve_linked_tracking_ids!({}, force: true)
-
+      each_resolved @create do |_, e|
         reply = @api.create e.class.api_resource, e.as_json
         id = reply.fetch(:id)
-
-        # resolve ids we could previously no resolve
-        changed.delete e
-        resolve_linked_tracking_ids! from: [reply], to: changed
-
+        populate_id_map [reply] # allow resolving ids we could previously no resolve
         Kennel.out.puts "Created #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.url(id)}"
       end
 
-      @update.each do |id, e|
-        e.resolve_linked_tracking_ids!({}, force: true)
+      each_resolved @update do |id, e|
         @api.update e.class.api_resource, id, e.as_json
         Kennel.out.puts "Updated #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.url(id)}"
       end
@@ -67,6 +57,37 @@ module Kennel
 
     private
 
+    # loop over items until everything is resolved or crash when we get stuck
+    # this solves cases like composite monitors depending on each other or monitor->monitor slo->slo monitor chains
+    def each_resolved(list)
+      list = list.dup
+      loop do
+        return if list.empty?
+        list.reject! do |id, e|
+          if resolved?(e)
+            yield id, e
+            true
+          else
+            false
+          end
+        end ||
+          assert_resolved(list[0][1]) # resolve something or show a circular dependency error
+      end
+    end
+
+    # TODO: optimize by storing an instance variable if already resolved
+    def resolved?(e)
+      assert_resolved e
+      true
+    rescue ValidationError
+      false
+    end
+
+    # raises ValidationError when not resolved
+    def assert_resolved(e)
+      resolve_linked_tracking_ids! [e], force: true
+    end
+
     def noop?
       @create.empty? && @update.empty? && @delete.empty?
     end
@@ -74,9 +95,15 @@ module Kennel
     def calculate_diff
       @update = []
       @delete = []
+      @id_map = {}
 
       actual = Progress.progress("Downloading definitions") { download_definitions }
-      resolve_linked_tracking_ids! from: actual, to: @expected
+
+      # resolve dependencies to avoid diff
+      populate_id_map actual
+      @expected.each { |e| @id_map[e.tracking_id] ||= :new }
+      resolve_linked_tracking_ids! @expected
+
       filter_by_project! actual
 
       Progress.progress "Diffing" do
@@ -89,10 +116,10 @@ module Kennel
           end
         end
 
-        details_cache do |cache|
-          # fill details of things we need to compare (only do this part in parallel for safety & balancing)
-          Utils.parallel(items.select { |e, _| e && e.class::API_LIST_INCOMPLETE }) { |_, a| fill_details(a, cache) }
-        end
+        # fill details of things we need to compare
+        detailed = Hash.new { |h, k| h[k] = [] }
+        items.each { |e, a| detailed[a[:api_resource]] << a if e }
+        detailed.each { |api_resource, actuals| @api.fill_details! api_resource, actuals }
 
         # pick out things to update or delete
         items.each do |e, a|
@@ -107,27 +134,11 @@ module Kennel
 
         ensure_all_ids_found
         @create = @expected.map { |e| [nil, e] }
-        @create.sort_by! { |_, e| -DELETE_ORDER.index(e.class.api_resource) }
       end
 
       @delete.sort_by! { |_, _, a| DELETE_ORDER.index a.fetch(:api_resource) }
     end
 
-    # Make diff work even though we cannot mass-fetch definitions
-    def fill_details(a, cache)
-      resource = a.fetch(:api_resource)
-      args = [resource, a.fetch(:id)]
-      full = cache.fetch(args, a[:modified] || a.fetch(:modified_at)) do
-        @api.show(*args)
-      end
-      a.merge!(full)
-    end
-
-    def details_cache(&block)
-      cache = FileCache.new CACHE_FILE, Kennel::VERSION
-      cache.open(&block)
-    end
-
     def download_definitions
       Utils.parallel(Models::Record.subclasses.map(&:api_resource)) do |api_resource|
         results = @api.list(api_resource, with_downtimes: false) # lookup monitors without adding unnecessary downtime information
@@ -214,10 +225,12 @@ module Kennel
       end
     end
 
-    def resolve_linked_tracking_ids!(from:, to:)
-      map = from.each_with_object({}) { |a, lookup| lookup[tracking_id(a)] = a.fetch(:id) }
-      to.each { |e| map[e.tracking_id] ||= :new }
-      to.each { |e| e.resolve_linked_tracking_ids!(map, force: false) }
+    def populate_id_map(actual)
+      actual.each { |a| @id_map[tracking_id(a)] = a.fetch(:id) }
+    end
+
+    def resolve_linked_tracking_ids!(list, force: false)
+      list.each { |e| e.resolve_linked_tracking_ids!(@id_map, force: force) }
     end
 
     def filter_by_project!(definitions)

data/lib/kennel/tasks.rb

@@ -132,8 +132,15 @@ namespace :kennel do
       else
         Kennel::Models::Record.subclasses.map(&:api_resource)
       end
+    api = Kennel.send(:api)
+    list = nil
+
     resources.each do |resource|
-      Kennel.send(:api).list(resource).each do |r|
+      Kennel::Progress.progress("Downloading #{resource}") do
+        list = api.list(resource)
+        api.fill_details!(resource, list)
+      end
+      list.each do |r|
         Kennel.out.puts JSON.pretty_generate(r)
       end
     end

data/lib/kennel/utils.rb

@@ -23,6 +23,11 @@ module Kennel
           .downcase
       end
 
+      # for child projects, not used internally
+      def title_case(string)
+        string.split(/[\s_]/).map(&:capitalize) * " "
+      end
+
       # simplified version of https://apidock.com/rails/ActiveSupport/Inflector/parameterize
       def parameterize(string)
         string

data/lib/kennel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Kennel
-  VERSION = "1.79.0"
+  VERSION = "1.82.0"
 end

data/template/Readme.md

@@ -67,7 +67,7 @@ end
  - `cp .env.example .env`
  - open [Datadog API Settings](https://app.datadoghq.com/account/settings#api)
  - create a `API Key` or get an existing one from an admin, then add it to `.env` as `DATADOG_API_KEY`
- - find or create (check last page) your personal "Application Key" and add it to `.env` as `DATADOG_APP_KEY=`
+ - open [Datadog API Settings](https://app.datadoghq.com/access/application-keys) and create a new key, then add it to `.env` as `DATADOG_APP_KEY=`
  - change the `DATADOG_SUBDOMAIN=app` in `.env` to your companies subdomain if you have one
  - verify it works by running `rake plan`, it might show some diff, but should not crash
 
@@ -194,14 +194,20 @@ removing the `id` will cause kennel to create a new resource in datadog.
 Some validations might be too strict for your usecase or just wrong, please [open an issue](https://github.com/grosser/kennel/issues) and
 to unblock use the `validate: -> { false }` option.
 
-### Linking with kennel_ids
+### Linking resources with kennel_id
 
-To link to existing monitors via their kennel_id `projects kennel_id` + `:` + `monitors kennel id`
+Link resources with their kennel_id in the format `project kennel_id` + `:` + `resource kennel_id`,
+this should be used to create dependent resources like monitor + slos,
+so they can be created in a single update and can be re-created if any of them is deleted.
 
- - Screens `uptime` widgets can use `monitor: {id: "foo:bar"}`
- - Screens `alert_graph` widgets can use `alert_id: "foo:bar"`
- - Monitors `composite` can use `query: -> { "%{foo:bar} || %{foo:baz}" }`
- - Slos can use `monitor_ids: -> ["foo:bar"]` 
+|Resource|Type|Syntax|
+|---|---|---|
+|Dashboard|uptime|`monitor: {id: "foo:bar"}`|
+|Dashboard|alert_graph|`alert_id: "foo:bar"`|
+|Dashboard|slo|`slo_id: "foo:bar"`|
+|Monitor|composite|`query: -> { "%{foo:bar} && %{foo:baz}" }`|
+|Monitor|slo alert|`query: -> { "error_budget(\"%{foo:bar}\").over(\"7d\") > 123.0" }`|
+|Slo|monitor|`monitor_ids: -> ["foo:bar"]`|
 
 ### Debugging changes locally
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kennel
 version: !ruby/object:Gem::Version
-  version: 1.79.0
+  version: 1.82.0
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-30 00:00:00.000000000 Z
+date: 2021-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday