kennel 1.81.1 → 1.84.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4ecb9c89748e6eeee268d0d744622b536242af1c1d7f615bf77fc5963e1630d8
-  data.tar.gz: e6cae4da66e0947163d1f441bce2afb19f1818dac0ed9418143b3f2a2480b220
+  metadata.gz: 69237d514f39fc9c8a1fad937bb038ce526424231fe4cc64ce3e69ac43820650
+  data.tar.gz: 7bf30b567652a4f6808073ff46487c7945570a6bb4c34afe60b60ad14bb30f80
 SHA512:
-  metadata.gz: 7a3342551a926a55c580571524cadbb32bf7b02b77a914f32e1edf2a07b6772091373286c36871d9651a57899134364d8c9c5894a8ef1e46f72ad091ca241990
-  data.tar.gz: 6eb25f647a3a7d47c0858ca03f0649e151f6437d711701ea24f1dd3b742b5adbfbd3f0318a531076b2236063dda194bbdec72237eaf3f1159f447ac86a95ee68
+  metadata.gz: 3cffb601fc240f3d6f87e41a369a772a0dce2c3de411d8eb290aa67ca89c30fbdde7d05c701e844cf12a92c398438c975175de2739129ad62d85f7c9eb667c30
+  data.tar.gz: 9bf61895741f6b762627f05a756355445fdd43bc488db04ae7a406a2fbc314311adf8e745ec87d4eb512cefd250d0cd3e6b777f3bb3d7cb7543e0cd6eb317ea1

data/Readme.md

@@ -216,7 +216,7 @@ to unblock use the `validate: -> { false }` option.
 
 Link resources with their kennel_id in the format `project kennel_id` + `:` + `resource kennel_id`,
 this should be used to create dependent resources like monitor + slos,
-so they can be created in a single update and can be re-created if any of them is deleted. 
+so they can be created in a single update and can be re-created if any of them is deleted.
 
 |Resource|Type|Syntax|
 |---|---|---|
@@ -273,9 +273,19 @@ Run `rake kennel:alerts TAG=service:my-service` to see all un-muted alerts for a
 
 ### Grepping through all of datadog
 
-`rake kennel:dump`
+```Bash
+rake kennel:dump > tmp/dump
+cat tmp/dump | grep foo
+```
 focus on a single type: `TYPE=monitors`
 
+Show full resources or just their urls by pattern:
+```Bash
+rake kennel:dump_grep DUMP=tmp/dump PATTERN=foo URLS=true
+https://foo.datadog.com/dasboard/123
+https://foo.datadog.com/monitor/123
+```
+
 ### Find all monitors with No-Data
 
 `rake kennel:nodata TAG=team:foo`

data/lib/kennel/api.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 module Kennel
+  # encapsulates knowledge around how the api works
   class Api
+    CACHE_FILE = "tmp/cache/details"
+
     def initialize(app_key, api_key)
       @app_key = app_key
       @api_key = api_key
@@ -49,8 +52,27 @@ module Kennel
       request :delete, "/api/v1/#{api_resource}/#{id}", params: { force: "true" }, ignore_404: true
     end
 
+    def fill_details!(api_resource, list)
+      return unless api_resource == "dashboard"
+      details_cache do |cache|
+        Utils.parallel(list) { |a| fill_detail!(api_resource, a, cache) }
+      end
+    end
+
     private
 
+    # Make diff work even though we cannot mass-fetch definitions
+    def fill_detail!(api_resource, a, cache)
+      args = [api_resource, a.fetch(:id)]
+      full = cache.fetch(args, a.fetch(:modified_at)) { show(*args) }
+      a.merge!(full)
+    end
+
+    def details_cache(&block)
+      cache = FileCache.new CACHE_FILE, Kennel::VERSION
+      cache.open(&block)
+    end
+
     def request(method, path, body: nil, params: {}, ignore_404: false)
       params = params.merge(application_key: @app_key, api_key: @api_key)
       query = Faraday::FlatParamsEncoder.encode(params)

data/lib/kennel/models/dashboard.rb

@@ -5,7 +5,6 @@ module Kennel
       include TemplateVariables
       include OptionalValidations
 
-      API_LIST_INCOMPLETE = true
       DASHBOARD_DEFAULTS = { template_variables: [] }.freeze
       READONLY_ATTRIBUTES = superclass::READONLY_ATTRIBUTES + [
         :author_handle, :author_name, :modified_at, :url, :is_read_only, :notify_list
@@ -124,7 +123,7 @@ module Kennel
         @json
       end
 
-      def url(id)
+      def self.url(id)
         Utils.path_to_url "/dashboard/#{id}"
       end
 
@@ -186,22 +185,26 @@ module Kennel
       end
 
       def render_definitions(definitions)
-        definitions.map do |title, type, display_type, queries, options = {}, ignored = nil|
-          # validate inputs
-          if ignored || (!title || !type || !queries || !options.is_a?(Hash))
-            raise ArgumentError, "Expected exactly 5 arguments for each definition (title, type, display_type, queries, options)"
-          end
-          if (SUPPORTED_DEFINITION_OPTIONS | options.keys) != SUPPORTED_DEFINITION_OPTIONS
-            raise ArgumentError, "Supported options are: #{SUPPORTED_DEFINITION_OPTIONS.map(&:inspect).join(", ")}"
-          end
+        definitions.map do |title, type, display_type, queries, options = {}, too_many_args = nil|
+          if title.is_a?(Hash) && !type
+            title # user gave a full widget, just use it
+          else
+            # validate inputs
+            if too_many_args || (!title || !type || !queries || !options.is_a?(Hash))
+              raise ArgumentError, "Expected exactly 5 arguments for each definition (title, type, display_type, queries, options)"
+            end
+            if (SUPPORTED_DEFINITION_OPTIONS | options.keys) != SUPPORTED_DEFINITION_OPTIONS
+              raise ArgumentError, "Supported options are: #{SUPPORTED_DEFINITION_OPTIONS.map(&:inspect).join(", ")}"
+            end
 
-          # build definition
-          requests = Array(queries).map do |q|
-            request = { q: q }
-            request[:display_type] = display_type if display_type
-            request
+            # build definition
+            requests = Array(queries).map do |q|
+              request = { q: q }
+              request[:display_type] = display_type if display_type
+              request
+            end
+            { definition: { title: title, type: type, requests: requests, **options } }
           end
-          { definition: { title: title, type: type, requests: requests, **options } }
         end
       end
     end

data/lib/kennel/models/monitor.rb

@@ -5,8 +5,6 @@ module Kennel
       include OptionalValidations
 
       RENOTIFY_INTERVALS = [0, 10, 20, 30, 40, 50, 60, 90, 120, 180, 240, 300, 360, 720, 1440].freeze # minutes
-      # 2d and 1w are valid timeframes for anomaly monitors
-      QUERY_INTERVALS = ["1m", "5m", "10m", "15m", "30m", "1h", "2h", "4h", "1d", "2d", "1w"].freeze
       OPTIONAL_SERVICE_CHECK_THRESHOLDS = [:ok, :warning].freeze
       READONLY_ATTRIBUTES = superclass::READONLY_ATTRIBUTES + [
         :multi, :matching_downtimes, :overall_state_modified, :overall_state, :restricted_roles
@@ -125,7 +123,7 @@ module Kennel
         "monitor"
       end
 
-      def url(id)
+      def self.url(id)
         Utils.path_to_url "/monitors##{id}/edit"
       end
 
@@ -202,14 +200,6 @@ module Kennel
           invalid! "renotify_interval must be one of #{RENOTIFY_INTERVALS.join(", ")}"
         end
 
-        if type == "query alert"
-          # verify interval is valid
-          interval = data.fetch(:query)[/\(last_(\S+?)\)/, 1]
-          if interval && !QUERY_INTERVALS.include?(interval)
-            invalid! "query interval was #{interval}, but must be one of #{QUERY_INTERVALS.join(", ")}"
-          end
-        end
-
         if ["query alert", "service check"].include?(type) # TODO: most likely more types need this
           # verify is_match/is_exact_match uses available variables
           message = data.fetch(:message)

data/lib/kennel/models/record.rb

@@ -6,7 +6,6 @@ module Kennel
       READONLY_ATTRIBUTES = [
         :deleted, :id, :created, :created_at, :creator, :org_id, :modified, :modified_at, :api_resource
       ].freeze
-      API_LIST_INCOMPLETE = false
 
       settings :id, :kennel_id
 
@@ -19,6 +18,10 @@ module Kennel
           end
         end
 
+        def api_resource_map
+          subclasses.map { |s| [s.api_resource, s] }.to_h
+        end
+
         private
 
         def normalize(_expected, actual)

data/lib/kennel/models/slo.rb

@@ -58,7 +58,7 @@ module Kennel
         "slo"
       end
 
-      def url(id)
+      def self.url(id)
         Utils.path_to_url "/slo?slo_id=#{id}"
       end
 

data/lib/kennel/syncer.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 module Kennel
   class Syncer
-    CACHE_FILE = "tmp/cache/details" # keep in sync with .travis.yml caching
     TRACKING_FIELDS = [:message, :description].freeze
     DELETE_ORDER = ["dashboard", "slo", "monitor"].freeze # dashboards references monitors + slos, slos reference monitors
 
@@ -42,12 +41,12 @@ module Kennel
         reply = @api.create e.class.api_resource, e.as_json
         id = reply.fetch(:id)
         populate_id_map [reply] # allow resolving ids we could previously no resolve
-        Kennel.out.puts "Created #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.url(id)}"
+        Kennel.out.puts "Created #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.class.url(id)}"
       end
 
       each_resolved @update do |id, e|
         @api.update e.class.api_resource, id, e.as_json
-        Kennel.out.puts "Updated #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.url(id)}"
+        Kennel.out.puts "Updated #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.class.url(id)}"
       end
 
       @delete.each do |id, _, a|
@@ -117,10 +116,10 @@ module Kennel
           end
         end
 
-        details_cache do |cache|
-          # fill details of things we need to compare (only do this part in parallel for safety & balancing)
-          Utils.parallel(items.select { |e, _| e && e.class::API_LIST_INCOMPLETE }) { |_, a| fill_details(a, cache) }
-        end
+        # fill details of things we need to compare
+        detailed = Hash.new { |h, k| h[k] = [] }
+        items.each { |e, a| detailed[a[:api_resource]] << a if e }
+        detailed.each { |api_resource, actuals| @api.fill_details! api_resource, actuals }
 
         # pick out things to update or delete
         items.each do |e, a|
@@ -140,21 +139,6 @@ module Kennel
       @delete.sort_by! { |_, _, a| DELETE_ORDER.index a.fetch(:api_resource) }
     end
 
-    # Make diff work even though we cannot mass-fetch definitions
-    def fill_details(a, cache)
-      resource = a.fetch(:api_resource)
-      args = [resource, a.fetch(:id)]
-      full = cache.fetch(args, a[:modified] || a.fetch(:modified_at)) do
-        @api.show(*args)
-      end
-      a.merge!(full)
-    end
-
-    def details_cache(&block)
-      cache = FileCache.new CACHE_FILE, Kennel::VERSION
-      cache.open(&block)
-    end
-
     def download_definitions
       Utils.parallel(Models::Record.subclasses.map(&:api_resource)) do |api_resource|
         results = @api.list(api_resource, with_downtimes: false) # lookup monitors without adding unnecessary downtime information

data/lib/kennel/tasks.rb

@@ -130,15 +130,43 @@ namespace :kennel do
       if type = ENV["TYPE"]
         [type]
       else
-        Kennel::Models::Record.subclasses.map(&:api_resource)
+        Kennel::Models::Record.api_resource_map.keys
       end
+    api = Kennel.send(:api)
+    list = nil
+
     resources.each do |resource|
-      Kennel.send(:api).list(resource).each do |r|
+      Kennel::Progress.progress("Downloading #{resource}") do
+        list = api.list(resource)
+        api.fill_details!(resource, list)
+      end
+      list.each do |r|
+        r[:api_resource] = resource
         Kennel.out.puts JSON.pretty_generate(r)
       end
     end
   end
 
+  desc "Find items from dump by pattern DUMP= PATTERN= [URLS=true]"
+  task dump_grep: :environment do
+    file = ENV.fetch("DUMP")
+    pattern = Regexp.new ENV.fetch("PATTERN")
+    items = File.read(file).gsub("}\n{", "}--SPLIT--{").split("--SPLIT--")
+    models = Kennel::Models::Record.api_resource_map
+    found = items.grep(pattern)
+    exit 1 if found.empty?
+    found.each do |resource|
+      if ENV["URLS"]
+        parsed = JSON.parse(resource)
+        url = models[parsed.fetch("api_resource")].url(parsed.fetch("id"))
+        title = parsed["title"] || parsed["name"]
+        Kennel.out.puts "#{url} # #{title}"
+      else
+        Kennel.out.puts resource
+      end
+    end
+  end
+
   task :environment do
     require "kennel"
     gem "dotenv"

data/lib/kennel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Kennel
-  VERSION = "1.81.1"
+  VERSION = "1.84.1"
 end

data/template/Readme.md

@@ -194,10 +194,11 @@ removing the `id` will cause kennel to create a new resource in datadog.
 Some validations might be too strict for your usecase or just wrong, please [open an issue](https://github.com/grosser/kennel/issues) and
 to unblock use the `validate: -> { false }` option.
 
-### Linking with kennel_ids
+### Linking resources with kennel_id
 
-Link resources via their kennel_id `projects kennel_id` + `:` + `monitors kennel id`,
-this should be used to create dependent resources like monitor + slos. 
+Link resources with their kennel_id in the format `project kennel_id` + `:` + `resource kennel_id`,
+this should be used to create dependent resources like monitor + slos,
+so they can be created in a single update and can be re-created if any of them is deleted.
 
 |Resource|Type|Syntax|
 |---|---|---|
@@ -254,9 +255,19 @@ Run `rake kennel:alerts TAG=service:my-service` to see all un-muted alerts for a
 
 ### Grepping through all of datadog
 
-`rake kennel:dump`
+```Bash
+rake kennel:dump > tmp/dump
+cat tmp/dump | grep foo
+```
 focus on a single type: `TYPE=monitors`
 
+Show full resources or just their urls by pattern:
+```Bash
+rake kennel:dump_grep DUMP=tmp/dump PATTERN=foo URLS=true
+https://foo.datadog.com/dasboard/123
+https://foo.datadog.com/monitor/123
+```
+
 ### Find all monitors with No-Data
 
 `rake kennel:nodata TAG=team:foo`

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kennel
 version: !ruby/object:Gem::Version
-  version: 1.81.1
+  version: 1.84.1
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-21 00:00:00.000000000 Z
+date: 2021-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday