kennel 1.89.0 → 1.89.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bdedd0f9bbcb9abf873cd2f9be0414700f77d2ea864e609db79773d1b3193e8e
-  data.tar.gz: 336875475665fac019c12daced64e474fb8e85e216e158d4328b62b287785e6a
+  metadata.gz: ba1d58935b541b89cb78318964470bb209933a491d9b81e23785de0fb6ab2654
+  data.tar.gz: aed4b13dc2c35ced80073e592fd0d2ba2afda8dfb763d2c6baa4b1f30851cf53
 SHA512:
-  metadata.gz: 0041e466ce433d82f9db8252829f8c64652815d7bf932ec93596d57af76dc13aaa9dfce963bbf82888e09d424937b597364954edc1cff14344143f0661e8e274
-  data.tar.gz: a286f227e597686085049ef0227b2e36b26d33ef41a7d96d13a535af73e1530254de6bf68ead11ebdd0c394d1562766a91a478c794ee17f35c288a7f102d46b7
+  metadata.gz: d3595608df6e40ec4241bf2b758f1ed1b3fde0e63dff7f5a4fa393a358cf9cb9fa33bb0c5f72899b7df54817cb9482366d9c387e3c21a33249e35e3bcc4fc8ea
+  data.tar.gz: 745ec8c762acc885a9d0f2ae23fc9f43ca05f56b8c580501075e2b9237e7199a09499053cebb45e5bfa477aa0a187cabf5899e05f7bf6c361f952347ab6cac34

data/Readme.md

@@ -52,6 +52,7 @@ end
 ```
 
 <!-- NOT IN template/Readme.md  -->
+
 ## Installation
 
  - create a new private `kennel` repo for your organization (do not fork this repo)
@@ -293,9 +294,12 @@ https://foo.datadog.com/monitor/123
 
 <!-- NOT IN template/Readme.md -->
 
-
 ## Development
 
+### Benchmarking
+
+Setting `FORCE_GET_CACHE=true` will cache all get requests, which makes benchmarking improvements more reliable.
+
 ### Integration testing
 
 ```Bash

data/lib/kennel/api.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
+# encapsulates knowledge around how the api works
+# especially 1-off weirdness that should not lak into other parts of the code
 module Kennel
-  # encapsulates knowledge around how the api works
   class Api
     CACHE_FILE = "tmp/cache/details"
 
@@ -11,34 +12,24 @@ module Kennel
     end
 
     def show(api_resource, id, params = {})
-      reply = request :get, "/api/v1/#{api_resource}/#{id}", params: params
-      api_resource == "slo" ? reply[:data] : reply
+      response = request :get, "/api/v1/#{api_resource}/#{id}", params: params
+      response = response.fetch(:data) if api_resource == "slo"
+      response
     end
 
     def list(api_resource, params = {})
-      if api_resource == "slo"
-        raise ArgumentError if params[:limit] || params[:offset]
-        limit = 1000
-        offset = 0
-        all = []
-
-        loop do
-          result = request :get, "/api/v1/#{api_resource}", params: params.merge(limit: limit, offset: offset)
-          data = result.fetch(:data)
-          all.concat data
-          break all if data.size < limit
-          offset += limit
-        end
-      else
-        result = request :get, "/api/v1/#{api_resource}", params: params
-        result = result.fetch(:dashboards) if api_resource == "dashboard"
-        result
+      with_pagination api_resource == "slo", params do |paginated_params|
+        response = request :get, "/api/v1/#{api_resource}", params: paginated_params
+        response = response.fetch(:dashboards) if api_resource == "dashboard"
+        response = response.fetch(:data) if api_resource == "slo"
+        response
       end
     end
 
     def create(api_resource, attributes)
-      reply = request :post, "/api/v1/#{api_resource}", body: attributes
-      api_resource == "slo" ? reply[:data].first : reply
+      response = request :post, "/api/v1/#{api_resource}", body: attributes
+      response = response.fetch(:data).first if api_resource == "slo"
+      response
     end
 
     def update(api_resource, id, attributes)
@@ -53,7 +44,6 @@ module Kennel
     end
 
     def fill_details!(api_resource, list)
-      return unless api_resource == "dashboard"
       details_cache do |cache|
         Utils.parallel(list) { |a| fill_detail!(api_resource, a, cache) }
       end
@@ -61,6 +51,21 @@ module Kennel
 
     private
 
+    def with_pagination(enabled, params)
+      return yield params unless enabled
+      raise ArgumentError if params[:limit] || params[:offset]
+      limit = 1000
+      offset = 0
+      all = []
+
+      loop do
+        response = yield params.merge(limit: limit, offset: offset)
+        all.concat response
+        return all if response.size < limit
+        offset += limit
+      end
+    end
+
     # Make diff work even though we cannot mass-fetch definitions
     def fill_detail!(api_resource, a, cache)
       args = [api_resource, a.fetch(:id)]
@@ -74,34 +79,52 @@ module Kennel
     end
 
     def request(method, path, body: nil, params: {}, ignore_404: false)
-      params = params.merge(application_key: @app_key, api_key: @api_key)
-      query = Faraday::FlatParamsEncoder.encode(params)
-      response = nil
-      tries = 2
-
-      tries.times do |i|
-        response = Utils.retry Faraday::ConnectionFailed, Faraday::TimeoutError, times: 2 do
-          @client.send(method, "#{path}?#{query}") do |request|
-            request.body = JSON.generate(body) if body
-            request.headers["Content-type"] = "application/json"
+      path = "#{path}?#{Faraday::FlatParamsEncoder.encode(params)}" if params.any?
+      with_cache ENV["FORCE_GET_CACHE"] && method == :get, path do
+        response = nil
+        tries = 2
+
+        tries.times do |i|
+          response = Utils.retry Faraday::ConnectionFailed, Faraday::TimeoutError, times: 2 do
+            @client.send(method, path) do |request|
+              request.body = JSON.generate(body) if body
+              request.headers["Content-type"] = "application/json"
+              request.headers["DD-API-KEY"] = @api_key
+              request.headers["DD-APPLICATION-KEY"] = @app_key
+            end
           end
+
+          break if i == tries - 1 || method != :get || response.status < 500
+          Kennel.err.puts "Retrying on server error #{response.status} for #{path}"
         end
 
-        break if i == tries - 1 || method != :get || response.status < 500
-        Kennel.err.puts "Retrying on server error #{response.status} for #{path}"
-      end
+        if !response.success? && (response.status != 404 || !ignore_404)
+          message = +"Error #{response.status} during #{method.upcase} #{path}\n"
+          message << "request:\n#{JSON.pretty_generate(body)}\nresponse:\n" if body
+          message << response.body
+          raise message
+        end
 
-      if !response.success? && (response.status != 404 || !ignore_404)
-        message = +"Error #{response.status} during #{method.upcase} #{path}\n"
-        message << "request:\n#{JSON.pretty_generate(body)}\nresponse:\n" if body
-        message << response.body
-        raise message
+        if response.body.empty?
+          {}
+        else
+          JSON.parse(response.body, symbolize_names: true)
+        end
       end
+    end
 
-      if response.body.empty?
-        {}
+    # allow caching all requests to speedup/benchmark logic that includes repeated requests
+    def with_cache(enabled, key)
+      return yield unless enabled
+      dir = "tmp/cache"
+      FileUtils.mkdir_p(dir) unless File.directory?(dir)
+      file = "#{dir}/#{key.delete("/?=")}" # TODO: encode nicely
+      if File.exist?(file)
+        Marshal.load(File.read(file)) # rubocop:disable Security/MarshalLoad
       else
-        JSON.parse(response.body, symbolize_names: true)
+        result = yield
+        File.write(file, Marshal.dump(result))
+        result
       end
     end
   end

data/lib/kennel/file_cache.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 # cache that reads everything from a single file
-# to avoid doing multiple disk reads while iterating all definitions
-# it also replaces updated keys and has an overall expiry to not keep deleted things forever
+# - avoids doing multiple disk reads while iterating all definitions
+# - has a global expiry to not keep deleted resources forever
 module Kennel
   class FileCache
     def initialize(file, cache_version)
@@ -22,10 +22,11 @@ module Kennel
 
     def fetch(key, key_version)
       old_value, old_version = @data[key]
-      return old_value if old_version == [key_version, @cache_version]
+      expected_version = [key_version, @cache_version]
+      return old_value if old_version == expected_version
 
       new_value = yield
-      @data[key] = [new_value, [key_version, @cache_version], @expires]
+      @data[key] = [new_value, expected_version, @expires]
       new_value
     end
 
@@ -46,8 +47,11 @@ module Kennel
       File.write(@file, Marshal.dump(@data))
     end
 
+    # keep the cache small to make loading it fast (5MB ~= 100ms)
+    # - delete expired keys
+    # - delete what would be deleted anyway when updating
     def expire_old_data
-      @data.reject! { |_, (_, _, ex)| ex < @now }
+      @data.reject! { |_, (_, (_, cv), expires)| expires < @now || cv != @cache_version }
     end
   end
 end

data/lib/kennel/importer.rb

@@ -113,8 +113,9 @@ module Kennel
         next if request[:formulas] && request[:formulas] != [{ formula: "query1" }]
         next if request[:queries]&.size != 1
         next if request[:queries].any? { |q| q[:data_source] != "metrics" }
+        next if widget.dig(:definition, :type) != request[:response_format]
         request.delete(:formulas)
-        request[:type] = request.delete(:response_format)
+        request.delete(:response_format)
         request[:q] = request.delete(:queries).first.fetch(:query)
       end
     end

data/lib/kennel/models/record.rb

@@ -6,7 +6,7 @@ module Kennel
       TRACKING_FIELDS = [:message, :description].freeze
       READONLY_ATTRIBUTES = [
         :deleted, :id, :created, :created_at, :creator, :org_id, :modified, :modified_at,
-        :klass # added by syncer.rb
+        :klass, :tracking_id # added by syncer.rb
       ].freeze
 
       settings :id, :kennel_id

data/lib/kennel/progress.rb

@@ -29,6 +29,8 @@ module Kennel
       Kennel.err.print "#{time.round(2)}s\n"
 
       result
+    ensure
+      stop = true
     end
   end
 end

data/lib/kennel/syncer.rb

@@ -2,21 +2,12 @@
 module Kennel
   class Syncer
     DELETE_ORDER = ["dashboard", "slo", "monitor"].freeze # dashboards references monitors + slos, slos reference monitors
-    LINE_UP = "\e[1A"
+    LINE_UP = "\e[1A\033[K" # go up and clear
 
     def initialize(api, expected, project: nil)
       @api = api
       @project_filter = project
       @expected = expected
-      if @project_filter
-        original = @expected
-        @expected = @expected.select { |e| e.project.kennel_id == @project_filter }
-        if @expected.empty?
-          possible = original.map { |e| e.project.kennel_id }.uniq.sort
-          raise "#{@project_filter} does not match any projects, try any of these:\n#{possible.join("\n")}"
-        end
-      end
-      @expected.each(&:add_tracking_id)
       calculate_diff
       prevent_irreversible_partial_updates
     end
@@ -41,9 +32,9 @@ module Kennel
         message = "#{e.class.api_resource} #{e.tracking_id}"
         Kennel.out.puts "Creating #{message}"
         reply = @api.create e.class.api_resource, e.as_json
-        reply[:klass] = e.class # store api resource class for later use
+        cache_metadata reply, e.class
         id = reply.fetch(:id)
-        populate_id_map [reply] # allow resolving ids we could previously no resolve
+        populate_id_map [], [reply] # allow resolving ids we could previously no resolve
         Kennel.out.puts "#{LINE_UP}Created #{message} #{e.class.url(id)}"
       end
 
@@ -56,8 +47,7 @@ module Kennel
 
       @delete.each do |id, _, a|
         klass = a.fetch(:klass)
-        tracking_id = klass.parse_tracking_id(a)
-        message = "#{klass.api_resource} #{tracking_id} #{id}"
+        message = "#{klass.api_resource} #{a.fetch(:tracking_id)} #{id}"
         Kennel.out.puts "Deleting #{message}"
         @api.delete klass.api_resource, id
         Kennel.out.puts "#{LINE_UP}Deleted #{message}"
@@ -108,14 +98,14 @@ module Kennel
 
       actual = Progress.progress("Downloading definitions") { download_definitions }
 
-      # resolve dependencies to avoid diff
-      populate_id_map actual
-      @expected.each { |e| @id_map[e.tracking_id] ||= :new }
-      resolve_linked_tracking_ids! @expected
+      Progress.progress "Diffing" do
+        filter_expected_by_project! @expected
+        populate_id_map @expected, actual
+        filter_actual_by_project! actual
+        resolve_linked_tracking_ids! @expected # resolve dependencies to avoid diff
 
-      filter_by_project! actual
+        @expected.each(&:add_tracking_id) # avoid diff with actual
 
-      Progress.progress "Diffing" do
         items = actual.map do |a|
           e = matching_expected(a)
           if e && @expected.delete(e)
@@ -126,9 +116,8 @@ module Kennel
         end
 
         # fill details of things we need to compare
-        detailed = Hash.new { |h, k| h[k] = [] }
-        items.each { |e, a| detailed[a[:klass]] << a if e }
-        detailed.each { |klass, actuals| @api.fill_details! klass.api_resource, actuals }
+        details = items.map { |e, a| a if e && e.class.api_resource == "dashboard" }.compact
+        @api.fill_details! "dashboard", details
 
         # pick out things to update or delete
         items.each do |e, a|
@@ -136,26 +125,30 @@ module Kennel
           if e
             diff = e.diff(a)
             @update << [id, e, a, diff] if diff.any?
-          elsif a.fetch(:klass).parse_tracking_id(a) # was previously managed
+          elsif a.fetch(:tracking_id) # was previously managed
             @delete << [id, nil, a]
           end
         end
 
         ensure_all_ids_found
         @create = @expected.map { |e| [nil, e] }
+        @delete.sort_by! { |_, _, a| DELETE_ORDER.index a.fetch(:klass).api_resource }
       end
-
-      @delete.sort_by! { |_, _, a| DELETE_ORDER.index a.fetch(:klass).api_resource }
     end
 
     def download_definitions
       Utils.parallel(Models::Record.subclasses) do |klass|
         results = @api.list(klass.api_resource, with_downtimes: false) # lookup monitors without adding unnecessary downtime information
         results = results[results.keys.first] if results.is_a?(Hash) # dashboards are nested in {dashboards: []}
-        results.each { |c| c[:klass] = klass } # store api resource for later diffing
+        results.each { |a| cache_metadata(a, klass) }
       end.flatten(1)
     end
 
+    def cache_metadata(a, klass)
+      a[:klass] = klass
+      a[:tracking_id] = a.fetch(:klass).parse_tracking_id(a)
+    end
+
     def ensure_all_ids_found
       @expected.each do |e|
         next unless id = e.id
@@ -176,14 +169,14 @@ module Kennel
       end
 
       klass = a.fetch(:klass)
-      @lookup_map["#{klass.api_resource}:#{a.fetch(:id)}"] || @lookup_map[klass.parse_tracking_id(a)]
+      @lookup_map["#{klass.api_resource}:#{a.fetch(:id)}"] || @lookup_map[a.fetch(:tracking_id)]
     end
 
     def print_plan(step, list, color)
       return if list.empty?
       list.each do |_, e, a, diff|
         klass = (e ? e.class : a.fetch(:klass))
-        Kennel.out.puts Utils.color(color, "#{step} #{klass.api_resource} #{e&.tracking_id || klass.parse_tracking_id(a)}")
+        Kennel.out.puts Utils.color(color, "#{step} #{klass.api_resource} #{e&.tracking_id || a.fetch(:tracking_id)}")
         print_diff(diff) if diff # only for update
       end
     end
@@ -232,19 +225,34 @@ module Kennel
       end
     end
 
-    def populate_id_map(actual)
-      actual.each { |a| @id_map[a.fetch(:klass).parse_tracking_id(a)] = a.fetch(:id) }
+    def populate_id_map(expected, actual)
+      actual.each do |a|
+        next unless tracking_id = a.fetch(:tracking_id)
+        @id_map[tracking_id] = a.fetch(:id)
+      end
+      expected.each { |e| @id_map[e.tracking_id] ||= :new }
     end
 
     def resolve_linked_tracking_ids!(list, force: false)
       list.each { |e| e.resolve_linked_tracking_ids!(@id_map, force: force) }
     end
 
-    def filter_by_project!(definitions)
+    def filter_actual_by_project!(actual)
+      return unless @project_filter
+      actual.select! do |a|
+        tracking_id = a.fetch(:tracking_id)
+        !tracking_id || tracking_id.start_with?("#{@project_filter}:")
+      end
+    end
+
+    def filter_expected_by_project!(expected)
       return unless @project_filter
-      definitions.select! do |a|
-        id = a.fetch(:klass).parse_tracking_id(a)
-        !id || id.start_with?("#{@project_filter}:")
+      original = expected.dup
+      expected.select! { |e| e.project.kennel_id == @project_filter }
+
+      if expected.empty?
+        possible = original.map { |e| e.project.kennel_id }.uniq.sort
+        raise "#{@project_filter} does not match any projects, try any of these:\n#{possible.join("\n")}"
       end
     end
   end

data/lib/kennel/tasks.rb

@@ -138,7 +138,7 @@ namespace :kennel do
     resources.each do |resource|
       Kennel::Progress.progress("Downloading #{resource}") do
         list = api.list(resource)
-        api.fill_details!(resource, list)
+        api.fill_details!(resource, list) if resource == "dashboard"
       end
       list.each do |r|
         r[:api_resource] = resource

data/lib/kennel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Kennel
-  VERSION = "1.89.0"
+  VERSION = "1.89.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kennel
 version: !ruby/object:Gem::Version
-  version: 1.89.0
+  version: 1.89.1
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-25 00:00:00.000000000 Z
+date: 2021-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday