kennel 1.89.0 → 1.91.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bdedd0f9bbcb9abf873cd2f9be0414700f77d2ea864e609db79773d1b3193e8e
-  data.tar.gz: 336875475665fac019c12daced64e474fb8e85e216e158d4328b62b287785e6a
+  metadata.gz: 9f3417e3f9c4f8b950c89047ed25c5f44a0da8e8c7f28667f33b549b2d6fadc3
+  data.tar.gz: 9fec0221be92d34d7ade923a9f92bee18e9f602d719c041e21fc274fb6d10c9f
 SHA512:
-  metadata.gz: 0041e466ce433d82f9db8252829f8c64652815d7bf932ec93596d57af76dc13aaa9dfce963bbf82888e09d424937b597364954edc1cff14344143f0661e8e274
-  data.tar.gz: a286f227e597686085049ef0227b2e36b26d33ef41a7d96d13a535af73e1530254de6bf68ead11ebdd0c394d1562766a91a478c794ee17f35c288a7f102d46b7
+  metadata.gz: b31a95230656072e45d260aa549010ad5add9313af68f2490ac3d3c2220b9fe525ee6af144a3ca988103503af7f4a15cdbeb52601e1ff0427a5809714807be73
+  data.tar.gz: 6754253f9cc10a00b663947860527a596308856e3beca48538eef1f19f9e02aa11c98ce981b5f7f7815bb177f4b329e971decf0dd8326ccc6a767d163c2970de

data/Readme.md

@@ -52,6 +52,7 @@ end
 ```
 
 <!-- NOT IN template/Readme.md  -->
+
 ## Installation
 
  - create a new private `kennel` repo for your organization (do not fork this repo)
@@ -293,9 +294,12 @@ https://foo.datadog.com/monitor/123
 
 <!-- NOT IN template/Readme.md -->
 
-
 ## Development
 
+### Benchmarking
+
+Setting `FORCE_GET_CACHE=true` will cache all get requests, which makes benchmarking improvements more reliable.
+
 ### Integration testing
 
 ```Bash

data/lib/kennel.rb

@@ -23,6 +23,7 @@ require "kennel/models/record"
 require "kennel/models/dashboard"
 require "kennel/models/monitor"
 require "kennel/models/slo"
+require "kennel/models/synthetic_test"
 
 # settings
 require "kennel/models/project"
@@ -55,7 +56,7 @@ module Kennel
 
     def store(parts)
       Progress.progress "Storing" do
-        old = Dir["generated/**/*"]
+        old = Dir["generated/#{project_filter || "**"}/*"]
         used = []
 
         Utils.parallel(parts, max: 2) do |part|
@@ -83,7 +84,7 @@ module Kennel
     end
 
     def syncer
-      @syncer ||= Syncer.new(api, generated, project: ENV["PROJECT"])
+      @syncer ||= Syncer.new(api, generated, project: project_filter)
     end
 
     def api
@@ -94,9 +95,21 @@ module Kennel
       @generated ||= begin
         Progress.progress "Generating" do
           load_all
+          known = []
           parts = Models::Project.recursive_subclasses.flat_map do |project_class|
-            project_class.new.validated_parts
+            project = project_class.new
+            kennel_id = project.kennel_id
+            if project_filter
+              known << kennel_id
+              next [] if kennel_id != project_filter
+            end
+            project.validated_parts
           end
+
+          if project_filter && parts.empty?
+            raise "#{project_filter} does not match any projects, try any of these:\n#{known.uniq.sort.join("\n")}"
+          end
+
           parts.group_by(&:tracking_id).each do |tracking_id, same|
             next if same.size == 1
             raise <<~ERROR
@@ -109,6 +122,10 @@ module Kennel
       end
     end
 
+    def project_filter
+      ENV["PROJECT"]
+    end
+
     def load_all
       ["teams", "parts", "projects"].each do |folder|
         Dir["#{folder}/**/*.rb"].sort.each { |f| require "./#{f}" }

data/lib/kennel/api.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
+# encapsulates knowledge around how the api works
+# especially 1-off weirdness that should not lak into other parts of the code
 module Kennel
-  # encapsulates knowledge around how the api works
   class Api
     CACHE_FILE = "tmp/cache/details"
 
@@ -11,49 +12,55 @@ module Kennel
     end
 
     def show(api_resource, id, params = {})
-      reply = request :get, "/api/v1/#{api_resource}/#{id}", params: params
-      api_resource == "slo" ? reply[:data] : reply
+      response = request :get, "/api/v1/#{api_resource}/#{id}", params: params
+      response = response.fetch(:data) if api_resource == "slo"
+      response[:id] = response.delete(:public_id) if api_resource == "synthetics/tests"
+      response
     end
 
     def list(api_resource, params = {})
-      if api_resource == "slo"
-        raise ArgumentError if params[:limit] || params[:offset]
-        limit = 1000
-        offset = 0
-        all = []
-
-        loop do
-          result = request :get, "/api/v1/#{api_resource}", params: params.merge(limit: limit, offset: offset)
-          data = result.fetch(:data)
-          all.concat data
-          break all if data.size < limit
-          offset += limit
+      with_pagination api_resource == "slo", params do |paginated_params|
+        response = request :get, "/api/v1/#{api_resource}", params: paginated_params
+        response = response.fetch(:dashboards) if api_resource == "dashboard"
+        response = response.fetch(:data) if api_resource == "slo"
+        if api_resource == "synthetics/tests"
+          response = response.fetch(:tests)
+          response.each { |r| r[:id] = r.delete(:public_id) }
         end
-      else
-        result = request :get, "/api/v1/#{api_resource}", params: params
-        result = result.fetch(:dashboards) if api_resource == "dashboard"
-        result
+
+        # ignore monitor synthetics create and that inherit the kennel_id, we do not directly manage them
+        response.reject! { |m| m[:type] == "synthetics alert" } if api_resource == "monitor"
+
+        response
       end
     end
 
     def create(api_resource, attributes)
-      reply = request :post, "/api/v1/#{api_resource}", body: attributes
-      api_resource == "slo" ? reply[:data].first : reply
+      response = request :post, "/api/v1/#{api_resource}", body: attributes
+      response = response.fetch(:data).first if api_resource == "slo"
+      response[:id] = response.delete(:public_id) if api_resource == "synthetics/tests"
+      response
     end
 
     def update(api_resource, id, attributes)
-      request :put, "/api/v1/#{api_resource}/#{id}", body: attributes
+      response = request :put, "/api/v1/#{api_resource}/#{id}", body: attributes
+      response[:id] = response.delete(:public_id) if api_resource == "synthetics/tests"
+      response
     end
 
     # - force=true to not dead-lock on dependent monitors+slos
     #   external dependency on kennel managed resources is their problem, we don't block on it
     #   (?force=true did not work, force for dashboard is not documented but does not blow up)
     def delete(api_resource, id)
-      request :delete, "/api/v1/#{api_resource}/#{id}", params: { force: "true" }, ignore_404: true
+      if api_resource == "synthetics/tests"
+        # https://docs.datadoghq.com/api/latest/synthetics/#delete-tests
+        request :post, "/api/v1/#{api_resource}/delete", body: { public_ids: [id] }, ignore_404: true
+      else
+        request :delete, "/api/v1/#{api_resource}/#{id}", params: { force: "true" }, ignore_404: true
+      end
     end
 
     def fill_details!(api_resource, list)
-      return unless api_resource == "dashboard"
       details_cache do |cache|
         Utils.parallel(list) { |a| fill_detail!(api_resource, a, cache) }
       end
@@ -61,6 +68,21 @@ module Kennel
 
     private
 
+    def with_pagination(enabled, params)
+      return yield params unless enabled
+      raise ArgumentError if params[:limit] || params[:offset]
+      limit = 1000
+      offset = 0
+      all = []
+
+      loop do
+        response = yield params.merge(limit: limit, offset: offset)
+        all.concat response
+        return all if response.size < limit
+        offset += limit
+      end
+    end
+
     # Make diff work even though we cannot mass-fetch definitions
     def fill_detail!(api_resource, a, cache)
       args = [api_resource, a.fetch(:id)]
@@ -74,34 +96,52 @@ module Kennel
     end
 
     def request(method, path, body: nil, params: {}, ignore_404: false)
-      params = params.merge(application_key: @app_key, api_key: @api_key)
-      query = Faraday::FlatParamsEncoder.encode(params)
-      response = nil
-      tries = 2
-
-      tries.times do |i|
-        response = Utils.retry Faraday::ConnectionFailed, Faraday::TimeoutError, times: 2 do
-          @client.send(method, "#{path}?#{query}") do |request|
-            request.body = JSON.generate(body) if body
-            request.headers["Content-type"] = "application/json"
+      path = "#{path}?#{Faraday::FlatParamsEncoder.encode(params)}" if params.any?
+      with_cache ENV["FORCE_GET_CACHE"] && method == :get, path do
+        response = nil
+        tries = 2
+
+        tries.times do |i|
+          response = Utils.retry Faraday::ConnectionFailed, Faraday::TimeoutError, times: 2 do
+            @client.send(method, path) do |request|
+              request.body = JSON.generate(body) if body
+              request.headers["Content-type"] = "application/json"
+              request.headers["DD-API-KEY"] = @api_key
+              request.headers["DD-APPLICATION-KEY"] = @app_key
+            end
           end
+
+          break if i == tries - 1 || method != :get || response.status < 500
+          Kennel.err.puts "Retrying on server error #{response.status} for #{path}"
         end
 
-        break if i == tries - 1 || method != :get || response.status < 500
-        Kennel.err.puts "Retrying on server error #{response.status} for #{path}"
-      end
+        if !response.success? && (response.status != 404 || !ignore_404)
+          message = +"Error #{response.status} during #{method.upcase} #{path}\n"
+          message << "request:\n#{JSON.pretty_generate(body)}\nresponse:\n" if body
+          message << response.body
+          raise message
+        end
 
-      if !response.success? && (response.status != 404 || !ignore_404)
-        message = +"Error #{response.status} during #{method.upcase} #{path}\n"
-        message << "request:\n#{JSON.pretty_generate(body)}\nresponse:\n" if body
-        message << response.body
-        raise message
+        if response.body.empty?
+          {}
+        else
+          JSON.parse(response.body, symbolize_names: true)
+        end
       end
+    end
 
-      if response.body.empty?
-        {}
+    # allow caching all requests to speedup/benchmark logic that includes repeated requests
+    def with_cache(enabled, key)
+      return yield unless enabled
+      dir = "tmp/cache"
+      FileUtils.mkdir_p(dir) unless File.directory?(dir)
+      file = "#{dir}/#{key.delete("/?=")}" # TODO: encode nicely
+      if File.exist?(file)
+        Marshal.load(File.read(file)) # rubocop:disable Security/MarshalLoad
       else
-        JSON.parse(response.body, symbolize_names: true)
+        result = yield
+        File.write(file, Marshal.dump(result))
+        result
       end
     end
   end

data/lib/kennel/file_cache.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 # cache that reads everything from a single file
-# to avoid doing multiple disk reads while iterating all definitions
-# it also replaces updated keys and has an overall expiry to not keep deleted things forever
+# - avoids doing multiple disk reads while iterating all definitions
+# - has a global expiry to not keep deleted resources forever
 module Kennel
   class FileCache
     def initialize(file, cache_version)
@@ -22,10 +22,11 @@ module Kennel
 
     def fetch(key, key_version)
       old_value, old_version = @data[key]
-      return old_value if old_version == [key_version, @cache_version]
+      expected_version = [key_version, @cache_version]
+      return old_value if old_version == expected_version
 
       new_value = yield
-      @data[key] = [new_value, [key_version, @cache_version], @expires]
+      @data[key] = [new_value, expected_version, @expires]
       new_value
     end
 
@@ -46,8 +47,11 @@ module Kennel
       File.write(@file, Marshal.dump(@data))
     end
 
+    # keep the cache small to make loading it fast (5MB ~= 100ms)
+    # - delete expired keys
+    # - delete what would be deleted anyway when updating
     def expire_old_data
-      @data.reject! { |_, (_, _, ex)| ex < @now }
+      @data.reject! { |_, (_, (_, cv), expires)| expires < @now || cv != @cache_version }
     end
   end
 end

data/lib/kennel/importer.rb

@@ -15,11 +15,8 @@ module Kennel
       end
 
       model =
-        begin
-          Kennel::Models.const_get(resource.capitalize)
-        rescue NameError
-          raise ArgumentError, "#{resource} is not supported"
-        end
+        Kennel::Models::Record.subclasses.detect { |c| c.api_resource == resource } ||
+        raise(ArgumentError, "#{resource} is not supported")
 
       data = @api.show(model.api_resource, id)
       id = data.fetch(:id) # keep native value
@@ -70,6 +67,8 @@ module Kennel
           dry_up_widget_metadata!(widget)
           (widget.dig(:definition, :markers) || []).each { |m| m[:label]&.delete! " " }
         end
+      when "synthetics/tests"
+        data[:locations] = :all if data[:locations].sort == Kennel::Models::SyntheticTest::LOCATIONS.sort
       else
         # noop
       end
@@ -113,8 +112,9 @@ module Kennel
         next if request[:formulas] && request[:formulas] != [{ formula: "query1" }]
         next if request[:queries]&.size != 1
         next if request[:queries].any? { |q| q[:data_source] != "metrics" }
+        next if widget.dig(:definition, :type) != request[:response_format]
         request.delete(:formulas)
-        request[:type] = request.delete(:response_format)
+        request.delete(:response_format)
         request[:q] = request.delete(:queries).first.fetch(:query)
       end
     end

data/lib/kennel/models/record.rb

@@ -6,7 +6,7 @@ module Kennel
       TRACKING_FIELDS = [:message, :description].freeze
       READONLY_ATTRIBUTES = [
         :deleted, :id, :created, :created_at, :creator, :org_id, :modified, :modified_at,
-        :klass # added by syncer.rb
+        :klass, :tracking_id # added by syncer.rb
       ].freeze
 
       settings :id, :kennel_id

data/lib/kennel/models/synthetic_test.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+module Kennel
+  module Models
+    class SyntheticTest < Record
+      TRACKING_FIELD = :message
+      DEFAULTS = {
+      }.freeze
+      READONLY_ATTRIBUTES = superclass::READONLY_ATTRIBUTES + [:status, :monitor_id]
+      LOCATIONS = ["aws:ca-central-1", "aws:eu-north-1", "aws:eu-west-1", "aws:eu-west-3", "aws:eu-west-2", "aws:ap-south-1", "aws:us-west-2", "aws:us-west-1", "aws:sa-east-1", "aws:us-east-2", "aws:ap-northeast-1", "aws:ap-northeast-2", "aws:eu-central-1", "aws:ap-southeast-2", "aws:ap-southeast-1"].freeze
+
+      settings :tags, :config, :message, :subtype, :type, :name, :locations, :options
+
+      defaults(
+        id: -> { nil },
+        tags: -> { @project.tags },
+        message: -> { "\n\n#{project.mention}" }
+      )
+
+      def as_json
+        return @as_json if @as_json
+        locations = locations()
+        data = {
+          message: message,
+          tags: tags,
+          config: config,
+          type: type,
+          subtype: subtype,
+          options: options,
+          name: name,
+          locations: locations == :all ? LOCATIONS : locations
+        }
+
+        if v = id
+          data[:id] = v
+        end
+
+        @as_json = data
+      end
+
+      def self.api_resource
+        "synthetics/tests"
+      end
+
+      def self.url(id)
+        Utils.path_to_url "/synthetics/details/#{id}"
+      end
+
+      def self.parse_url(url)
+        url[/\/synthetics\/details\/([a-z\d-]{11,})/, 1] # id format is 1ab-2ab-3ab
+      end
+
+      def self.normalize(expected, actual)
+        super
+
+        # tags come in a semi-random order and order is never updated
+        expected[:tags]&.sort!
+        actual[:tags].sort!
+
+        ignore_default(expected, actual, DEFAULTS)
+      end
+    end
+  end
+end

data/lib/kennel/progress.rb

@@ -29,6 +29,8 @@ module Kennel
       Kennel.err.print "#{time.round(2)}s\n"
 
       result
+    ensure
+      stop = true
     end
   end
 end

data/lib/kennel/syncer.rb

@@ -1,22 +1,13 @@
 # frozen_string_literal: true
 module Kennel
   class Syncer
-    DELETE_ORDER = ["dashboard", "slo", "monitor"].freeze # dashboards references monitors + slos, slos reference monitors
-    LINE_UP = "\e[1A"
+    DELETE_ORDER = ["dashboard", "slo", "monitor", "synthetics/tests"].freeze # dashboards references monitors + slos, slos reference monitors
+    LINE_UP = "\e[1A\033[K" # go up and clear
 
     def initialize(api, expected, project: nil)
       @api = api
       @project_filter = project
       @expected = expected
-      if @project_filter
-        original = @expected
-        @expected = @expected.select { |e| e.project.kennel_id == @project_filter }
-        if @expected.empty?
-          possible = original.map { |e| e.project.kennel_id }.uniq.sort
-          raise "#{@project_filter} does not match any projects, try any of these:\n#{possible.join("\n")}"
-        end
-      end
-      @expected.each(&:add_tracking_id)
       calculate_diff
       prevent_irreversible_partial_updates
     end
@@ -41,9 +32,9 @@ module Kennel
         message = "#{e.class.api_resource} #{e.tracking_id}"
         Kennel.out.puts "Creating #{message}"
         reply = @api.create e.class.api_resource, e.as_json
-        reply[:klass] = e.class # store api resource class for later use
+        cache_metadata reply, e.class
         id = reply.fetch(:id)
-        populate_id_map [reply] # allow resolving ids we could previously no resolve
+        populate_id_map [], [reply] # allow resolving ids we could previously no resolve
         Kennel.out.puts "#{LINE_UP}Created #{message} #{e.class.url(id)}"
       end
 
@@ -56,8 +47,7 @@ module Kennel
 
       @delete.each do |id, _, a|
         klass = a.fetch(:klass)
-        tracking_id = klass.parse_tracking_id(a)
-        message = "#{klass.api_resource} #{tracking_id} #{id}"
+        message = "#{klass.api_resource} #{a.fetch(:tracking_id)} #{id}"
         Kennel.out.puts "Deleting #{message}"
         @api.delete klass.api_resource, id
         Kennel.out.puts "#{LINE_UP}Deleted #{message}"
@@ -108,14 +98,13 @@ module Kennel
 
       actual = Progress.progress("Downloading definitions") { download_definitions }
 
-      # resolve dependencies to avoid diff
-      populate_id_map actual
-      @expected.each { |e| @id_map[e.tracking_id] ||= :new }
-      resolve_linked_tracking_ids! @expected
+      Progress.progress "Diffing" do
+        populate_id_map @expected, actual
+        filter_actual_by_project! actual
+        resolve_linked_tracking_ids! @expected # resolve dependencies to avoid diff
 
-      filter_by_project! actual
+        @expected.each(&:add_tracking_id) # avoid diff with actual
 
-      Progress.progress "Diffing" do
         items = actual.map do |a|
           e = matching_expected(a)
           if e && @expected.delete(e)
@@ -126,9 +115,8 @@ module Kennel
         end
 
         # fill details of things we need to compare
-        detailed = Hash.new { |h, k| h[k] = [] }
-        items.each { |e, a| detailed[a[:klass]] << a if e }
-        detailed.each { |klass, actuals| @api.fill_details! klass.api_resource, actuals }
+        details = items.map { |e, a| a if e && e.class.api_resource == "dashboard" }.compact
+        @api.fill_details! "dashboard", details
 
         # pick out things to update or delete
         items.each do |e, a|
@@ -136,26 +124,30 @@ module Kennel
           if e
             diff = e.diff(a)
             @update << [id, e, a, diff] if diff.any?
-          elsif a.fetch(:klass).parse_tracking_id(a) # was previously managed
+          elsif a.fetch(:tracking_id) # was previously managed
             @delete << [id, nil, a]
           end
         end
 
         ensure_all_ids_found
         @create = @expected.map { |e| [nil, e] }
+        @delete.sort_by! { |_, _, a| DELETE_ORDER.index a.fetch(:klass).api_resource }
       end
-
-      @delete.sort_by! { |_, _, a| DELETE_ORDER.index a.fetch(:klass).api_resource }
     end
 
     def download_definitions
       Utils.parallel(Models::Record.subclasses) do |klass|
         results = @api.list(klass.api_resource, with_downtimes: false) # lookup monitors without adding unnecessary downtime information
         results = results[results.keys.first] if results.is_a?(Hash) # dashboards are nested in {dashboards: []}
-        results.each { |c| c[:klass] = klass } # store api resource for later diffing
+        results.each { |a| cache_metadata(a, klass) }
       end.flatten(1)
     end
 
+    def cache_metadata(a, klass)
+      a[:klass] = klass
+      a[:tracking_id] = a.fetch(:klass).parse_tracking_id(a)
+    end
+
     def ensure_all_ids_found
       @expected.each do |e|
         next unless id = e.id
@@ -176,14 +168,14 @@ module Kennel
       end
 
       klass = a.fetch(:klass)
-      @lookup_map["#{klass.api_resource}:#{a.fetch(:id)}"] || @lookup_map[klass.parse_tracking_id(a)]
+      @lookup_map["#{klass.api_resource}:#{a.fetch(:id)}"] || @lookup_map[a.fetch(:tracking_id)]
     end
 
     def print_plan(step, list, color)
       return if list.empty?
       list.each do |_, e, a, diff|
         klass = (e ? e.class : a.fetch(:klass))
-        Kennel.out.puts Utils.color(color, "#{step} #{klass.api_resource} #{e&.tracking_id || klass.parse_tracking_id(a)}")
+        Kennel.out.puts Utils.color(color, "#{step} #{klass.api_resource} #{e&.tracking_id || a.fetch(:tracking_id)}")
         print_diff(diff) if diff # only for update
       end
     end
@@ -232,19 +224,23 @@ module Kennel
       end
     end
 
-    def populate_id_map(actual)
-      actual.each { |a| @id_map[a.fetch(:klass).parse_tracking_id(a)] = a.fetch(:id) }
+    def populate_id_map(expected, actual)
+      actual.each do |a|
+        next unless tracking_id = a.fetch(:tracking_id)
+        @id_map[tracking_id] = a.fetch(:id)
+      end
+      expected.each { |e| @id_map[e.tracking_id] ||= :new }
     end
 
     def resolve_linked_tracking_ids!(list, force: false)
       list.each { |e| e.resolve_linked_tracking_ids!(@id_map, force: force) }
     end
 
-    def filter_by_project!(definitions)
+    def filter_actual_by_project!(actual)
       return unless @project_filter
-      definitions.select! do |a|
-        id = a.fetch(:klass).parse_tracking_id(a)
-        !id || id.start_with?("#{@project_filter}:")
+      actual.select! do |a|
+        tracking_id = a.fetch(:tracking_id)
+        !tracking_id || tracking_id.start_with?("#{@project_filter}:")
       end
     end
   end

data/lib/kennel/tasks.rb

@@ -138,7 +138,7 @@ namespace :kennel do
     resources.each do |resource|
       Kennel::Progress.progress("Downloading #{resource}") do
         list = api.list(resource)
-        api.fill_details!(resource, list)
+        api.fill_details!(resource, list) if resource == "dashboard"
       end
       list.each do |r|
         r[:api_resource] = resource

data/lib/kennel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Kennel
-  VERSION = "1.89.0"
+  VERSION = "1.91.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kennel
 version: !ruby/object:Gem::Version
-  version: 1.89.0
+  version: 1.91.1
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-25 00:00:00.000000000 Z
+date: 2021-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday
@@ -70,6 +70,7 @@ files:
 - lib/kennel/models/project.rb
 - lib/kennel/models/record.rb
 - lib/kennel/models/slo.rb
+- lib/kennel/models/synthetic_test.rb
 - lib/kennel/models/team.rb
 - lib/kennel/optional_validations.rb
 - lib/kennel/progress.rb