kennel 1.118.2 → 1.120.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 73ea25369567c46bcfd2af7ce3ac6c5b64754f9a16f5506d46c23edf9f8fbadf
-  data.tar.gz: b07ae565f4c905e60b784688d15fa704bcbce470283632ff249e26ca51924a1a
+  metadata.gz: 5cbbb48418612701154d8a4aef95ef0267ca8993425b23977f550b91f185a0cb
+  data.tar.gz: d50f6890094e8af69346047cdae827ea5ca63d28e119c0de58792f7dc969a11d
 SHA512:
-  metadata.gz: '079bf981abd354955807478b3e7ee4e90cdeb418053b1054271c5460421ba7b175afeb77c2b512e835890eff5cbbb121abf2be9ae3f4897376ff6d4cb2f0c772'
-  data.tar.gz: 51444f447bec09ff09b5b854b0d759d270b8b72f9145d9e5d1bb40647215841384cad9a7d4e4079678fdedf7485dcea83c2b91a1af9de577ebdddf06d0b27424
+  metadata.gz: eac3443d6277ff33da2c1b4dec30ad389ecae8b6c211b65eb6e5b4c19404d2b32adc1299d6d207de58839b321f1343ee3186f2b488d375db92db411735e3bf69
+  data.tar.gz: 83e9083b77bbe269985986478a2a021ee32f9ac88712e662b3e817c6a70f1020ac68f0484944d8c9e7e4920c88440baf5e4cd111cc7be80bb943fdfd17f3a854

data/lib/kennel/filter.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Kennel
+  class Filter
+    attr_reader :project_filter, :tracking_id_filter
+
+    def initialize
+      # build early so we fail fast on invalid user input
+      @tracking_id_filter = build_tracking_id_filter
+      @project_filter = build_project_filter
+    end
+
+    def filter_projects(projects)
+      filter_resources(projects, :kennel_id, project_filter, "projects", "PROJECT")
+    end
+
+    def filter_parts(parts)
+      filter_resources(parts, :tracking_id, tracking_id_filter, "resources", "TRACKING_ID")
+    end
+
+    private
+
+    def build_project_filter
+      project_names = ENV["PROJECT"]&.split(",")&.sort&.uniq
+      tracking_project_names = tracking_id_filter&.map { |id| id.split(":", 2).first }&.sort&.uniq
+      if project_names && tracking_project_names && project_names != tracking_project_names
+        raise "do not set PROJECT= when using TRACKING_ID="
+      end
+      (project_names || tracking_project_names)
+    end
+
+    def build_tracking_id_filter
+      (tracking_id = ENV["TRACKING_ID"]) && tracking_id.split(",").sort.uniq
+    end
+
+    def filter_resources(resources, by, expected, name, env)
+      return resources unless expected
+
+      expected = expected.uniq
+      before = resources.dup
+      resources = resources.select { |p| expected.include?(p.send(by)) }
+      keeping = resources.uniq(&by).size
+      return resources if keeping == expected.size
+
+      raise <<~MSG.rstrip
+        #{env}=#{expected.join(",")} matched #{keeping} #{name}, try any of these:
+        #{before.map(&by).sort.uniq.join("\n")}
+      MSG
+    end
+  end
+end

data/lib/kennel/models/monitor.rb

@@ -27,11 +27,12 @@ module Kennel
       }.freeze
       DEFAULT_ESCALATION_MESSAGE = ["", nil].freeze
       ALLOWED_PRIORITY_CLASSES = [NilClass, Integer].freeze
+      ALLOWED_UNLINKED = [] # rubocop:disable Style/MutableConstant placeholder for custom overrides
 
       settings(
         :query, :name, :message, :escalation_message, :critical, :type, :renotify_interval, :warning, :timeout_h, :evaluation_delay,
         :ok, :no_data_timeframe, :notify_no_data, :notify_audit, :tags, :critical_recovery, :warning_recovery, :require_full_window,
-        :threshold_windows, :new_host_delay, :new_group_delay, :priority
+        :threshold_windows, :new_host_delay, :new_group_delay, :priority, :validate_using_links
       )
 
       defaults(
@@ -238,6 +239,8 @@ module Kennel
           validate_message_variables(data)
         end
 
+        validate_using_links(data)
+
         if type == "service check" && !data[:query].to_s.include?(".by(")
           invalid! "query must include a .by() at least .by(\"*\")"
         end
@@ -286,6 +289,21 @@ module Kennel
           Group or filter the query by #{forbidden.map { |f| f.sub(".name", "") }.join(", ")} to use it.
         MSG
       end
+
+      def validate_using_links(data)
+        case data[:type]
+        when "composite" # TODO: add slo to mirror resolve_linked_tracking_ids! logic
+          ids = data[:query].tr("-", "_").scan(/\b\d+\b/)
+          ids.reject! { |id| ALLOWED_UNLINKED.include?([tracking_id, id]) }
+          if ids.any?
+            invalid! <<~MSG.rstrip
+              Used #{ids} in the query, but should only use links in the form of %{<project id>:<monitor id>}
+              If that is not possible, add `validate_using_links: ->(*){} # linked monitors are not in kennel
+            MSG
+          end
+        else # do nothing
+        end
+      end
     end
   end
 end

data/lib/kennel/parts_serializer.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Kennel
+  class PartsSerializer
+    def initialize(filter:)
+      @filter = filter
+    end
+
+    def write(parts)
+      Progress.progress "Storing" do
+        if filter.tracking_id_filter
+          write_changed(parts)
+        else
+          old = old_paths
+          used = write_changed(parts)
+          (old - used).uniq.each { |p| FileUtils.rm_rf(p) }
+        end
+      end
+    end
+
+    private
+
+    attr_reader :filter
+
+    def write_changed(parts)
+      used = []
+
+      Utils.parallel(parts, max: 2) do |part|
+        path = "generated/#{part.tracking_id.tr("/", ":").sub(":", "/")}.json"
+
+        used << File.dirname(path) # only 1 level of sub folders, so this is enough
+        used << path
+
+        payload = part.as_json.merge(api_resource: part.class.api_resource)
+        write_file_if_necessary(path, JSON.pretty_generate(payload) << "\n")
+      end
+
+      used
+    end
+
+    def directories_to_clean_up
+      if filter.project_filter
+        filter.project_filter.map { |project| "generated/#{project}" }
+      else
+        ["generated"]
+      end
+    end
+
+    def old_paths
+      Dir["{#{directories_to_clean_up.join(",")}}/**/*"]
+    end
+
+    def write_file_if_necessary(path, content)
+      # 99% case
+      begin
+        return if File.read(path) == content
+      rescue Errno::ENOENT
+        FileUtils.mkdir_p(File.dirname(path))
+      end
+
+      # slow 1% case
+      File.write(path, content)
+    end
+  end
+end

data/lib/kennel/projects_provider.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+module Kennel
+  class ProjectsProvider
+    def projects
+      load_all
+      Models::Project.recursive_subclasses.map(&:new)
+    end
+
+    private
+
+    def load_all
+      # load_all's purpose is to "require" all the .rb files under './projects',
+      # also with reference to ./teams and ./parts. What happens if you call it
+      # more than once?
+      #
+      # For a reason yet to be investigated, Zeitwerk rejects second and subsequent calls.
+      # But even if we skip over the Zeitwerk part, the nature of 'require' is
+      # somewhat one-way: we're not providing any mechanism to *un*load things.
+      # As long as the contents of `./projects`, `./teams` and `./parts` doesn't
+      # change between calls, then simply by no-op'ing subsequent calls to `load_all`
+      # we can have `load_all` appear to be idempotent.
+      loader = Zeitwerk::Loader.new
+      Dir.exist?("teams") && loader.push_dir("teams", namespace: Teams)
+      Dir.exist?("parts") && loader.push_dir("parts")
+      loader.setup
+      loader.eager_load # TODO: this should not be needed but we see hanging CI processes when it's not added
+
+      # TODO: also auto-load projects and update expected path too
+      ["projects"].each do |folder|
+        Dir["#{folder}/**/*.rb"].sort.each { |f| require "./#{f}" }
+      end
+    rescue NameError => e
+      message = e.message
+      raise unless klass = message[/uninitialized constant (.*)/, 1]
+
+      # inverse of zeitwerk lib/zeitwerk/inflector.rb
+      path = klass.gsub("::", "/").gsub(/([a-z])([A-Z])/, "\\1_\\2").downcase + ".rb"
+      expected_path = (path.start_with?("teams/") ? path : "parts/#{path}")
+
+      # TODO: prefer to raise a new exception with the old backtrace attacked
+      e.define_singleton_method(:message) do
+        "\n" + <<~MSG.gsub(/^/, "  ")
+          #{message}
+          Unable to load #{klass} from #{expected_path}
+          - Option 1: rename the constant or the file it lives in, to make them match
+          - Option 2: Use `require` or `require_relative` to load the constant
+        MSG
+      end
+
+      raise
+    end
+  end
+end

data/lib/kennel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Kennel
-  VERSION = "1.118.2"
+  VERSION = "1.120.0"
 end

data/lib/kennel.rb

@@ -8,6 +8,9 @@ require "kennel/version"
 require "kennel/compatibility"
 require "kennel/utils"
 require "kennel/progress"
+require "kennel/filter"
+require "kennel/parts_serializer"
+require "kennel/projects_provider"
 require "kennel/syncer"
 require "kennel/id_map"
 require "kennel/api"
@@ -54,9 +57,9 @@ module Kennel
     attr_accessor :out, :err, :strict_imports
 
     def generate
-      out = generated
-      store out if ENV["STORE"] != "false" # quicker when debugging
-      out
+      parts = generated
+      parts_serializer.write(parts) if ENV["STORE"] != "false" # quicker when debugging
+      parts
     end
 
     def plan
@@ -74,63 +77,34 @@ module Kennel
 
     private
 
-    def store(parts)
-      Progress.progress "Storing" do
-        old = Dir[[
-          "generated",
-          if project_filter || tracking_id_filter
-            [
-              "{" + (project_filter || ["*"]).join(",") + "}",
-              "{" + (tracking_id_filter || ["*"]).join(",") + "}.json"
-            ]
-          else
-            "**"
-          end
-        ].join("/")]
-        used = []
-
-        Utils.parallel(parts, max: 2) do |part|
-          path = "generated/#{part.tracking_id.tr("/", ":").sub(":", "/")}.json"
-          used.concat [File.dirname(path), path] # only 1 level of sub folders, so this is safe
-          payload = part.as_json.merge(api_resource: part.class.api_resource)
-          write_file_if_necessary(path, JSON.pretty_generate(payload) << "\n")
-        end
-
-        # deleting all is slow, so only delete the extras
-        (old - used).each { |p| FileUtils.rm_rf(p) }
-      end
-    end
-
-    def write_file_if_necessary(path, content)
-      # 99% case
-      begin
-        return if File.read(path) == content
-      rescue Errno::ENOENT
-        FileUtils.mkdir_p(File.dirname(path))
-      end
-
-      # slow 1% case
-      File.write(path, content)
+    def filter
+      @filter ||= Filter.new
     end
 
     def syncer
-      @syncer ||= Syncer.new(api, generated, project_filter: project_filter, tracking_id_filter: tracking_id_filter)
+      @syncer ||= Syncer.new(api, generated, project_filter: filter.project_filter, tracking_id_filter: filter.tracking_id_filter)
     end
 
     def api
       @api ||= Api.new(ENV.fetch("DATADOG_APP_KEY"), ENV.fetch("DATADOG_API_KEY"))
     end
 
+    def projects_provider
+      @projects_provider ||= ProjectsProvider.new
+    end
+
+    def parts_serializer
+      @parts_serializer ||= PartsSerializer.new(filter: filter)
+    end
+
     def generated
       @generated ||= begin
         Progress.progress "Generating" do
-          load_all
-
-          projects = Models::Project.recursive_subclasses.map(&:new)
-          filter_resources!(projects, :kennel_id, project_filter, "projects", "PROJECT")
+          projects = projects_provider.projects
+          projects = filter.filter_projects projects
 
           parts = Utils.parallel(projects, &:validated_parts).flatten(1)
-          filter_resources!(parts, :tracking_id, tracking_id_filter, "resources", "TRACKING_ID")
+          parts = filter.filter_parts parts
 
           parts.group_by(&:tracking_id).each do |tracking_id, same|
             next if same.size == 1
@@ -147,74 +121,5 @@ module Kennel
         end
       end
     end
-
-    def project_filter
-      projects = ENV["PROJECT"]&.split(",")
-      tracking_projects = tracking_id_filter&.map { |id| id.split(":", 2).first }
-      if projects && tracking_projects && projects != tracking_projects
-        raise "do not set PROJECT= when using TRACKING_ID="
-      end
-      projects || tracking_projects
-    end
-
-    def tracking_id_filter
-      (tracking_id = ENV["TRACKING_ID"]) && tracking_id.split(",")
-    end
-
-    def filter_resources!(resources, by, against, name, env)
-      return unless against
-
-      before = resources.dup
-      resources.select! { |p| against.include?(p.send(by)) }
-      keeping = resources.uniq(&by).size
-      return if keeping == against.size
-
-      raise <<~MSG.rstrip
-        #{env}=#{against.join(",")} matched #{keeping} #{name}, try any of these:
-        #{before.map(&by).sort.uniq.join("\n")}
-      MSG
-    end
-
-    def load_all
-      # load_all's purpose is to "require" all the .rb files under './projects',
-      # also with reference to ./teams and ./parts. What happens if you call it
-      # more than once?
-      #
-      # For a reason yet to be investigated, Zeitwerk rejects second and subsequent calls.
-      # But even if we skip over the Zeitwerk part, the nature of 'require' is
-      # somewhat one-way: we're not providing any mechanism to *un*load things.
-      # As long as the contents of `./projects`, `./teams` and `./parts` doesn't
-      # change between calls, then simply by no-op'ing subsequent calls to `load_all`
-      # we can have `load_all` appear to be idempotent.
-      loader = Zeitwerk::Loader.new
-      Dir.exist?("teams") && loader.push_dir("teams", namespace: Teams)
-      Dir.exist?("parts") && loader.push_dir("parts")
-      loader.setup
-      loader.eager_load # TODO: this should not be needed but we see hanging CI processes when it's not added
-
-      # TODO: also auto-load projects and update expected path too
-      ["projects"].each do |folder|
-        Dir["#{folder}/**/*.rb"].sort.each { |f| require "./#{f}" }
-      end
-    rescue NameError => e
-      message = e.message
-      raise unless klass = message[/uninitialized constant (.*)/, 1]
-
-      # inverse of zeitwerk lib/zeitwerk/inflector.rb
-      path = klass.gsub("::", "/").gsub(/([a-z])([A-Z])/, "\\1_\\2").downcase + ".rb"
-      expected_path = (path.start_with?("teams/") ? path : "parts/#{path}")
-
-      # TODO: prefer to raise a new exception with the old backtrace attacked
-      e.define_singleton_method(:message) do
-        "\n" + <<~MSG.gsub(/^/, "  ")
-          #{message}
-          Unable to load #{klass} from #{expected_path}
-          - Option 1: rename the constant or the file it lives in, to make them match
-          - Option 2: Use `require` or `require_relative` to load the constant
-        MSG
-      end
-
-      raise
-    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kennel
 version: !ruby/object:Gem::Version
-  version: 1.118.2
+  version: 1.120.0
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-09-23 00:00:00.000000000 Z
+date: 2022-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: diff-lcs
@@ -91,6 +91,7 @@ files:
 - lib/kennel/api.rb
 - lib/kennel/compatibility.rb
 - lib/kennel/file_cache.rb
+- lib/kennel/filter.rb
 - lib/kennel/github_reporter.rb
 - lib/kennel/id_map.rb
 - lib/kennel/importer.rb
@@ -103,7 +104,9 @@ files:
 - lib/kennel/models/synthetic_test.rb
 - lib/kennel/models/team.rb
 - lib/kennel/optional_validations.rb
+- lib/kennel/parts_serializer.rb
 - lib/kennel/progress.rb
+- lib/kennel/projects_provider.rb
 - lib/kennel/settings_as_methods.rb
 - lib/kennel/subclass_tracking.rb
 - lib/kennel/syncer.rb