kennel 1.114.1 → 1.116.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 248d75d411fad53dd8011982ec16d2104547991ec35480bf9d085feaed419285
-  data.tar.gz: fe11bc68ac149ff7ef989c8bd2b2b497eb465f6f0eccc66d5fbf304de9801666
+  metadata.gz: 49677de3c690744756d70810dbe71c49cb0313014c5c325af21aa161463738bc
+  data.tar.gz: c52c465238c8043ba7a735712dcaf73edd589875b680b7f45f235477c696cff2
 SHA512:
-  metadata.gz: d9888da18e41995160fb03516fc02037cb07f43acae46b298941952ec053df4b2ca62ea918a5b1b17664c76d926bb702b6e28a375af1f3047451e2463f88226c
-  data.tar.gz: 92917f89e251d9345f1cf580ebb0206b1206882abe036d1e44284835c025ca61822e261d35a120fe7194303ede139e3c56db48b878fdb137f8d176cc4796441f
+  metadata.gz: 50968ff4039145263ea380d790ebd8d757afb96bd2966a8b7dddf5a58373dbf3fd2e13f0576ad44bf42ceecd7086095865d4edc7c85bc5df1b1f9cc7b9757638
+  data.tar.gz: 7facba1805b0d2690b4081e18d46d314161544326a8e2e772bd17284c9065d7a8d50431690ca7562425510b9a12db70726453e3a4296aac6cbfd00d3e6954172

data/Readme.md

@@ -289,23 +289,27 @@ Remove the code that created the resource. The next update will delete it (see a
  ```
 
 ### Updating existing resources with id
-
 Setting `id` makes kennel take over a manually created datadog resource.
 When manually creating to import, it is best to remove the `id` and delete the manually created resource.
 
 When an `id` is set and the original resource is deleted, kennel will fail to update,
 removing the `id` will cause kennel to create a new resource in datadog.
 
-### Organizing many projects
-Having many projects (and their sub-resources) can quickly get out of hand.
+### Organizing projects with many resources
+When project files get too long, this structure can keep things bite-sized.
 
-Use this class structure to keep things organized:
 ```Ruby
 # projects/project_a/base.rb
 module ProjectA
   class Base < Kennel::Models::Project
     defaults(
       kennel_id: -> { "project_a" },
+      parts: -> {
+        [
+          Monitors::FooAlert.new(self),
+          ...
+        ]
+      }
       ...
 
 # projects/project_a/monitors/foo_alert.rb
@@ -315,6 +319,17 @@ module ProjectA
       ...
 ```
 
+### Updating a single project or resource
+
+- Use `PROJECT=<kennel_id>` for single project:
+
+  Use the projects `kennel_id` (and if none is set then snake_case of the class name including modules)
+  to refer to the project. For example for `class ProjectA` use `PROJECT=project_a` but for `Foo::ProjectA` use `foo_project_a`.
+
+- Use `TRACKING_ID=<project-kennel_id>:<resource-kennel_id>` for single resource:
+
+  Use the project kennel_id and the resources kennel_id, for example `class ProjectA` and `FooAlert` would give `project_a:foo_alert`.
+
 ### Skipping validations
 Some validations might be too strict for your usecase or just wrong, please [open an issue](https://github.com/grosser/kennel/issues) and
 to unblock use the `validate: -> { false }` option.
@@ -335,7 +350,7 @@ so they can be created in a single update and can be re-created if any of them i
 
 ### Debugging changes locally
  - rebase on updated `master` to not undo other changes
- - figure out project name by converting the class name to snake-case
+ - figure out project name by converting the class name to snake_case
  - run `PROJECT=foo bundle exec rake kennel:update_datadog` to test changes for a single project (monitors: remove mentions while debugging to avoid alert spam)
    - use `PROJECT=foo,bar,...` for multiple projects
 

data/lib/kennel/models/project.rb

@@ -19,12 +19,21 @@ module Kennel
 
       def validated_parts
         all = parts
+        unless all.is_a?(Array) && all.all? { |part| part.is_a?(Record) }
+          invalid! "#parts must return an array of Records"
+        end
+
         validate_parts(all)
         all
       end
 
       private
 
+      # let users know which project/resource failed when something happens during diffing where the backtrace is hidden
+      def invalid!(message)
+        raise ValidationError, "#{kennel_id} #{message}"
+      end
+
       # hook for users to add custom validations via `prepend`
       def validate_parts(parts)
       end

data/lib/kennel/progress.rb

@@ -7,12 +7,12 @@ module Kennel
     def self.progress(name)
       Kennel.err.print "#{name} ... "
 
-      animation = "-\\|/"
-      count = 0
       stop = false
       result = nil
 
       spinner = Thread.new do
+        animation = "-\\|/"
+        count = 0
         loop do
           break if stop
           Kennel.err.print animation[count % animation.size]
@@ -30,7 +30,7 @@ module Kennel
 
       result
     ensure
-      stop = true
+      stop = true # make thread stop without killing it
     end
   end
 end

data/lib/kennel/syncer.rb

@@ -4,9 +4,10 @@ module Kennel
     DELETE_ORDER = ["dashboard", "slo", "monitor", "synthetics/tests"].freeze # dashboards references monitors + slos, slos reference monitors
     LINE_UP = "\e[1A\033[K" # go up and clear
 
-    def initialize(api, expected, project_filter: nil)
+    def initialize(api, expected, project_filter: nil, tracking_id_filter: nil)
       @api = api
       @project_filter = project_filter
+      @tracking_id_filter = tracking_id_filter
       @expected = Set.new expected # need set to speed up deletion
       calculate_diff
       validate_plan
@@ -105,7 +106,7 @@ module Kennel
 
       Progress.progress "Diffing" do
         populate_id_map @expected, actual
-        filter_actual_by_project! actual
+        filter_actual! actual
         resolve_linked_tracking_ids! @expected # resolve dependencies to avoid diff
 
         @expected.each(&:add_tracking_id) # avoid diff with actual
@@ -261,11 +262,12 @@ module Kennel
         next unless tracking_id = a.fetch(:tracking_id)
 
         # ignore when deleted from the codebase
-        # (when running with project filter we cannot see the other resources in the codebase)
+        # (when running with filters we cannot see the other resources in the codebase)
         api_resource = a.fetch(:klass).api_resource
         next if
           !@id_map.get(api_resource, tracking_id) &&
-          (!project_prefixes || tracking_id.start_with?(*project_prefixes))
+          (!project_prefixes || tracking_id.start_with?(*project_prefixes)) &&
+          (!@tracking_id_filter || @tracking_id_filter.include?(tracking_id))
 
         @id_map.set(api_resource, tracking_id, a.fetch(:id))
         if a[:klass].api_resource == "synthetics/tests"
@@ -278,12 +280,18 @@ module Kennel
       list.each { |e| e.resolve_linked_tracking_ids!(@id_map, force: force) }
     end
 
-    def filter_actual_by_project!(actual)
-      return unless @project_filter
-      project_prefixes = @project_filter&.map { |p| "#{p}:" }
-      actual.select! do |a|
-        tracking_id = a.fetch(:tracking_id)
-        !tracking_id || tracking_id.start_with?(*project_prefixes)
+    def filter_actual!(actual)
+      if @tracking_id_filter
+        actual.select! do |a|
+          tracking_id = a.fetch(:tracking_id)
+          !tracking_id || @tracking_id_filter.include?(tracking_id)
+        end
+      elsif @project_filter
+        project_prefixes = @project_filter.map { |p| "#{p}:" }
+        actual.select! do |a|
+          tracking_id = a.fetch(:tracking_id)
+          !tracking_id || tracking_id.start_with?(*project_prefixes)
+        end
       end
     end
   end

data/lib/kennel/utils.rb

@@ -113,7 +113,7 @@ module Kennel
               done[i] =
                 begin
                   yield item
-                rescue StandardError => e
+                rescue Exception => e # rubocop:disable Lint/RescueException
                   work.clear
                   e
                 end
@@ -121,7 +121,7 @@ module Kennel
           end
         end
         workers.each(&:join)
-        done.each { |d| raise d if d.is_a?(StandardError) }
+        done.each { |d| raise d if d.is_a?(Exception) }
       end
 
       def natural_order(name)

data/lib/kennel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Kennel
-  VERSION = "1.114.1"
+  VERSION = "1.116.1"
 end

data/lib/kennel.rb

@@ -65,13 +65,22 @@ module Kennel
 
     def store(parts)
       Progress.progress "Storing" do
-        old = Dir["generated/{#{(project_filter || ["**"]).join(",")}}/*"]
+        old = Dir[[
+          "generated",
+          if project_filter || tracking_id_filter
+            [
+              "{" + (project_filter || ["*"]).join(",") + "}",
+              "{" + (tracking_id_filter || ["*"]).join(",") + "}.json"
+            ]
+          else
+            "**"
+          end
+        ].join("/")]
         used = []
 
         Utils.parallel(parts, max: 2) do |part|
           path = "generated/#{part.tracking_id.tr("/", ":").sub(":", "/")}.json"
-          used << File.dirname(path) # only 1 level of sub folders, so this is safe
-          used << path
+          used.concat [File.dirname(path), path] # only 1 level of sub folders, so this is safe
           payload = part.as_json.merge(api_resource: part.class.api_resource)
           write_file_if_necessary(path, JSON.pretty_generate(payload) << "\n")
         end
@@ -94,7 +103,7 @@ module Kennel
     end
 
     def syncer
-      @syncer ||= Syncer.new(api, generated, project_filter: project_filter)
+      @syncer ||= Syncer.new(api, generated, project_filter: project_filter, tracking_id_filter: tracking_id_filter)
     end
 
     def api
@@ -105,22 +114,12 @@ module Kennel
       @generated ||= begin
         Progress.progress "Generating" do
           load_all
-          known = []
-          filter = project_filter
-
-          parts = Utils.parallel(Models::Project.recursive_subclasses) do |project_class|
-            project = project_class.new
-            kennel_id = project.kennel_id
-            if filter
-              known << kennel_id
-              next [] unless filter.include?(kennel_id)
-            end
-            project.validated_parts
-          end.flatten(1)
-
-          if filter && parts.empty?
-            raise "#{filter.join(", ")} does not match any projects, try any of these:\n#{known.uniq.sort.join("\n")}"
-          end
+
+          projects = Models::Project.recursive_subclasses.map(&:new)
+          filter_resources!(projects, :kennel_id, project_filter, "projects", "PROJECT")
+
+          parts = Utils.parallel(projects, &:validated_parts).flatten(1)
+          filter_resources!(parts, :tracking_id, tracking_id_filter, "resources", "TRACKING_ID")
 
           parts.group_by(&:tracking_id).each do |tracking_id, same|
             next if same.size == 1
@@ -139,7 +138,25 @@ module Kennel
     end
 
     def project_filter
-      ENV["PROJECT"]&.split(",")
+      raise "either use PROJECT= or TRACKING_ID=" if ENV["PROJECT"] && ENV["TRACKING_ID"]
+      ENV["PROJECT"]&.split(",") || tracking_id_filter&.map { |id| id.split(":", 2).first }
+    end
+
+    def tracking_id_filter
+      (tracking_id = ENV["TRACKING_ID"]) && tracking_id.split(",")
+    end
+
+    def filter_resources!(resources, by, against, name, env)
+      return unless against
+
+      before = resources.dup
+      resources.select! { |p| against.include?(p.send(by)) }
+      return if resources.size == against.size
+
+      raise <<~MSG.rstrip
+        #{env}=#{against.join(",")} matched #{resources.size} #{name}, try any of these:
+        #{before.map(&by).sort.uniq.join("\n")}
+      MSG
     end
 
     def load_all
@@ -147,8 +164,9 @@ module Kennel
       Dir.exist?("teams") && loader.push_dir("teams", namespace: Teams)
       Dir.exist?("parts") && loader.push_dir("parts")
       loader.setup
+      loader.eager_load # TODO: this should not be needed but we see hanging CI processes when it's not added
 
-      # TODO: also do projects and update expected path too
+      # TODO: also auto-load projects and update expected path too
       ["projects"].each do |folder|
         Dir["#{folder}/**/*.rb"].sort.each { |f| require "./#{f}" }
       end

data/template/Readme.md

@@ -271,23 +271,27 @@ Remove the code that created the resource. The next update will delete it (see a
  ```
 
 ### Updating existing resources with id
-
 Setting `id` makes kennel take over a manually created datadog resource.
 When manually creating to import, it is best to remove the `id` and delete the manually created resource.
 
 When an `id` is set and the original resource is deleted, kennel will fail to update,
 removing the `id` will cause kennel to create a new resource in datadog.
 
-### Organizing many projects
-Having many projects (and their sub-resources) can quickly get out of hand.
+### Organizing projects with many resources
+When project files get too long, this structure can keep things bite-sized.
 
-Use this class structure to keep things organized:
 ```Ruby
 # projects/project_a/base.rb
 module ProjectA
   class Base < Kennel::Models::Project
     defaults(
       kennel_id: -> { "project_a" },
+      parts: -> {
+        [
+          Monitors::FooAlert.new(self),
+          ...
+        ]
+      }
       ...
 
 # projects/project_a/monitors/foo_alert.rb
@@ -297,6 +301,17 @@ module ProjectA
       ...
 ```
 
+### Updating a single project or resource
+
+- Use `PROJECT=<kennel_id>` for single project:
+
+  Use the projects `kennel_id` (and if none is set then snake_case of the class name including modules)
+  to refer to the project. For example for `class ProjectA` use `PROJECT=project_a` but for `Foo::ProjectA` use `foo_project_a`.
+
+- Use `TRACKING_ID=<project-kennel_id>:<resource-kennel_id>` for single resource:
+
+  Use the project kennel_id and the resources kennel_id, for example `class ProjectA` and `FooAlert` would give `project_a:foo_alert`.
+
 ### Skipping validations
 Some validations might be too strict for your usecase or just wrong, please [open an issue](https://github.com/grosser/kennel/issues) and
 to unblock use the `validate: -> { false }` option.
@@ -317,7 +332,7 @@ so they can be created in a single update and can be re-created if any of them i
 
 ### Debugging changes locally
  - rebase on updated `master` to not undo other changes
- - figure out project name by converting the class name to snake-case
+ - figure out project name by converting the class name to snake_case
  - run `PROJECT=foo bundle exec rake kennel:update_datadog` to test changes for a single project (monitors: remove mentions while debugging to avoid alert spam)
    - use `PROJECT=foo,bar,...` for multiple projects
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kennel
 version: !ruby/object:Gem::Version
-  version: 1.114.1
+  version: 1.116.1
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-08-13 00:00:00.000000000 Z
+date: 2022-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday