kennel 1.118.2 → 1.119.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 73ea25369567c46bcfd2af7ce3ac6c5b64754f9a16f5506d46c23edf9f8fbadf
-  data.tar.gz: b07ae565f4c905e60b784688d15fa704bcbce470283632ff249e26ca51924a1a
+  metadata.gz: 67b43081a187354fb146269d929da674d533263b44b3f2cee00b2baa7cc8bc57
+  data.tar.gz: c14b369716513a92791496a92c29c7648499d0a06f05e34d36c628f4a9bb9305
 SHA512:
-  metadata.gz: '079bf981abd354955807478b3e7ee4e90cdeb418053b1054271c5460421ba7b175afeb77c2b512e835890eff5cbbb121abf2be9ae3f4897376ff6d4cb2f0c772'
-  data.tar.gz: 51444f447bec09ff09b5b854b0d759d270b8b72f9145d9e5d1bb40647215841384cad9a7d4e4079678fdedf7485dcea83c2b91a1af9de577ebdddf06d0b27424
+  metadata.gz: b0eb58d97d4be73227f791379f623a188ab45256749cd4b11a969444a8598d8b30f70490c12dbad12ff0a46d42f2ff1820998304b7167cc11a9876e8aaef06cc
+  data.tar.gz: cb38270a791164a92a06522921234838fe4e944e54ebd53a54c00939d531861847de0d4b6cef8d3605a9b2692874e9ac5715ce4fa4df07550f26094e810be845

data/lib/kennel/filter.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Kennel
+  class Filter
+    def initialize
+      project_filter
+      tracking_id_filter
+    end
+
+    def project_filter
+      projects = ENV["PROJECT"]&.split(",")&.sort&.uniq
+      tracking_projects = tracking_id_filter&.map { |id| id.split(":", 2).first }&.sort&.uniq
+      if projects && tracking_projects && projects != tracking_projects
+        raise "do not set PROJECT= when using TRACKING_ID="
+      end
+      (projects || tracking_projects)
+    end
+
+    def tracking_id_filter
+      (tracking_id = ENV["TRACKING_ID"]) && tracking_id.split(",").sort.uniq
+    end
+
+    def self.filter_resources!(resources, by, against, name, env)
+      return unless against
+
+      against = against.uniq
+      before = resources.dup
+      resources.select! { |p| against.include?(p.send(by)) }
+      keeping = resources.uniq(&by).size
+      return if keeping == against.size
+
+      raise <<~MSG.rstrip
+        #{env}=#{against.join(",")} matched #{keeping} #{name}, try any of these:
+        #{before.map(&by).sort.uniq.join("\n")}
+      MSG
+    end
+  end
+end

data/lib/kennel/parts_serializer.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Kennel
+  class PartsSerializer
+    def initialize(filter:)
+      @filter = filter
+    end
+
+    def write(parts)
+      Progress.progress "Storing" do
+        if filter.tracking_id_filter
+          write_changed(parts)
+        else
+          old = old_paths
+          used = write_changed(parts)
+          (old - used).uniq.each { |p| FileUtils.rm_rf(p) }
+        end
+      end
+    end
+
+    private
+
+    attr_reader :filter
+
+    def write_changed(parts)
+      used = []
+
+      Utils.parallel(parts, max: 2) do |part|
+        path = "generated/#{part.tracking_id.tr("/", ":").sub(":", "/")}.json"
+
+        used << File.dirname(path) # only 1 level of sub folders, so this is enough
+        used << path
+
+        payload = part.as_json.merge(api_resource: part.class.api_resource)
+        write_file_if_necessary(path, JSON.pretty_generate(payload) << "\n")
+      end
+
+      used
+    end
+
+    def directories_to_clean_up
+      if filter.project_filter
+        filter.project_filter.map { |project| "generated/#{project}" }
+      else
+        ["generated"]
+      end
+    end
+
+    def old_paths
+      Dir["{#{directories_to_clean_up.join(",")}}/**/*"]
+    end
+
+    def write_file_if_necessary(path, content)
+      # 99% case
+      begin
+        return if File.read(path) == content
+      rescue Errno::ENOENT
+        FileUtils.mkdir_p(File.dirname(path))
+      end
+
+      # slow 1% case
+      File.write(path, content)
+    end
+  end
+end

data/lib/kennel/projects_provider.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+module Kennel
+  class ProjectsProvider
+    def projects
+      load_all
+      Models::Project.recursive_subclasses.map(&:new)
+    end
+
+    private
+
+    def load_all
+      # load_all's purpose is to "require" all the .rb files under './projects',
+      # also with reference to ./teams and ./parts. What happens if you call it
+      # more than once?
+      #
+      # For a reason yet to be investigated, Zeitwerk rejects second and subsequent calls.
+      # But even if we skip over the Zeitwerk part, the nature of 'require' is
+      # somewhat one-way: we're not providing any mechanism to *un*load things.
+      # As long as the contents of `./projects`, `./teams` and `./parts` doesn't
+      # change between calls, then simply by no-op'ing subsequent calls to `load_all`
+      # we can have `load_all` appear to be idempotent.
+      loader = Zeitwerk::Loader.new
+      Dir.exist?("teams") && loader.push_dir("teams", namespace: Teams)
+      Dir.exist?("parts") && loader.push_dir("parts")
+      loader.setup
+      loader.eager_load # TODO: this should not be needed but we see hanging CI processes when it's not added
+
+      # TODO: also auto-load projects and update expected path too
+      ["projects"].each do |folder|
+        Dir["#{folder}/**/*.rb"].sort.each { |f| require "./#{f}" }
+      end
+    rescue NameError => e
+      message = e.message
+      raise unless klass = message[/uninitialized constant (.*)/, 1]
+
+      # inverse of zeitwerk lib/zeitwerk/inflector.rb
+      path = klass.gsub("::", "/").gsub(/([a-z])([A-Z])/, "\\1_\\2").downcase + ".rb"
+      expected_path = (path.start_with?("teams/") ? path : "parts/#{path}")
+
+      # TODO: prefer to raise a new exception with the old backtrace attacked
+      e.define_singleton_method(:message) do
+        "\n" + <<~MSG.gsub(/^/, "  ")
+          #{message}
+          Unable to load #{klass} from #{expected_path}
+          - Option 1: rename the constant or the file it lives in, to make them match
+          - Option 2: Use `require` or `require_relative` to load the constant
+        MSG
+      end
+
+      raise
+    end
+  end
+end

data/lib/kennel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Kennel
-  VERSION = "1.118.2"
+  VERSION = "1.119.0"
 end

data/lib/kennel.rb

@@ -8,6 +8,9 @@ require "kennel/version"
 require "kennel/compatibility"
 require "kennel/utils"
 require "kennel/progress"
+require "kennel/filter"
+require "kennel/parts_serializer"
+require "kennel/projects_provider"
 require "kennel/syncer"
 require "kennel/id_map"
 require "kennel/api"
@@ -54,9 +57,9 @@ module Kennel
     attr_accessor :out, :err, :strict_imports
 
     def generate
-      out = generated
-      store out if ENV["STORE"] != "false" # quicker when debugging
-      out
+      parts = generated
+      parts_serializer.write(parts) if ENV["STORE"] != "false" # quicker when debugging
+      parts
     end
 
     def plan
@@ -74,63 +77,34 @@ module Kennel
 
     private
 
-    def store(parts)
-      Progress.progress "Storing" do
-        old = Dir[[
-          "generated",
-          if project_filter || tracking_id_filter
-            [
-              "{" + (project_filter || ["*"]).join(",") + "}",
-              "{" + (tracking_id_filter || ["*"]).join(",") + "}.json"
-            ]
-          else
-            "**"
-          end
-        ].join("/")]
-        used = []
-
-        Utils.parallel(parts, max: 2) do |part|
-          path = "generated/#{part.tracking_id.tr("/", ":").sub(":", "/")}.json"
-          used.concat [File.dirname(path), path] # only 1 level of sub folders, so this is safe
-          payload = part.as_json.merge(api_resource: part.class.api_resource)
-          write_file_if_necessary(path, JSON.pretty_generate(payload) << "\n")
-        end
-
-        # deleting all is slow, so only delete the extras
-        (old - used).each { |p| FileUtils.rm_rf(p) }
-      end
-    end
-
-    def write_file_if_necessary(path, content)
-      # 99% case
-      begin
-        return if File.read(path) == content
-      rescue Errno::ENOENT
-        FileUtils.mkdir_p(File.dirname(path))
-      end
-
-      # slow 1% case
-      File.write(path, content)
+    def filter
+      @filter ||= Filter.new
     end
 
     def syncer
-      @syncer ||= Syncer.new(api, generated, project_filter: project_filter, tracking_id_filter: tracking_id_filter)
+      @syncer ||= Syncer.new(api, generated, project_filter: filter.project_filter, tracking_id_filter: filter.tracking_id_filter)
     end
 
     def api
       @api ||= Api.new(ENV.fetch("DATADOG_APP_KEY"), ENV.fetch("DATADOG_API_KEY"))
     end
 
+    def projects_provider
+      @projects_provider ||= ProjectsProvider.new
+    end
+
+    def parts_serializer
+      @parts_serializer ||= PartsSerializer.new(filter: filter)
+    end
+
     def generated
       @generated ||= begin
         Progress.progress "Generating" do
-          load_all
-
-          projects = Models::Project.recursive_subclasses.map(&:new)
-          filter_resources!(projects, :kennel_id, project_filter, "projects", "PROJECT")
+          projects = projects_provider.projects
+          Kennel::Filter.filter_resources!(projects, :kennel_id, filter.project_filter, "projects", "PROJECT")
 
           parts = Utils.parallel(projects, &:validated_parts).flatten(1)
-          filter_resources!(parts, :tracking_id, tracking_id_filter, "resources", "TRACKING_ID")
+          Kennel::Filter.filter_resources!(parts, :tracking_id, filter.tracking_id_filter, "resources", "TRACKING_ID")
 
           parts.group_by(&:tracking_id).each do |tracking_id, same|
             next if same.size == 1
@@ -147,74 +121,5 @@ module Kennel
         end
       end
     end
-
-    def project_filter
-      projects = ENV["PROJECT"]&.split(",")
-      tracking_projects = tracking_id_filter&.map { |id| id.split(":", 2).first }
-      if projects && tracking_projects && projects != tracking_projects
-        raise "do not set PROJECT= when using TRACKING_ID="
-      end
-      projects || tracking_projects
-    end
-
-    def tracking_id_filter
-      (tracking_id = ENV["TRACKING_ID"]) && tracking_id.split(",")
-    end
-
-    def filter_resources!(resources, by, against, name, env)
-      return unless against
-
-      before = resources.dup
-      resources.select! { |p| against.include?(p.send(by)) }
-      keeping = resources.uniq(&by).size
-      return if keeping == against.size
-
-      raise <<~MSG.rstrip
-        #{env}=#{against.join(",")} matched #{keeping} #{name}, try any of these:
-        #{before.map(&by).sort.uniq.join("\n")}
-      MSG
-    end
-
-    def load_all
-      # load_all's purpose is to "require" all the .rb files under './projects',
-      # also with reference to ./teams and ./parts. What happens if you call it
-      # more than once?
-      #
-      # For a reason yet to be investigated, Zeitwerk rejects second and subsequent calls.
-      # But even if we skip over the Zeitwerk part, the nature of 'require' is
-      # somewhat one-way: we're not providing any mechanism to *un*load things.
-      # As long as the contents of `./projects`, `./teams` and `./parts` doesn't
-      # change between calls, then simply by no-op'ing subsequent calls to `load_all`
-      # we can have `load_all` appear to be idempotent.
-      loader = Zeitwerk::Loader.new
-      Dir.exist?("teams") && loader.push_dir("teams", namespace: Teams)
-      Dir.exist?("parts") && loader.push_dir("parts")
-      loader.setup
-      loader.eager_load # TODO: this should not be needed but we see hanging CI processes when it's not added
-
-      # TODO: also auto-load projects and update expected path too
-      ["projects"].each do |folder|
-        Dir["#{folder}/**/*.rb"].sort.each { |f| require "./#{f}" }
-      end
-    rescue NameError => e
-      message = e.message
-      raise unless klass = message[/uninitialized constant (.*)/, 1]
-
-      # inverse of zeitwerk lib/zeitwerk/inflector.rb
-      path = klass.gsub("::", "/").gsub(/([a-z])([A-Z])/, "\\1_\\2").downcase + ".rb"
-      expected_path = (path.start_with?("teams/") ? path : "parts/#{path}")
-
-      # TODO: prefer to raise a new exception with the old backtrace attacked
-      e.define_singleton_method(:message) do
-        "\n" + <<~MSG.gsub(/^/, "  ")
-          #{message}
-          Unable to load #{klass} from #{expected_path}
-          - Option 1: rename the constant or the file it lives in, to make them match
-          - Option 2: Use `require` or `require_relative` to load the constant
-        MSG
-      end
-
-      raise
-    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kennel
 version: !ruby/object:Gem::Version
-  version: 1.118.2
+  version: 1.119.0
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-09-23 00:00:00.000000000 Z
+date: 2022-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: diff-lcs
@@ -91,6 +91,7 @@ files:
 - lib/kennel/api.rb
 - lib/kennel/compatibility.rb
 - lib/kennel/file_cache.rb
+- lib/kennel/filter.rb
 - lib/kennel/github_reporter.rb
 - lib/kennel/id_map.rb
 - lib/kennel/importer.rb
@@ -103,7 +104,9 @@ files:
 - lib/kennel/models/synthetic_test.rb
 - lib/kennel/models/team.rb
 - lib/kennel/optional_validations.rb
+- lib/kennel/parts_serializer.rb
 - lib/kennel/progress.rb
+- lib/kennel/projects_provider.rb
 - lib/kennel/settings_as_methods.rb
 - lib/kennel/subclass_tracking.rb
 - lib/kennel/syncer.rb