packwerk 2.0.0 → 2.2.0

data/lib/packwerk/cache.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+# typed: strict
+
+require "digest"
+
+module Packwerk
+  class Cache
+    extend T::Sig
+
+    class CacheContents < T::Struct
+      extend T::Sig
+
+      const :file_contents_digest, String
+      const :unresolved_references, T::Array[UnresolvedReference]
+
+      sig { returns(String) }
+      def serialize
+        to_json
+      end
+
+      sig { params(serialized_cache_contents: String).returns(CacheContents) }
+      def self.deserialize(serialized_cache_contents)
+        cache_contents_json = JSON.parse(serialized_cache_contents)
+        unresolved_references = cache_contents_json["unresolved_references"].map do |json|
+          UnresolvedReference.new(
+            json["constant_name"],
+            json["namespace_path"],
+            json["relative_path"],
+            Node::Location.new(json["source_location"]["line"], json["source_location"]["column"],)
+          )
+        end
+
+        CacheContents.new(
+          file_contents_digest: cache_contents_json["file_contents_digest"],
+          unresolved_references: unresolved_references,
+        )
+      end
+    end
+
+    CACHE_SHAPE = T.type_alias do
+      T::Hash[
+        String,
+        CacheContents
+      ]
+    end
+
+    sig { params(enable_cache: T::Boolean, cache_directory: Pathname, config_path: T.nilable(String)).void }
+    def initialize(enable_cache:, cache_directory:, config_path:)
+      @enable_cache = enable_cache
+      @cache = T.let({}, CACHE_SHAPE)
+      @files_by_digest = T.let({}, T::Hash[String, String])
+      @config_path = config_path
+      @cache_directory = cache_directory
+
+      if @enable_cache
+        create_cache_directory!
+        bust_cache_if_packwerk_yml_has_changed!
+        bust_cache_if_inflections_have_changed!
+      end
+    end
+
+    sig { void }
+    def bust_cache!
+      FileUtils.rm_rf(@cache_directory)
+    end
+
+    sig do
+      params(
+        file_path: String,
+        block: T.proc.returns(T::Array[UnresolvedReference])
+      ).returns(T::Array[UnresolvedReference])
+    end
+    def with_cache(file_path, &block)
+      return block.call unless @enable_cache
+
+      cache_location = @cache_directory.join(digest_for_string(file_path))
+
+      cache_contents = if cache_location.exist?
+        T.let(CacheContents.deserialize(cache_location.read),
+          CacheContents)
+      end
+
+      file_contents_digest = digest_for_file(file_path)
+
+      if !cache_contents.nil? && cache_contents.file_contents_digest == file_contents_digest
+        Debug.out("Cache hit for #{file_path}")
+
+        cache_contents.unresolved_references
+      else
+        Debug.out("Cache miss for #{file_path}")
+
+        unresolved_references = block.call
+
+        cache_contents = CacheContents.new(
+          file_contents_digest: file_contents_digest,
+          unresolved_references: unresolved_references,
+        )
+        cache_location.write(cache_contents.serialize)
+
+        unresolved_references
+      end
+    end
+
+    sig { params(file: String).returns(String) }
+    def digest_for_file(file)
+      digest_for_string(File.read(file))
+    end
+
+    sig { params(str: String).returns(String) }
+    def digest_for_string(str)
+      # MD5 appears to be the fastest
+      # https://gist.github.com/morimori/1330095
+      Digest::MD5.hexdigest(str)
+    end
+
+    sig { void }
+    def bust_cache_if_packwerk_yml_has_changed!
+      return nil if @config_path.nil?
+      bust_cache_if_contents_have_changed(File.read(@config_path), :packwerk_yml)
+    end
+
+    sig { void }
+    def bust_cache_if_inflections_have_changed!
+      bust_cache_if_contents_have_changed(YAML.dump(ActiveSupport::Inflector.inflections), :inflections)
+    end
+
+    sig { params(contents: String, contents_key: Symbol).void }
+    def bust_cache_if_contents_have_changed(contents, contents_key)
+      current_digest = digest_for_string(contents)
+      cached_digest_path = @cache_directory.join(contents_key.to_s)
+
+      if !cached_digest_path.exist?
+        # In this case, we have nothing cached
+        # We save the current digest. This way the next time we compare current digest to cached digest,
+        # we can accurately determine if we should bust the cache
+        cached_digest_path.write(current_digest)
+
+        nil
+      elsif cached_digest_path.read == current_digest
+        Debug.out("#{contents_key} contents have NOT changed, preserving cache")
+      else
+        Debug.out("#{contents_key} contents have changed, busting cache")
+
+        bust_cache!
+        create_cache_directory!
+
+        cached_digest_path.write(current_digest)
+      end
+    end
+
+    sig { void }
+    def create_cache_directory!
+      FileUtils.mkdir_p(@cache_directory)
+    end
+  end
+
+  class Debug
+    extend T::Sig
+
+    sig { params(out: String).void }
+    def self.out(out)
+      if ENV["DEBUG_PACKWERK_CACHE"]
+        puts(out)
+      end
+    end
+  end
+
+  private_constant :Debug
+end

data/lib/packwerk/cli.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: strict
 # frozen_string_literal: true
 
 require "optparse"
@@ -30,9 +30,12 @@ module Packwerk
       @err_out = err_out
       @environment = environment
       @style = style
-      @configuration = configuration || Configuration.from_path
-      @progress_formatter = Formatters::ProgressFormatter.new(@out, style: style)
-      @offenses_formatter = offenses_formatter || Formatters::OffensesFormatter.new(style: @style)
+      @configuration = T.let(configuration || Configuration.from_path, Configuration)
+      @progress_formatter = T.let(Formatters::ProgressFormatter.new(@out, style: style), Formatters::ProgressFormatter)
+      @offenses_formatter = T.let(
+        offenses_formatter || Formatters::OffensesFormatter.new(style: @style),
+        OffensesFormatter
+      )
     end
 
     sig { params(args: T::Array[String]).returns(T.noreturn) }
@@ -78,12 +81,14 @@ module Packwerk
 
     private
 
+    sig { returns(T::Boolean) }
     def init
       @out.puts("📦 Initializing Packwerk...")
 
       generate_configs
     end
 
+    sig { returns(T::Boolean) }
     def generate_configs
       configuration_file = Packwerk::Generators::ConfigurationFile.generate(
         root: @configuration.root_path,
@@ -112,23 +117,31 @@ module Packwerk
       success
     end
 
+    sig { params(result: Result).returns(T::Boolean) }
     def output_result(result)
       @out.puts
       @out.puts(result.message)
       result.status
     end
 
-    def fetch_files_to_process(paths, ignore_nested_packages)
-      files = FilesForProcessing.fetch(
-        paths: paths,
+    sig do
+      params(
+        relative_file_paths: T::Array[String],
+        ignore_nested_packages: T::Boolean
+      ).returns(FilesForProcessing::AbsoluteFileSet)
+    end
+    def fetch_files_to_process(relative_file_paths, ignore_nested_packages)
+      absolute_file_set = FilesForProcessing.fetch(
+        relative_file_paths: relative_file_paths,
         ignore_nested_packages: ignore_nested_packages,
         configuration: @configuration
       )
       abort("No files found or given. "\
-        "Specify files or check the include and exclude glob in the config file.") if files.empty?
-      files
+        "Specify files or check the include and exclude glob in the config file.") if absolute_file_set.empty?
+      absolute_file_set
     end
 
+    sig { params(_paths: T::Array[String]).returns(T::Boolean) }
     def validate(_paths)
       @progress_formatter.started_validation do
         result = checker.check_all
@@ -139,6 +152,7 @@ module Packwerk
       end
     end
 
+    sig { returns(ApplicationValidator) }
     def checker
       Packwerk::ApplicationValidator.new(
         config_file_path: @configuration.config_path,
@@ -147,6 +161,7 @@ module Packwerk
       )
     end
 
+    sig { params(result: ApplicationValidator::Result).void }
     def list_validation_errors(result)
       @out.puts
       if result.ok?
@@ -157,24 +172,25 @@ module Packwerk
       end
     end
 
+    sig { params(params: T.untyped).returns(ParseRun) }
     def parse_run(params)
-      paths = T.let([], T::Array[String])
+      relative_file_paths = T.let([], T::Array[String])
       ignore_nested_packages = nil
 
       if params.any? { |p| p.include?("--packages") }
         OptionParser.new do |parser|
           parser.on("--packages=PACKAGESLIST", Array, "package names, comma separated") do |p|
-            paths = p
+            relative_file_paths = p
           end
         end.parse!(params)
         ignore_nested_packages = true
       else
-        paths = params
+        relative_file_paths = params
         ignore_nested_packages = false
       end
 
       ParseRun.new(
-        files: fetch_files_to_process(paths, ignore_nested_packages),
+        absolute_file_set: fetch_files_to_process(relative_file_paths, ignore_nested_packages),
         configuration: @configuration,
         progress_formatter: @progress_formatter,
         offenses_formatter: @offenses_formatter

data/lib/packwerk/configuration.rb

@@ -34,10 +34,21 @@ module Packwerk
     DEFAULT_EXCLUDE_GLOBS = ["{bin,node_modules,script,tmp,vendor}/**/*"]
 
     attr_reader(
-      :include, :exclude, :root_path, :package_paths, :custom_associations, :config_path
+      :include, :exclude, :root_path, :package_paths, :custom_associations, :config_path, :cache_directory
     )
 
     def initialize(configs = {}, config_path: nil)
+      @include = configs["include"] || DEFAULT_INCLUDE_GLOBS
+      @exclude = configs["exclude"] || DEFAULT_EXCLUDE_GLOBS
+      root = config_path ? File.dirname(config_path) : "."
+      @root_path = File.expand_path(root)
+      @package_paths = configs["package_paths"] || "**/"
+      @custom_associations = configs["custom_associations"] || []
+      @parallel = configs.key?("parallel") ? configs["parallel"] : true
+      @cache_enabled = configs.key?("cache") ? configs["cache"] : false
+      @cache_directory = Pathname.new(configs["cache_directory"] || "tmp/cache/packwerk")
+      @config_path = config_path
+
       if configs["load_paths"]
         warning = <<~WARNING
           DEPRECATION WARNING: The 'load_paths' key in `packwerk.yml` is deprecated.
@@ -47,7 +58,6 @@ module Packwerk
         warn(warning)
       end
 
-      inflection_file = File.expand_path(configs["inflections_file"] || "config/inflections.yml", @root_path)
       if configs["inflections_file"]
         warning = <<~WARNING
           DEPRECATION WARNING: The 'inflections_file' key in `packwerk.yml` is deprecated.
@@ -58,6 +68,7 @@ module Packwerk
         warn(warning)
       end
 
+      inflection_file = File.expand_path(configs["inflections_file"] || "config/inflections.yml", @root_path)
       if Pathname.new(inflection_file).exist?
         warning = <<~WARNING
           DEPRECATION WARNING: Inflections YMLs in packwerk are now deprecated.
@@ -66,16 +77,6 @@ module Packwerk
 
         warn(warning)
       end
-
-      @include = configs["include"] || DEFAULT_INCLUDE_GLOBS
-      @exclude = configs["exclude"] || DEFAULT_EXCLUDE_GLOBS
-      root = config_path ? File.dirname(config_path) : "."
-      @root_path = File.expand_path(root)
-      @package_paths = configs["package_paths"] || "**/"
-      @custom_associations = configs["custom_associations"] || []
-      @parallel = configs.key?("parallel") ? configs["parallel"] : true
-
-      @config_path = config_path
     end
 
     def load_paths
@@ -85,5 +86,9 @@ module Packwerk
     def parallel?
       @parallel
     end
+
+    def cache_enabled?
+      @cache_enabled
+    end
   end
 end

data/lib/packwerk/constant_discovery.rb

@@ -4,7 +4,7 @@
 require "constant_resolver"
 
 module Packwerk
-  # Get information about (partially qualified) constants without loading the application code.
+  # Get information about unresolved constants without loading the application code.
   # Information gathered: Fully qualified name, path to file containing the definition, package,
   # and visibility (public/private to the package).
   #
@@ -15,10 +15,15 @@ module Packwerk
   #   have no way of inferring the file it is defined in. You could argue though that inheritance means that another
   #   constant with the same name exists in the inheriting class, and this view is sufficient for all our use cases.
   class ConstantDiscovery
+    extend T::Sig
+
     ConstantContext = Struct.new(:name, :location, :package, :public?)
 
     # @param constant_resolver [ConstantResolver]
     # @param packages [Packwerk::PackageSet]
+    sig do
+      params(constant_resolver: ConstantResolver, packages: Packwerk::PackageSet).void
+    end
     def initialize(constant_resolver:, packages:)
       @packages = packages
       @resolver = constant_resolver
@@ -30,17 +35,28 @@ module Packwerk
     #
     # @return [Packwerk::Package] the package that contains the given file,
     #   or nil if the path is not owned by any component
+    sig do
+      params(
+        path: String,
+      ).returns(Packwerk::Package)
+    end
     def package_from_path(path)
       @packages.package_from_path(path)
     end
 
     # Analyze a constant via its name.
-    # If the name is partially qualified, we need the current namespace path to correctly infer its full name
+    # If the constant is unresolved, we need the current namespace path to correctly infer its full name
     #
-    # @param const_name [String] The constant's name, fully or partially qualified.
+    # @param const_name [String] The unresolved constant's name.
     # @param current_namespace_path [Array<String>] (optional) The namespace of the context in which the constant is
     #   used, e.g. ["Apps", "Models"] for `Apps::Models`. Defaults to [] which means top level.
     # @return [Packwerk::ConstantDiscovery::ConstantContext]
+    sig do
+      params(
+        const_name: String,
+        current_namespace_path: T.nilable(T::Array[String]),
+      ).returns(T.nilable(ConstantDiscovery::ConstantContext))
+    end
     def context_for(const_name, current_namespace_path: [])
       begin
         constant = @resolver.resolve(const_name, current_namespace_path: current_namespace_path)
@@ -55,7 +71,7 @@ module Packwerk
         constant.name,
         constant.location,
         package,
-        package&.public_path?(constant.location),
+        package.public_path?(constant.location),
       )
     end
   end

data/lib/packwerk/constant_name_inspector.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: strict
 # frozen_string_literal: true
 
 require "ast"

data/lib/packwerk/deprecated_references.rb

@@ -16,6 +16,7 @@ module Packwerk
       @package = package
       @filepath = filepath
       @new_entries = T.let({}, ENTRIES_TYPE)
+      @deprecated_references = T.let(nil, T.nilable(ENTRIES_TYPE))
     end
 
     sig do
@@ -103,7 +104,6 @@ module Packwerk
 
     sig { returns(ENTRIES_TYPE) }
     def deprecated_references
-      @deprecated_references ||= T.let(@deprecated_references, T.nilable(ENTRIES_TYPE))
       @deprecated_references ||= if File.exist?(@filepath)
         load_yaml(@filepath)
       else

data/lib/packwerk/file_processor.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: strict
 # frozen_string_literal: true
 
 require "ast/node"
@@ -8,55 +8,76 @@ module Packwerk
     extend T::Sig
 
     class UnknownFileTypeResult < Offense
+      extend T::Sig
+
+      sig { params(file: String).void }
       def initialize(file:)
         super(file: file, message: "unknown file type")
       end
     end
 
-    def initialize(node_processor_factory:, parser_factory: nil)
+    sig do
+      params(
+        node_processor_factory: NodeProcessorFactory,
+        cache: Cache,
+        parser_factory: T.nilable(Parsers::Factory)
+      ).void
+    end
+    def initialize(node_processor_factory:, cache:, parser_factory: nil)
       @node_processor_factory = node_processor_factory
-      @parser_factory = parser_factory || Packwerk::Parsers::Factory.instance
+      @cache = cache
+      @parser_factory = T.let(parser_factory || Packwerk::Parsers::Factory.instance, Parsers::Factory)
+    end
+
+    class ProcessedFile < T::Struct
+      const :unresolved_references, T::Array[UnresolvedReference], default: []
+      const :offenses, T::Array[Offense], default: []
     end
 
     sig do
-      params(file_path: String).returns(
-        T::Array[
-          T.any(
-            Packwerk::Reference,
-            Packwerk::Offense,
-          )
-        ]
-      )
+      params(absolute_file: String).returns(ProcessedFile)
     end
-    def call(file_path)
-      return [UnknownFileTypeResult.new(file: file_path)] if parser_for(file_path).nil?
+    def call(absolute_file)
+      parser = parser_for(absolute_file)
+      if parser.nil?
+        return ProcessedFile.new(offenses: [UnknownFileTypeResult.new(file: absolute_file)])
+      end
+
+      unresolved_references = @cache.with_cache(absolute_file) do
+        node = parse_into_ast(absolute_file, parser)
+        return ProcessedFile.new unless node
 
-      node = parse_into_ast(file_path)
-      return [] unless node
+        references_from_ast(node, absolute_file)
+      end
 
-      references_from_ast(node, file_path)
+      ProcessedFile.new(unresolved_references: unresolved_references)
     rescue Parsers::ParseError => e
-      [e.result]
+      ProcessedFile.new(offenses: [e.result])
     end
 
     private
 
-    def references_from_ast(node, file_path)
+    sig do
+      params(node: Parser::AST::Node, absolute_file: String).returns(T::Array[UnresolvedReference])
+    end
+    def references_from_ast(node, absolute_file)
       references = []
 
-      node_processor = @node_processor_factory.for(filename: file_path, node: node)
+      node_processor = @node_processor_factory.for(absolute_file: absolute_file, node: node)
       node_visitor = Packwerk::NodeVisitor.new(node_processor: node_processor)
       node_visitor.visit(node, ancestors: [], result: references)
 
       references
     end
 
-    def parse_into_ast(file_path)
-      File.open(file_path, "r", nil, external_encoding: Encoding::UTF_8) do |file|
-        parser_for(file_path).call(io: file, file_path: file_path)
+    sig { params(absolute_file: String, parser: Parsers::ParserInterface).returns(T.untyped) }
+    def parse_into_ast(absolute_file, parser)
+      File.open(absolute_file, "r", nil, external_encoding: Encoding::UTF_8) do |file|
+        parser.call(io: file, file_path: absolute_file)
       end
     end
 
+    sig { params(file_path: String).returns(T.nilable(Parsers::ParserInterface)) }
     def parser_for(file_path)
       @parser_factory.for_path(file_path)
     end

data/lib/packwerk/files_for_processing.rb

@@ -1,20 +1,42 @@
-# typed: true
+# typed: strict
 # frozen_string_literal: true
 
 module Packwerk
   class FilesForProcessing
+    extend T::Sig
+
+    AbsoluteFileSet = T.type_alias { T::Set[String] }
+
     class << self
-      def fetch(paths:, configuration:, ignore_nested_packages: false)
-        new(paths, configuration, ignore_nested_packages).files
+      extend T::Sig
+
+      sig do
+        params(
+          relative_file_paths: T::Array[String],
+          configuration: Configuration,
+          ignore_nested_packages: T::Boolean
+        ).returns(AbsoluteFileSet)
+      end
+      def fetch(relative_file_paths:, configuration:, ignore_nested_packages: false)
+        new(relative_file_paths, configuration, ignore_nested_packages).files
       end
     end
 
-    def initialize(paths, configuration, ignore_nested_packages)
-      @paths = paths
+    sig do
+      params(
+        relative_file_paths: T::Array[String],
+        configuration: Configuration,
+        ignore_nested_packages: T::Boolean
+      ).void
+    end
+    def initialize(relative_file_paths, configuration, ignore_nested_packages)
+      @relative_file_paths = relative_file_paths
       @configuration = configuration
       @ignore_nested_packages = ignore_nested_packages
+      @custom_files = T.let(nil, T.nilable(AbsoluteFileSet))
     end
 
+    sig { returns(AbsoluteFileSet) }
     def files
       include_files = if custom_files.empty?
         configured_included_files
@@ -27,52 +49,59 @@ module Packwerk
 
     private
 
+    sig { returns(AbsoluteFileSet) }
     def custom_files
-      @custom_files ||= @paths.flat_map do |path|
-        path = File.expand_path(path, @configuration.root_path)
-        if File.file?(path)
-          path
-        else
-          custom_included_files(path)
+      @custom_files ||= Set.new(
+        @relative_file_paths.map do |relative_file_path|
+          absolute_file_path = File.expand_path(relative_file_path, @configuration.root_path)
+          if File.file?(absolute_file_path)
+            absolute_file_path
+          else
+            custom_included_files(absolute_file_path)
+          end
         end
-      end
+      ).flatten
     end
 
-    def custom_included_files(path)
+    sig { params(absolute_file_path: String).returns(AbsoluteFileSet) }
+    def custom_included_files(absolute_file_path)
       # Note, assuming include globs are always relative paths
       absolute_includes = @configuration.include.map do |glob|
         File.expand_path(glob, @configuration.root_path)
       end
 
-      files = Dir.glob([File.join(path, "**", "*")]).select do |file_path|
+      absolute_files = Dir.glob([File.join(absolute_file_path, "**", "*")]).select do |absolute_path|
         absolute_includes.any? do |pattern|
-          File.fnmatch?(pattern, file_path, File::FNM_EXTGLOB)
+          File.fnmatch?(pattern, absolute_path, File::FNM_EXTGLOB)
         end
       end
 
       if @ignore_nested_packages
-        nested_packages_paths = Dir.glob(File.join(path, "*", "**", "package.yml"))
-        nested_packages_globs = nested_packages_paths.map { |npp| npp.gsub("package.yml", "**/*") }
-        nested_packages_globs.each do |glob|
-          files -= Dir.glob(glob)
+        nested_packages_absolute_file_paths = Dir.glob(File.join(absolute_file_path, "*", "**", "package.yml"))
+        nested_packages_absolute_globs = nested_packages_absolute_file_paths.map do |npp|
+          npp.gsub("package.yml", "**/*")
+        end
+        nested_packages_absolute_globs.each do |absolute_glob|
+          absolute_files -= Dir.glob(absolute_glob)
         end
       end
 
-      files
+      Set.new(absolute_files)
     end
 
+    sig { returns(AbsoluteFileSet) }
     def configured_included_files
-      files_for_globs(@configuration.include)
+      absolute_files_for_globs(@configuration.include)
     end
 
+    sig { returns(AbsoluteFileSet) }
     def configured_excluded_files
-      files_for_globs(@configuration.exclude)
+      absolute_files_for_globs(@configuration.exclude)
     end
 
-    def files_for_globs(globs)
-      globs
-        .flat_map { |glob| Dir[File.expand_path(glob, @configuration.root_path)] }
-        .uniq
+    sig { params(relative_globs: T::Array[String]).returns(AbsoluteFileSet) }
+    def absolute_files_for_globs(relative_globs)
+      Set.new(relative_globs.flat_map { |glob| Dir[File.expand_path(glob, @configuration.root_path)] })
     end
   end
 end

data/lib/packwerk/generators/templates/packwerk.yml.erb

@@ -15,3 +15,9 @@
 # List of custom associations, if any
 # custom_associations:
 # - "cache_belongs_to"
+
+# Whether or not you want the cache enabled (disabled by default)
+# cache: true
+
+# Where you want the cache to be stored (default below)
+# cache_directory: 'tmp/cache/packwerk'