packwerk 1.3.1 → 2.1.0

data/lib/packwerk/cache.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+# typed: strict
+
+require "digest"
+
+module Packwerk
+  class Cache
+    extend T::Sig
+
+    class CacheContents < T::Struct
+      extend T::Sig
+
+      const :file_contents_digest, String
+      const :unresolved_references, T::Array[UnresolvedReference]
+
+      sig { returns(String) }
+      def serialize
+        to_json
+      end
+
+      sig { params(serialized_cache_contents: String).returns(CacheContents) }
+      def self.deserialize(serialized_cache_contents)
+        cache_contents_json = JSON.parse(serialized_cache_contents)
+        unresolved_references = cache_contents_json["unresolved_references"].map do |json|
+          UnresolvedReference.new(
+            json["constant_name"],
+            json["namespace_path"],
+            json["relative_path"],
+            Node::Location.new(json["source_location"]["line"], json["source_location"]["column"],)
+          )
+        end
+
+        CacheContents.new(
+          file_contents_digest: cache_contents_json["file_contents_digest"],
+          unresolved_references: unresolved_references,
+        )
+      end
+    end
+
+    CACHE_SHAPE = T.type_alias do
+      T::Hash[
+        String,
+        CacheContents
+      ]
+    end
+
+    sig { params(enable_cache: T::Boolean, cache_directory: Pathname, config_path: String).void }
+    def initialize(enable_cache:, cache_directory:, config_path:)
+      @enable_cache = enable_cache
+      @cache = T.let({}, CACHE_SHAPE)
+      @files_by_digest = T.let({}, T::Hash[String, String])
+      @config_path = config_path
+      @cache_directory = cache_directory
+
+      if @enable_cache
+        create_cache_directory!
+        bust_cache_if_packwerk_yml_has_changed!
+        bust_cache_if_inflections_have_changed!
+      end
+    end
+
+    sig { void }
+    def bust_cache!
+      FileUtils.rm_rf(@cache_directory)
+    end
+
+    sig do
+      params(
+        file_path: String,
+        block: T.proc.returns(T::Array[UnresolvedReference])
+      ).returns(T::Array[UnresolvedReference])
+    end
+    def with_cache(file_path, &block)
+      return block.call unless @enable_cache
+
+      cache_location = @cache_directory.join(digest_for_string(file_path))
+
+      cache_contents = if cache_location.exist?
+        T.let(CacheContents.deserialize(cache_location.read),
+          CacheContents)
+      end
+
+      file_contents_digest = digest_for_file(file_path)
+
+      if !cache_contents.nil? && cache_contents.file_contents_digest == file_contents_digest
+        Debug.out("Cache hit for #{file_path}")
+
+        cache_contents.unresolved_references
+      else
+        Debug.out("Cache miss for #{file_path}")
+
+        unresolved_references = block.call
+
+        cache_contents = CacheContents.new(
+          file_contents_digest: file_contents_digest,
+          unresolved_references: unresolved_references,
+        )
+        cache_location.write(cache_contents.serialize)
+
+        unresolved_references
+      end
+    end
+
+    sig { params(file: String).returns(String) }
+    def digest_for_file(file)
+      digest_for_string(File.read(file))
+    end
+
+    sig { params(str: String).returns(String) }
+    def digest_for_string(str)
+      # MD5 appears to be the fastest
+      # https://gist.github.com/morimori/1330095
+      Digest::MD5.hexdigest(str)
+    end
+
+    sig { void }
+    def bust_cache_if_packwerk_yml_has_changed!
+      bust_cache_if_contents_have_changed(File.read(@config_path), :packwerk_yml)
+    end
+
+    sig { void }
+    def bust_cache_if_inflections_have_changed!
+      bust_cache_if_contents_have_changed(YAML.dump(ActiveSupport::Inflector.inflections), :inflections)
+    end
+
+    sig { params(contents: String, contents_key: Symbol).void }
+    def bust_cache_if_contents_have_changed(contents, contents_key)
+      current_digest = digest_for_string(contents)
+      cached_digest_path = @cache_directory.join(contents_key.to_s)
+
+      if !cached_digest_path.exist?
+        # In this case, we have nothing cached
+        # We save the current digest. This way the next time we compare current digest to cached digest,
+        # we can accurately determine if we should bust the cache
+        cached_digest_path.write(current_digest)
+
+        nil
+      elsif cached_digest_path.read == current_digest
+        Debug.out("#{contents_key} contents have NOT changed, preserving cache")
+      else
+        Debug.out("#{contents_key} contents have changed, busting cache")
+
+        bust_cache!
+        create_cache_directory!
+
+        cached_digest_path.write(current_digest)
+      end
+    end
+
+    sig { void }
+    def create_cache_directory!
+      FileUtils.mkdir_p(@cache_directory)
+    end
+  end
+
+  class Debug
+    extend T::Sig
+
+    sig { params(out: String).void }
+    def self.out(out)
+      if ENV["DEBUG_PACKWERK_CACHE"]
+        puts(out)
+      end
+    end
+  end
+
+  private_constant :Debug
+end

data/lib/packwerk/cli.rb

@@ -1,7 +1,10 @@
 # typed: true
 # frozen_string_literal: true
 
+require "optparse"
+
 module Packwerk
+  # A command-line interface to Packwerk.
   class Cli
     extend T::Sig
 
@@ -50,8 +53,6 @@ module Packwerk
         output_result(parse_run(args).check)
       when "detect-stale-violations"
         output_result(parse_run(args).detect_stale_violations)
-      when "update"
-        update(args)
       when "update-deprecations"
         output_result(parse_run(args).update_deprecations)
       when "validate"
@@ -85,19 +86,18 @@ module Packwerk
 
     def generate_configs
       configuration_file = Packwerk::Generators::ConfigurationFile.generate(
-        load_paths: Packwerk::ApplicationLoadPaths.extract_relevant_paths(@configuration.root_path, @environment),
         root: @configuration.root_path,
         out: @out
       )
-      inflections_file = Packwerk::Generators::InflectionsFile.generate(root: @configuration.root_path, out: @out)
+
       root_package = Packwerk::Generators::RootPackage.generate(root: @configuration.root_path, out: @out)
 
-      success = configuration_file && inflections_file && root_package
+      success = configuration_file && root_package
 
       result = if success
         <<~EOS
 
-          🎉 Packwerk is ready to be used. You can start defining packages and run `packwerk check`.
+          🎉 Packwerk is ready to be used. You can start defining packages and run `bin/packwerk check`.
           For more information on how to use Packwerk, see: https://github.com/Shopify/packwerk/blob/main/USAGE.md
         EOS
       else
@@ -112,19 +112,18 @@ module Packwerk
       success
     end
 
-    def update(paths)
-      warn("`packwerk update` is deprecated in favor of `packwerk update-deprecations`.")
-      output_result(parse_run(paths).update_deprecations)
-    end
-
     def output_result(result)
       @out.puts
       @out.puts(result.message)
       result.status
     end
 
-    def fetch_files_to_process(paths)
-      files = FilesForProcessing.fetch(paths: paths, configuration: @configuration)
+    def fetch_files_to_process(paths, ignore_nested_packages)
+      files = FilesForProcessing.fetch(
+        paths: paths,
+        ignore_nested_packages: ignore_nested_packages,
+        configuration: @configuration
+      )
       abort("No files found or given. "\
         "Specify files or check the include and exclude glob in the config file.") if files.empty?
       files
@@ -132,11 +131,6 @@ module Packwerk
 
     def validate(_paths)
       @progress_formatter.started_validation do
-        checker = Packwerk::ApplicationValidator.new(
-          config_file_path: @configuration.config_path,
-          configuration: @configuration,
-          environment: @environment,
-        )
         result = checker.check_all
 
         list_validation_errors(result)
@@ -145,6 +139,14 @@ module Packwerk
       end
     end
 
+    def checker
+      Packwerk::ApplicationValidator.new(
+        config_file_path: @configuration.config_path,
+        configuration: @configuration,
+        environment: @environment,
+      )
+    end
+
     def list_validation_errors(result)
       @out.puts
       if result.ok?
@@ -155,9 +157,24 @@ module Packwerk
       end
     end
 
-    def parse_run(paths)
+    def parse_run(params)
+      paths = T.let([], T::Array[String])
+      ignore_nested_packages = nil
+
+      if params.any? { |p| p.include?("--packages") }
+        OptionParser.new do |parser|
+          parser.on("--packages=PACKAGESLIST", Array, "package names, comma separated") do |p|
+            paths = p
+          end
+        end.parse!(params)
+        ignore_nested_packages = true
+      else
+        paths = params
+        ignore_nested_packages = false
+      end
+
       ParseRun.new(
-        files: fetch_files_to_process(paths),
+        files: fetch_files_to_process(paths, ignore_nested_packages),
         configuration: @configuration,
         progress_formatter: @progress_formatter,
         offenses_formatter: @offenses_formatter

data/lib/packwerk/configuration.rb

@@ -34,8 +34,7 @@ module Packwerk
     DEFAULT_EXCLUDE_GLOBS = ["{bin,node_modules,script,tmp,vendor}/**/*"]
 
     attr_reader(
-      :include, :exclude, :root_path, :package_paths, :custom_associations, :load_paths, :inflections_file,
-      :config_path,
+      :include, :exclude, :root_path, :package_paths, :custom_associations, :config_path, :cache_directory
     )
 
     def initialize(configs = {}, config_path: nil)
@@ -45,15 +44,51 @@ module Packwerk
       @root_path = File.expand_path(root)
       @package_paths = configs["package_paths"] || "**/"
       @custom_associations = configs["custom_associations"] || []
-      @load_paths = configs["load_paths"] || []
-      @inflections_file = File.expand_path(configs["inflections_file"] || "config/inflections.yml", @root_path)
       @parallel = configs.key?("parallel") ? configs["parallel"] : true
-
+      @cache_enabled = configs.key?("cache") ? configs["cache"] : false
+      @cache_directory = Pathname.new(configs["cache_directory"] || "tmp/cache/packwerk")
       @config_path = config_path
+
+      if configs["load_paths"]
+        warning = <<~WARNING
+          DEPRECATION WARNING: The 'load_paths' key in `packwerk.yml` is deprecated.
+          This value is no longer cached, and you can remove the key from `packwerk.yml`.
+        WARNING
+
+        warn(warning)
+      end
+
+      if configs["inflections_file"]
+        warning = <<~WARNING
+          DEPRECATION WARNING: The 'inflections_file' key in `packwerk.yml` is deprecated.
+          This value is no longer cached, and you can remove the key from `packwerk.yml`.
+          You can also delete #{configs["inflections_file"]}.
+        WARNING
+
+        warn(warning)
+      end
+
+      inflection_file = File.expand_path(configs["inflections_file"] || "config/inflections.yml", @root_path)
+      if Pathname.new(inflection_file).exist?
+        warning = <<~WARNING
+          DEPRECATION WARNING: Inflections YMLs in packwerk are now deprecated.
+          This value is no longer cached, and you can now delete #{inflection_file}
+        WARNING
+
+        warn(warning)
+      end
+    end
+
+    def load_paths
+      @load_paths ||= ApplicationLoadPaths.extract_relevant_paths(@root_path, "test")
     end
 
     def parallel?
       @parallel
     end
+
+    def cache_enabled?
+      @cache_enabled
+    end
   end
 end

data/lib/packwerk/const_node_inspector.rb

@@ -15,6 +15,9 @@ module Packwerk
     def constant_name_from_node(node, ancestors:)
       return nil unless Node.constant?(node)
       parent = ancestors.first
+
+      # Only process the root `const` node for namespaced constant references. For example, in the
+      # reference `Spam::Eggs::Thing`, we only process the const node associated with `Spam`.
       return nil unless root_constant?(parent)
 
       if parent && constant_in_module_or_class_definition?(node, parent: parent)
@@ -30,8 +33,6 @@ module Packwerk
 
     private
 
-    # Only process the root `const` node for namespaced constant references. For example, in the
-    # reference `Spam::Eggs::Thing`, we only process the const node associated with `Spam`.
     sig { params(parent: T.nilable(AST::Node)).returns(T::Boolean) }
     def root_constant?(parent)
       !(parent && Node.constant?(parent))

data/lib/packwerk/constant_discovery.rb

@@ -4,7 +4,7 @@
 require "constant_resolver"
 
 module Packwerk
-  # Get information about (partially qualified) constants without loading the application code.
+  # Get information about unresolved constants without loading the application code.
   # Information gathered: Fully qualified name, path to file containing the definition, package,
   # and visibility (public/private to the package).
   #
@@ -35,9 +35,9 @@ module Packwerk
     end
 
     # Analyze a constant via its name.
-    # If the name is partially qualified, we need the current namespace path to correctly infer its full name
+    # If the constant is unresolved, we need the current namespace path to correctly infer its full name
     #
-    # @param const_name [String] The constant's name, fully or partially qualified.
+    # @param const_name [String] The unresolved constant's name.
     # @param current_namespace_path [Array<String>] (optional) The namespace of the context in which the constant is
     #   used, e.g. ["Apps", "Models"] for `Apps::Models`. Defaults to [] which means top level.
     # @return [Packwerk::ConstantDiscovery::ConstantContext]
@@ -45,7 +45,7 @@ module Packwerk
       begin
         constant = @resolver.resolve(const_name, current_namespace_path: current_namespace_path)
       rescue ConstantResolver::Error => e
-        raise(ConstantResolver::Error, e.message + "\n Make sure autoload paths are added to the config file.")
+        raise(ConstantResolver::Error, e.message)
       end
 
       return unless constant

data/lib/packwerk/constant_name_inspector.rb

@@ -4,7 +4,7 @@
 require "ast"
 
 module Packwerk
-  # An interface describing some object that can extract a constant name from an AST node
+  # An interface describing an object that can extract a constant name from an AST node.
   module ConstantNameInspector
     extend T::Sig
     extend T::Helpers

data/lib/packwerk/deprecated_references.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: strict
 # frozen_string_literal: true
 
 require "yaml"
@@ -7,11 +7,16 @@ module Packwerk
   class DeprecatedReferences
     extend T::Sig
 
+    ENTRIES_TYPE = T.type_alias do
+      T::Hash[String, T.untyped]
+    end
+
     sig { params(package: Packwerk::Package, filepath: String).void }
     def initialize(package, filepath)
       @package = package
       @filepath = filepath
-      @new_entries = {}
+      @new_entries = T.let({}, ENTRIES_TYPE)
+      @deprecated_references = T.let(nil, T.nilable(ENTRIES_TYPE))
     end
 
     sig do
@@ -73,7 +78,7 @@ module Packwerk
           #
           # You can regenerate this file using the following command:
           #
-          # packwerk update-deprecations #{@package.name}
+          # bin/packwerk update-deprecations #{@package.name}
         MESSAGE
         File.open(@filepath, "w") do |f|
           f.write(message)
@@ -84,7 +89,7 @@ module Packwerk
 
     private
 
-    sig { returns(Hash) }
+    sig { returns(ENTRIES_TYPE) }
     def prepare_entries_for_dump
       @new_entries.each do |package_name, package_violations|
         package_violations.each do |_, entries_for_file|
@@ -97,13 +102,20 @@ module Packwerk
       @new_entries = @new_entries.sort.to_h
     end
 
-    sig { returns(Hash) }
+    sig { returns(ENTRIES_TYPE) }
     def deprecated_references
       @deprecated_references ||= if File.exist?(@filepath)
-        YAML.load_file(@filepath) || {}
+        load_yaml(@filepath)
       else
         {}
       end
     end
+
+    sig { params(filepath: String).returns(ENTRIES_TYPE) }
+    def load_yaml(filepath)
+      YAML.load_file(filepath) || {}
+    rescue Psych::Exception
+      {}
+    end
   end
 end

data/lib/packwerk/file_processor.rb

@@ -1,39 +1,78 @@
-# typed: false
+# typed: true
 # frozen_string_literal: true
 
 require "ast/node"
 
 module Packwerk
   class FileProcessor
+    extend T::Sig
+
     class UnknownFileTypeResult < Offense
       def initialize(file:)
         super(file: file, message: "unknown file type")
       end
     end
 
-    def initialize(node_processor_factory:, parser_factory: nil)
+    sig do
+      params(
+        node_processor_factory: NodeProcessorFactory,
+        cache: Cache,
+        parser_factory: T.nilable(Parsers::Factory)
+      ).void
+    end
+    def initialize(node_processor_factory:, cache:, parser_factory: nil)
       @node_processor_factory = node_processor_factory
+      @cache = cache
       @parser_factory = parser_factory || Packwerk::Parsers::Factory.instance
     end
 
+    sig do
+      params(file_path: String).returns(
+        T::Array[
+          T.any(
+            Packwerk::UnresolvedReference,
+            Packwerk::Offense,
+          )
+        ]
+      )
+    end
     def call(file_path)
-      parser = @parser_factory.for_path(file_path)
-      return [UnknownFileTypeResult.new(file: file_path)] if parser.nil?
+      return [UnknownFileTypeResult.new(file: file_path)] if parser_for(file_path).nil?
+
+      @cache.with_cache(file_path) do
+        node = parse_into_ast(file_path)
 
-      node = File.open(file_path, "r", external_encoding: Encoding::UTF_8) do |file|
-        parser.call(io: file, file_path: file_path)
-      rescue Parsers::ParseError => e
-        return [e.result]
+        return [] unless node
+
+        references_from_ast(node, file_path)
       end
+    rescue Parsers::ParseError => e
+      [e.result]
+    end
 
-      result = []
-      if node
-        node_processor = @node_processor_factory.for(filename: file_path, node: node)
-        node_visitor = Packwerk::NodeVisitor.new(node_processor: node_processor)
+    private
 
-        node_visitor.visit(node, ancestors: [], result: result)
+    sig do
+      params(node: Parser::AST::Node, file_path: String).returns(T::Array[UnresolvedReference])
+    end
+    def references_from_ast(node, file_path)
+      references = []
+
+      node_processor = @node_processor_factory.for(filename: file_path, node: node)
+      node_visitor = Packwerk::NodeVisitor.new(node_processor: node_processor)
+      node_visitor.visit(node, ancestors: [], result: references)
+
+      references
+    end
+
+    def parse_into_ast(file_path)
+      File.open(file_path, "r", nil, external_encoding: Encoding::UTF_8) do |file|
+        parser_for(file_path).call(io: file, file_path: file_path)
       end
-      result
+    end
+
+    def parser_for(file_path)
+      @parser_factory.for_path(file_path)
     end
   end
 end

data/lib/packwerk/files_for_processing.rb

@@ -4,14 +4,15 @@
 module Packwerk
   class FilesForProcessing
     class << self
-      def fetch(paths:, configuration:)
-        new(paths, configuration).files
+      def fetch(paths:, configuration:, ignore_nested_packages: false)
+        new(paths, configuration, ignore_nested_packages).files
       end
     end
 
-    def initialize(paths, configuration)
+    def initialize(paths, configuration, ignore_nested_packages)
       @paths = paths
       @configuration = configuration
+      @ignore_nested_packages = ignore_nested_packages
     end
 
     def files
@@ -43,11 +44,21 @@ module Packwerk
         File.expand_path(glob, @configuration.root_path)
       end
 
-      Dir.glob([File.join(path, "**", "*")]).select do |file_path|
+      files = Dir.glob([File.join(path, "**", "*")]).select do |file_path|
         absolute_includes.any? do |pattern|
           File.fnmatch?(pattern, file_path, File::FNM_EXTGLOB)
         end
       end
+
+      if @ignore_nested_packages
+        nested_packages_paths = Dir.glob(File.join(path, "*", "**", "package.yml"))
+        nested_packages_globs = nested_packages_paths.map { |npp| npp.gsub("package.yml", "**/*") }
+        nested_packages_globs.each do |glob|
+          files -= Dir.glob(glob)
+        end
+      end
+
+      files
     end
 
     def configured_included_files

data/lib/packwerk/formatters/offenses_formatter.rb

@@ -44,7 +44,7 @@ module Packwerk
 
       sig { params(offenses: T::Array[T.nilable(Offense)]).returns(String) }
       def offenses_summary(offenses)
-        offenses_string = Inflector.default.pluralize("offense", offenses.length)
+        offenses_string = "offense".pluralize(offenses.length)
         "#{offenses.length} #{offenses_string} detected"
       end
     end

data/lib/packwerk/formatters/progress_formatter.rb

@@ -16,7 +16,7 @@ module Packwerk
 
       def started(target_files)
         files_size = target_files.size
-        files_string = Inflector.default.pluralize("file", files_size)
+        files_string = "file".pluralize(files_size)
         @out.puts("📦 Packwerk is inspecting #{files_size} #{files_string}")
       end
 

data/lib/packwerk/generators/configuration_file.rb

@@ -11,18 +11,15 @@ module Packwerk
       CONFIGURATION_TEMPLATE_FILE_PATH = "templates/packwerk.yml.erb"
 
       class << self
-        def generate(load_paths:, root:, out:)
-          new(load_paths: load_paths, root: root, out: out).generate
+        def generate(root:, out:)
+          new(root: root, out: out).generate
         end
       end
 
-      sig { params(load_paths: T::Array[String], root: String, out: T.any(StringIO, IO)).void }
-      def initialize(load_paths:, root:, out: $stdout)
-        @load_paths = load_paths
+      sig { params(root: String, out: T.any(StringIO, IO)).void }
+      def initialize(root:, out: $stdout)
         @root = root
         @out = out
-
-        set_template_variables
       end
 
       sig { returns(T::Boolean) }
@@ -43,18 +40,6 @@ module Packwerk
 
       private
 
-      def set_template_variables
-        @load_paths_formatted = if @load_paths.empty?
-          "# load_paths:\n# - 'app/models'\n"
-        else
-          @load_paths.map { |path| "- #{path}\n" }.join
-        end
-
-        @load_paths_comment = unless @load_paths.empty?
-          "# These load paths were auto generated by Packwerk.\nload_paths:\n"
-        end
-      end
-
       def render
         ERB.new(template, trim_mode: "-").result(binding)
       end

data/lib/packwerk/generators/templates/package.yml

@@ -1,5 +1,5 @@
 # This file represents the root package of the application
-# Please validate the configuration using `bin/packwerk validate` (for Rails applications) or running the auto generated
+# Please validate the configuration using `packwerk validate` (for Rails applications) or running the auto generated
 # test case (for non-Rails projects). You can then use `packwerk check` to check your code.
 
 # Turn on dependency checks for this package