rails-ai-context 5.30.2 → 5.32.0

data/lib/rails_ai_context/ast_cache.rb

@@ -6,15 +6,15 @@ require "prism"
 
 module RailsAiContext
   # Thread-safe AST parse cache backed by Concurrent::Map.
-  # Keyed by path + content hash + mtime - automatically invalidates
-  # when file content changes. Used by all Prism-based introspectors.
+  # Keyed by content hash, so a file and the same text read as a string share one
+  # parse, and a changed file misses. Used by all Prism-based introspectors.
   # A settled file's stat stands in for the read and hash (see RACY_WINDOW).
   #
   # Bounded: evicts entries when MAX_SIZE is exceeded, and a file's recorded
   # stat goes with its parse.
   module AstCache
     STORE = Concurrent::Map.new
-    SEEN = Concurrent::Map.new # path => [stat signature, STORE key, whether the stat can be trusted]
+    SEEN = Concurrent::Map.new # path, or [path, ruby grammar] => [stat signature, STORE key, whether the stat can be trusted]
     MAX_SIZE = 500
     EVICTION_MUTEX = Mutex.new
 
@@ -27,12 +27,15 @@ module RailsAiContext
     #
     # Reads content first, then checks size - avoids TOCTOU race where the
     # file could change between File.size and File.read.
-    def self.parse(path)
-      # An unchanged file answers from its stat: the same concern file reaches
-      # every model's walk, and reading and hashing it each time dominated the
-      # model tier on Canvas. Within one run the stat is asked once.
+    #
+    # `ruby:` is the Ruby version whose grammar decides what is a syntax error;
+    # nil parses as the newest Ruby prism knows.
+    def self.parse(path, ruby: nil)
+      version = prism_version(ruby)
+      seen_key = version ? [ path, version ] : path
+      # An unchanged file answers from its stat; one concern file reaches every model's walk.
       signature = RunCache.fetch([ :stat_signature, path.to_s ]) { stat_signature(path) }
-      seen = SEEN[path]
+      seen = SEEN[seen_key]
       if seen && seen[2] && seen.first == signature && (cached = STORE[seen[1]])
         return cached
       end
@@ -42,16 +45,15 @@ module RailsAiContext
       size = content.bytesize
       raise ArgumentError, "File too large for AST parsing: #{path} (#{size} bytes, max #{MAX_PARSE_SIZE})" if size > MAX_PARSE_SIZE
 
-      mtime = File.mtime(path).to_i
-      key   = "#{path}:#{Digest::SHA256.hexdigest(content)}:#{mtime}"
+      key = content_key(content, version)
 
       cached = STORE[key]
       unless cached
         # Evict BEFORE inserting to avoid running inside compute_if_absent
         evict_if_full
-        cached = STORE.compute_if_absent(key) { Prism.parse(content) }
+        cached = STORE.compute_if_absent(key) { prism_parse(content, version) }
       end
-      SEEN[path] = [ signature, key, settled?(signature, read_at) ]
+      SEEN[seen_key] = [ signature, key, settled?(signature, read_at) ]
       cached
     end
 
@@ -83,18 +85,51 @@ module RailsAiContext
 
     # Parse a Ruby source string, cached by content digest, so every extractor
     # handed the same text shares one parse.
-    def self.parse_string(source)
-      return Prism.parse(source) if source.bytesize > MAX_PARSE_SIZE
+    def self.parse_string(source, ruby: nil)
+      version = prism_version(ruby)
+      return prism_parse(source, version) if source.bytesize > MAX_PARSE_SIZE
 
-      key = "string:#{Digest::SHA256.hexdigest(source)}"
+      key = content_key(source, version)
 
       cached = STORE[key]
       return cached if cached
 
       evict_if_full
 
-      STORE.compute_if_absent(key) { Prism.parse(source) }
+      STORE.compute_if_absent(key) { prism_parse(source, version) }
+    end
+
+    def self.content_key(source, version)
+      "src:#{Digest::SHA256.hexdigest(source)}#{":#{version}" if version}"
+    end
+    private_class_method :content_key
+
+    OLDEST_GRAMMAR = "3.3"
+    GRAMMARS = Concurrent::Map.new # major.minor => whether the loaded prism parses it
+
+    # The grammar prism parses `ruby` with: its own major.minor, the oldest prism
+    # knows for an older Ruby, and nil (the newest) for one newer than prism knows.
+    # ponytail: Ruby 3.1 and 3.2 parse as 3.3; syntax only they accept reads as an error.
+    def self.prism_version(ruby)
+      wanted = ruby.to_s.b[/\A\d+\.\d+/]
+      return nil unless wanted
+
+      wanted = OLDEST_GRAMMAR if Gem::Version.new(wanted) < Gem::Version.new(OLDEST_GRAMMAR)
+      wanted if GRAMMARS.compute_if_absent(wanted) { supported?(wanted) }
+    end
+
+    def self.supported?(version)
+      Prism.parse("", version: version)
+      true
+    rescue ArgumentError
+      false
+    end
+    private_class_method :supported?
+
+    def self.prism_parse(source, version)
+      version ? Prism.parse(source, version: version) : Prism.parse(source)
     end
+    private_class_method :prism_parse
 
     # Clear the entire cache.
     def self.clear
@@ -114,8 +149,8 @@ module RailsAiContext
         return if STORE.size < MAX_SIZE
         keys = STORE.keys
         evicted = keys.first(keys.size / 4).each { |k| STORE.delete(k) }.to_set
-        # A stat that points at an evicted parse can only send the next call to
-        # read the file, so it goes with it, and SEEN stays within MAX_SIZE.
+        # A stat goes with the parse it points at; files with the same bytes share
+        # one parse, so SEEN can outgrow STORE but not the app's file count.
         SEEN.each_pair { |path, entry| SEEN.delete(path) if evicted.include?(entry[1]) }
       end
     end

data/lib/rails_ai_context/base_mixins.rb

@@ -6,6 +6,8 @@ module RailsAiContext
   module BaseMixins
     # `path` is nil when the declaring file is not the app's (a gem's module).
     Mixin = Struct.new(:name, :path, :macro)
+    # A base body's statements, the base it runs in, and its file's place in load order (sorted, as Rails loads initializers).
+    Body = Struct.new(:statements, :target, :order)
 
     TARGETS = %w[ActiveRecord::Base ApplicationRecord].freeze
     HOOK = :active_record
@@ -21,12 +23,22 @@ module RailsAiContext
       RunCache.fetch([ :base_mixins, root ]) { discover(root) }
     end
 
-    def discover(root)
-      scanned_files(root).flat_map do |file|
-        source = SafeFile.read(file)
-        next [] unless source && MARKERS.any? { |marker| source.include?(marker) }
+    # Each model base body the app writes outside app/models, where a `def self.x` or
+    # `class << self` defines a class method every model under that base has.
+    # @return [Array<Body>]
+    def bodies(root)
+      root = File.expand_path(root.to_s)
+      RunCache.fetch([ :base_bodies, root ]) do
+        marked(root).each_with_index.flat_map do |(_, tree), order|
+          base_bodies(tree).map { |statements, target, _| Body.new(statements, target, order) }
+        end
+      end
+    rescue StandardError => e
+      RailsAiContext.debug_fail(e, [], label: "BaseMixins.bodies")
+    end
 
-        tree = AstCache.parse(file).value
+    def discover(root)
+      marked(root).flat_map do |file, tree|
         mixed_in(tree).map { |macro, name| Mixin.new(name, declaring_file(root, name, file), macro) }
       end.uniq(&:name)
     rescue StandardError => e
@@ -34,6 +46,17 @@ module RailsAiContext
     end
     private_class_method :discover
 
+    # [file, tree] for each scanned file that names a model base; models and bodies both ask.
+    def marked(root)
+      RunCache.fetch([ :base_mixins_marked, root ]) do
+        scanned_files(root).filter_map do |file|
+          source = SafeFile.read(file)
+          [ file, AstCache.parse(file).value ] if source && MARKERS.any? { |marker| source.include?(marker) }
+        end
+      end
+    end
+    private_class_method :marked
+
     # Everywhere the app configures itself but app/models: a model file's includes are
     # its own, and the model walk reads that file already.
     def scanned_files(root)
@@ -50,25 +73,31 @@ module RailsAiContext
     private_class_method :scanned_files
 
     # [macro, constant] for each module the tree mixes into a model base:
-    # sent to it, run in its `on_load` hook, or written in its reopened body.
+    # sent to it, run in its `on_load` hook, or written in its own body.
     def mixed_in(tree)
       Introspectors::AstWalk.each(tree).flat_map do |node|
         next [] unless node.is_a?(Prism::CallNode)
         next hook_mixins(node) if hook?(node)
 
         target?(node.receiver) ? mixin_calls(node) : []
-      end + reopened_mixins(tree)
+      end + base_bodies(tree).reject(&:last).flat_map(&:first).flat_map { |call| call.is_a?(Prism::CallNode) && call.receiver.nil? ? mixin_calls(call) : [] }
     end
 
-    # `class ActiveRecord::Base; include X; end`, however the namespace is written.
-    def reopened_mixins(tree)
-      Introspectors::DeclaredConstant.constants(tree).flat_map do |name, node|
-        next [] unless node.is_a?(Prism::ClassNode) && TARGETS.include?(name) && node.body
+    # [statements, base, hook?] for each body a base runs as self: `class ActiveRecord::Base` reopened
+    # however the namespace is written, `ActiveRecord::Base.class_eval do`, and an `on_load` block.
+    def base_bodies(tree)
+      reopened = Introspectors::DeclaredConstant.constants(tree).filter_map do |name, node|
+        [ node.body, name, false ] if node.is_a?(Prism::ClassNode) && TARGETS.include?(name)
+      end
+      evaluated = Introspectors::AstWalk.each(tree).filter_map do |node|
+        next unless node.is_a?(Prism::CallNode) && node.block
+        next [ node.block.body, TARGETS.first, true ] if hook?(node)
 
-        node.body.body.flat_map { |call| call.is_a?(Prism::CallNode) && call.receiver.nil? ? mixin_calls(call) : [] }
+        [ node.block.body, node.receiver.slice.delete_prefix("::"), false ] if %i[class_eval class_exec].include?(node.name) && target?(node.receiver)
       end
+      (reopened + evaluated).filter_map { |body, *rest| [ body.body, *rest ] if body.is_a?(Prism::StatementsNode) }
     end
-    private_class_method :reopened_mixins
+    private_class_method :base_bodies
 
     # Inside `on_load(:active_record) { ... }` the block runs in the base.
     def hook_mixins(node)
@@ -109,11 +138,12 @@ module RailsAiContext
     private_class_method :constants
 
     # Past Zeitwerk's path, a plugin's module sits in a lib file its init.rb
-    # requires, under a name its path does not spell.
+    # requires, under a name its path does not spell; a namespace stub is not it.
     def declaring_file(root, name, file)
-      ConcernPaths.find_file(root, name) ||
+      ConcernPaths.find_file(root, name, outer: false) ||
         ([ file ] + required_files(root, file, 2)).find do |candidate|
-          Introspectors::DeclaredConstant.module_node(AstCache.parse(candidate).value, name)
+          node = Introspectors::DeclaredConstant.module_node(AstCache.parse(candidate).value, name)
+          node && !Introspectors::DeclaredConstant.stub?(node)
         end
     rescue StandardError => e
       RailsAiContext.debug_fail(e, nil, label: "BaseMixins.declaring_file #{name}")

data/lib/rails_ai_context/boot_manager.rb

@@ -113,7 +113,11 @@ module RailsAiContext
         )
       end
 
-      guard(timeout: timeout, announce_exit: false) { require environment_rb }
+      result = guard(timeout: timeout, announce_exit: false) { require environment_rb }
+      return result unless result.booted?
+      return result if defined?(::Rails) && ::Rails.respond_to?(:application) && ::Rails.application
+
+      Result.new(status: :failed, error: BootError.new("config/environment.rb loaded but defined no Rails application"))
     rescue SystemExit => e
       Result.new(status: :failed, error: BootExitError.new("App called exit(#{e.status}) during boot"))
     end

data/lib/rails_ai_context/cli/entry_boot.rb

@@ -40,6 +40,7 @@ module RailsAiContext
       # source: an engine keeps its dummy app under spec/dummy, so its root
       # has app/ and no config/, and it is a real target.
       def self.app_present?(root, allow_source_only: false)
+        return false if other_framework?(root)
         return true if File.exist?(File.join(root, "config", "environment.rb"))
         return false unless allow_source_only
 
@@ -47,6 +48,19 @@ module RailsAiContext
           Dir.glob(File.join(root, "app", "**", "*.rb")).any?
       end
 
+      # Sinatra MVC trees have config/environment.rb too; without application.rb the lockfile, else the Gemfile, decides.
+      def self.other_framework?(root)
+        return false if File.exist?(File.join(root, "config", "application.rb"))
+
+        require_relative "../gem_lock"
+        lock = GemLock.for(root)
+        return !lock.any?("rails", "railties") unless lock.missing?
+
+        names = lock.gemfile_gems
+        !names.nil? && (names & %w[rails railties]).empty?
+      end
+      private_class_method :other_framework?
+
       def self.call(root:, allow_static:, no_boot: false, allow_source_only: false, command: nil)
         messages = []
 
@@ -63,7 +77,7 @@ module RailsAiContext
         unless app_present?(root)
           return absent(root, messages, command: command) unless allow_static
 
-          return enter_static("no config/environment.rb in #{root}", :source_only, root, messages)
+          return enter_static("no config/environment.rb in the app root", :source_only, root, messages)
         end
 
         # Bundler.setup (in config/boot.rb) strips $LOAD_PATH and the spec
@@ -166,7 +180,8 @@ module RailsAiContext
       def self.enter_static(reason, kind, root, messages)
         require_gem_without_app!
         RailsAiContext.tier = :static
-        RailsAiContext.static_reason = reason
+        # Every tool response carries this in its footer, so an absolute path in a boot error goes root-relative.
+        RailsAiContext.static_reason = RailsAiContext::PortablePath.relativize_text(reason, root)
         RailsAiContext.static_kind = kind
         RailsAiContext.configuration.app_root = root
         Configuration.auto_load!(root)

data/lib/rails_ai_context/cli/tool_runner.rb

@@ -191,7 +191,7 @@ module RailsAiContext
       def build_kwargs
         kwargs = case raw_args
         when Hash
-                   raw_args.transform_keys(&:to_sym).except(:server_context)
+                   raw_args.transform_keys(&:to_sym).except(:server_context).transform_values { |value| utf8(value) }
         when Array
                    return with_environment(parse_cli_args(raw_args).except(:server_context))
         else
@@ -219,6 +219,7 @@ module RailsAiContext
 
       # Parse ["--table", "users", "--detail", "full", "--app-only"] into { table: "users", ... }
       def parse_cli_args(args)
+        args = args.map { |arg| utf8(arg) }
         result = {}
         i = 0
         properties = (tool_schema[:properties] || {})
@@ -308,6 +309,19 @@ module RailsAiContext
         result
       end
 
+      # ARGV (here and in the rake task) carries the locale's encoding, BINARY
+      # under LANG=C; MCP values are always UTF-8.
+      def utf8(arg)
+        return arg unless arg.is_a?(String)
+        return arg if arg.encoding == Encoding::UTF_8 && arg.valid_encoding?
+
+        if [ Encoding::UTF_8, Encoding::BINARY, Encoding::US_ASCII ].include?(arg.encoding)
+          arg.dup.force_encoding(Encoding::UTF_8).scrub
+        else
+          arg.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+        end
+      end
+
       # The flag a stray word was most likely meant for: a required param
       # first, since that is the one a caller must supply.
       def suggested_flag