rails-ai-context 5.32.2 → 5.34.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1491 -0
  3. data/CONTEXT.md +27 -3
  4. data/README.md +16 -12
  5. data/Rakefile +6 -0
  6. data/app/controllers/rails_ai_context/mcp_controller.rb +112 -84
  7. data/config/routes.rb +2 -3
  8. data/docs/ARCHITECTURE.md +7 -5
  9. data/docs/CLI.md +79 -6
  10. data/docs/COMPATIBILITY.md +75 -1
  11. data/docs/CONFIGURATION.md +30 -11
  12. data/docs/CUSTOM_TOOLS.md +2 -1
  13. data/docs/FAQ.md +16 -6
  14. data/docs/GUIDE.md +117 -71
  15. data/docs/INTROSPECTORS.md +10 -8
  16. data/docs/QUICKSTART.md +3 -3
  17. data/docs/RECIPES.md +1 -1
  18. data/docs/SECURITY.md +109 -25
  19. data/docs/SETUP.md +30 -6
  20. data/docs/STANDALONE.md +10 -3
  21. data/docs/TOOLS.md +152 -34
  22. data/docs/TROUBLESHOOTING.md +90 -12
  23. data/docs/adr/0003-shared-tool-cache-semantics.md +8 -0
  24. data/docs/adr/0005-workspace-mcp-entries.md +39 -0
  25. data/exe/rails-ai-context +733 -104
  26. data/lib/generators/rails_ai_context/install/install_generator.rb +341 -125
  27. data/lib/rails_ai_context/app_kind.rb +18 -5
  28. data/lib/rails_ai_context/ast_cache.rb +7 -0
  29. data/lib/rails_ai_context/base_mixins.rb +8 -8
  30. data/lib/rails_ai_context/boot_manager.rb +22 -3
  31. data/lib/rails_ai_context/change_watch.rb +14 -11
  32. data/lib/rails_ai_context/cli/app_root.rb +401 -0
  33. data/lib/rails_ai_context/cli/entry_boot.rb +58 -22
  34. data/lib/rails_ai_context/cli/tool_runner.rb +47 -21
  35. data/lib/rails_ai_context/code_reloader.rb +115 -1
  36. data/lib/rails_ai_context/concern_membership.rb +3 -1
  37. data/lib/rails_ai_context/concern_paths.rb +8 -3
  38. data/lib/rails_ai_context/configuration.rb +41 -9
  39. data/lib/rails_ai_context/context_file_report.rb +19 -2
  40. data/lib/rails_ai_context/database_file.rb +26 -0
  41. data/lib/rails_ai_context/doctor.rb +1122 -184
  42. data/lib/rails_ai_context/engine.rb +18 -1
  43. data/lib/rails_ai_context/erb_source.rb +19 -0
  44. data/lib/rails_ai_context/executable.rb +21 -0
  45. data/lib/rails_ai_context/facts_formatter.rb +5 -2
  46. data/lib/rails_ai_context/file_walk.rb +61 -10
  47. data/lib/rails_ai_context/fingerprinter/snapshot.rb +234 -0
  48. data/lib/rails_ai_context/fingerprinter.rb +40 -12
  49. data/lib/rails_ai_context/gem_lock.rb +186 -27
  50. data/lib/rails_ai_context/git_ignore.rb +22 -0
  51. data/lib/rails_ai_context/install/ai_tool.rb +31 -5
  52. data/lib/rails_ai_context/install/cleanup.rb +21 -0
  53. data/lib/rails_ai_context/install/program.rb +209 -35
  54. data/lib/rails_ai_context/install/selection_record.rb +258 -97
  55. data/lib/rails_ai_context/install/surface.rb +6 -1
  56. data/lib/rails_ai_context/install/validation_hook.rb +282 -0
  57. data/lib/rails_ai_context/install/workspace.rb +166 -0
  58. data/lib/rails_ai_context/install_mode.rb +71 -13
  59. data/lib/rails_ai_context/introspectors/action_mailbox_introspector.rb +1 -1
  60. data/lib/rails_ai_context/introspectors/action_presence.rb +11 -6
  61. data/lib/rails_ai_context/introspectors/action_source.rb +182 -0
  62. data/lib/rails_ai_context/introspectors/action_text_introspector.rb +1 -2
  63. data/lib/rails_ai_context/introspectors/active_record_settings.rb +1 -1
  64. data/lib/rails_ai_context/introspectors/active_storage_introspector.rb +1 -3
  65. data/lib/rails_ai_context/introspectors/active_support_introspector.rb +3 -3
  66. data/lib/rails_ai_context/introspectors/admin_resources.rb +1 -1
  67. data/lib/rails_ai_context/introspectors/anyway_configs.rb +1 -1
  68. data/lib/rails_ai_context/introspectors/api_introspector.rb +3 -3
  69. data/lib/rails_ai_context/introspectors/auth_introspector.rb +15 -10
  70. data/lib/rails_ai_context/introspectors/call_site_expansion.rb +24 -1
  71. data/lib/rails_ai_context/introspectors/component_introspector.rb +3 -2
  72. data/lib/rails_ai_context/introspectors/controller_introspector.rb +387 -60
  73. data/lib/rails_ai_context/introspectors/convention_introspector.rb +1 -1
  74. data/lib/rails_ai_context/introspectors/database_stats_introspector.rb +15 -3
  75. data/lib/rails_ai_context/introspectors/devops_introspector.rb +3 -1
  76. data/lib/rails_ai_context/introspectors/eager_load.rb +201 -9
  77. data/lib/rails_ai_context/introspectors/engine_introspector.rb +1 -1
  78. data/lib/rails_ai_context/introspectors/env_config_introspector.rb +1 -1
  79. data/lib/rails_ai_context/introspectors/env_references.rb +10 -1
  80. data/lib/rails_ai_context/introspectors/frontend_framework_introspector.rb +1 -1
  81. data/lib/rails_ai_context/introspectors/gem_introspector.rb +23 -10
  82. data/lib/rails_ai_context/introspectors/grape_endpoints.rb +1 -1
  83. data/lib/rails_ai_context/introspectors/habtm_join_tables.rb +1 -1
  84. data/lib/rails_ai_context/introspectors/helper_names.rb +1 -1
  85. data/lib/rails_ai_context/introspectors/i18n_introspector.rb +3 -3
  86. data/lib/rails_ai_context/introspectors/job_introspector.rb +8 -6
  87. data/lib/rails_ai_context/introspectors/listeners/base_listener.rb +12 -8
  88. data/lib/rails_ai_context/introspectors/listeners/callbacks_listener.rb +78 -4
  89. data/lib/rails_ai_context/introspectors/listeners/config_assignment_listener.rb +1 -0
  90. data/lib/rails_ai_context/introspectors/listeners/env_access_listener.rb +10 -1
  91. data/lib/rails_ai_context/introspectors/listeners/foreign_evals_listener.rb +61 -0
  92. data/lib/rails_ai_context/introspectors/listeners/macros_listener.rb +13 -4
  93. data/lib/rails_ai_context/introspectors/listeners/migration_dsl_listener.rb +2 -1
  94. data/lib/rails_ai_context/introspectors/listeners/migration_replay_listener.rb +3 -2
  95. data/lib/rails_ai_context/introspectors/listeners/routes_dsl_listener.rb +29 -8
  96. data/lib/rails_ai_context/introspectors/listeners/schema_dsl_listener.rb +2 -2
  97. data/lib/rails_ai_context/introspectors/listeners/scopes_listener.rb +88 -12
  98. data/lib/rails_ai_context/introspectors/middleware_introspector.rb +2 -2
  99. data/lib/rails_ai_context/introspectors/migration_introspector.rb +27 -0
  100. data/lib/rails_ai_context/introspectors/migration_replay/helpers.rb +1 -1
  101. data/lib/rails_ai_context/introspectors/migration_replay/statements.rb +83 -27
  102. data/lib/rails_ai_context/introspectors/migration_replay.rb +15 -14
  103. data/lib/rails_ai_context/introspectors/model_introspector.rb +231 -23
  104. data/lib/rails_ai_context/introspectors/n_plus_one_scan.rb +620 -0
  105. data/lib/rails_ai_context/introspectors/performance_introspector.rb +11 -174
  106. data/lib/rails_ai_context/introspectors/route_introspector.rb +10 -1
  107. data/lib/rails_ai_context/introspectors/schema_conventions.rb +23 -3
  108. data/lib/rails_ai_context/introspectors/schema_introspector.rb +66 -16
  109. data/lib/rails_ai_context/introspectors/schema_reader.rb +4 -3
  110. data/lib/rails_ai_context/introspectors/seeds_introspector.rb +1 -1
  111. data/lib/rails_ai_context/introspectors/source_scan.rb +39 -9
  112. data/lib/rails_ai_context/introspectors/stimulus_introspector.rb +15 -6
  113. data/lib/rails_ai_context/introspectors/test_introspector.rb +1 -1
  114. data/lib/rails_ai_context/introspectors/turbo_introspector.rb +131 -12
  115. data/lib/rails_ai_context/introspectors/view_introspector.rb +4 -4
  116. data/lib/rails_ai_context/introspectors/view_template_introspector.rb +15 -1
  117. data/lib/rails_ai_context/legacy_cleanup.rb +17 -4
  118. data/lib/rails_ai_context/live_reload.rb +20 -20
  119. data/lib/rails_ai_context/mcp_config_generator.rb +916 -109
  120. data/lib/rails_ai_context/mcp_edge.rb +182 -32
  121. data/lib/rails_ai_context/middleware.rb +5 -1
  122. data/lib/rails_ai_context/migration_status.rb +88 -25
  123. data/lib/rails_ai_context/output_guard.rb +48 -2
  124. data/lib/rails_ai_context/package_json.rb +1 -1
  125. data/lib/rails_ai_context/path_resolver.rb +46 -13
  126. data/lib/rails_ai_context/payload.rb +92 -8
  127. data/lib/rails_ai_context/pending_migrations.rb +5 -1
  128. data/lib/rails_ai_context/portable_path.rb +3 -2
  129. data/lib/rails_ai_context/presets.rb +15 -11
  130. data/lib/rails_ai_context/redaction.rb +262 -51
  131. data/lib/rails_ai_context/resources.rb +55 -20
  132. data/lib/rails_ai_context/route_coverage.rb +8 -0
  133. data/lib/rails_ai_context/safe_file.rb +14 -1
  134. data/lib/rails_ai_context/safe_path.rb +101 -4
  135. data/lib/rails_ai_context/schema_adapter.rb +8 -0
  136. data/lib/rails_ai_context/schema_version.rb +9 -0
  137. data/lib/rails_ai_context/serializers/claude_rules_serializer.rb +88 -33
  138. data/lib/rails_ai_context/serializers/compact_serializer_helper.rb +36 -34
  139. data/lib/rails_ai_context/serializers/context_file_serializer.rb +21 -29
  140. data/lib/rails_ai_context/serializers/copilot_instructions_serializer.rb +8 -6
  141. data/lib/rails_ai_context/serializers/copilot_serializer.rb +2 -7
  142. data/lib/rails_ai_context/serializers/cursor_rules_serializer.rb +19 -16
  143. data/lib/rails_ai_context/serializers/guide_examples.rb +142 -0
  144. data/lib/rails_ai_context/serializers/markdown_serializer.rb +146 -30
  145. data/lib/rails_ai_context/serializers/opencode_rules_serializer.rb +38 -15
  146. data/lib/rails_ai_context/serializers/section_facts.rb +44 -1
  147. data/lib/rails_ai_context/serializers/section_marker_writer.rb +94 -12
  148. data/lib/rails_ai_context/serializers/stack_overview_helper.rb +24 -12
  149. data/lib/rails_ai_context/serializers/tool_guide_helper.rb +244 -129
  150. data/lib/rails_ai_context/server.rb +304 -71
  151. data/lib/rails_ai_context/tasks/rails_ai_context.rake +447 -372
  152. data/lib/rails_ai_context/tools/analyze_feature.rb +72 -18
  153. data/lib/rails_ai_context/tools/argument_schema.rb +49 -0
  154. data/lib/rails_ai_context/tools/base_tool.rb +367 -46
  155. data/lib/rails_ai_context/tools/brakeman_guard.rb +24 -0
  156. data/lib/rails_ai_context/tools/dependency_graph.rb +63 -28
  157. data/lib/rails_ai_context/tools/diagnose.rb +218 -11
  158. data/lib/rails_ai_context/tools/generate_test.rb +339 -39
  159. data/lib/rails_ai_context/tools/get_callbacks.rb +24 -13
  160. data/lib/rails_ai_context/tools/get_component_catalog.rb +1 -1
  161. data/lib/rails_ai_context/tools/get_concern.rb +91 -27
  162. data/lib/rails_ai_context/tools/get_config.rb +13 -4
  163. data/lib/rails_ai_context/tools/get_context.rb +71 -14
  164. data/lib/rails_ai_context/tools/get_controllers.rb +179 -44
  165. data/lib/rails_ai_context/tools/get_conventions.rb +28 -6
  166. data/lib/rails_ai_context/tools/get_edit_context.rb +27 -14
  167. data/lib/rails_ai_context/tools/get_env.rb +18 -9
  168. data/lib/rails_ai_context/tools/get_frontend_stack.rb +3 -1
  169. data/lib/rails_ai_context/tools/get_gems.rb +1 -1
  170. data/lib/rails_ai_context/tools/get_helper_methods.rb +10 -6
  171. data/lib/rails_ai_context/tools/get_i18n.rb +1 -1
  172. data/lib/rails_ai_context/tools/get_job_pattern.rb +3 -2
  173. data/lib/rails_ai_context/tools/get_mailers.rb +3 -3
  174. data/lib/rails_ai_context/tools/get_model_details.rb +30 -10
  175. data/lib/rails_ai_context/tools/get_partial_interface.rb +242 -23
  176. data/lib/rails_ai_context/tools/get_routes.rb +29 -8
  177. data/lib/rails_ai_context/tools/get_schema.rb +52 -19
  178. data/lib/rails_ai_context/tools/get_service_pattern.rb +24 -10
  179. data/lib/rails_ai_context/tools/get_stimulus.rb +37 -1
  180. data/lib/rails_ai_context/tools/get_test_info.rb +18 -4
  181. data/lib/rails_ai_context/tools/get_turbo_map.rb +221 -58
  182. data/lib/rails_ai_context/tools/get_view.rb +14 -14
  183. data/lib/rails_ai_context/tools/migration_advisor.rb +355 -48
  184. data/lib/rails_ai_context/tools/onboard.rb +52 -8
  185. data/lib/rails_ai_context/tools/performance_check.rb +5 -3
  186. data/lib/rails_ai_context/tools/query.rb +914 -86
  187. data/lib/rails_ai_context/tools/read_logs.rb +69 -15
  188. data/lib/rails_ai_context/tools/review_changes.rb +215 -53
  189. data/lib/rails_ai_context/tools/runtime_info.rb +51 -1
  190. data/lib/rails_ai_context/tools/safe_call.rb +35 -14
  191. data/lib/rails_ai_context/tools/search_code.rb +243 -67
  192. data/lib/rails_ai_context/tools/search_docs.rb +4 -4
  193. data/lib/rails_ai_context/tools/security_scan.rb +143 -33
  194. data/lib/rails_ai_context/tools/session_context.rb +39 -5
  195. data/lib/rails_ai_context/tools/validate.rb +259 -40
  196. data/lib/rails_ai_context/tools/validate_semantics.rb +130 -56
  197. data/lib/rails_ai_context/version.rb +1 -1
  198. data/lib/rails_ai_context/vfs.rb +67 -40
  199. data/lib/rails_ai_context/view_file.rb +41 -3
  200. data/lib/rails_ai_context/watcher.rb +51 -13
  201. data/lib/rails_ai_context.rb +21 -0
  202. data/server.json +1 -1
  203. metadata +14 -1
@@ -284,7 +284,7 @@ module RailsAiContext
284
284
  # only a .keep file), and some apps keep only validator classes there
285
285
  # (one app keeps 37), so the claim needs one file that declares no class: a module.
286
286
  def concern_files_exist?(relative_path)
287
- Dir.glob(File.join(root, relative_path, "**", "*.rb")).any? do |path|
287
+ ConcernPaths.files(File.join(root, relative_path), root).any? do |path|
288
288
  source = SafeFile.read(path)
289
289
  source && !DeclaredConstant.declares_class?(source, path_name: File.basename(path, ".rb").camelize)
290
290
  end
@@ -19,6 +19,10 @@ module RailsAiContext
19
19
  def call
20
20
  return { skipped: true, reason: "ActiveRecord not available" } unless defined?(ActiveRecord::Base)
21
21
 
22
+ # Connecting would create a SQLite file that is not there yet.
23
+ missing = DatabaseFile.missing(ActiveRecord::Base.connection_db_config)
24
+ raise missing if missing
25
+
22
26
  adapter = ActiveRecord::Base.connection.adapter_name.downcase
23
27
  case adapter
24
28
  when POSTGRES_ADAPTER
@@ -62,15 +66,23 @@ module RailsAiContext
62
66
  { adapter: "mysql", tables: tables, total_tables: tables.size }
63
67
  end
64
68
 
69
+ # SQLite keeps no row estimate a query can read without ANALYZE, and
70
+ # COUNT(*) walks the whole table on every introspection. A count that
71
+ # stops at the cap reads at most that many rows; a table past it is
72
+ # reported as at least the cap.
73
+ SQLITE_COUNT_CAP = 100_000
74
+
65
75
  def collect_sqlite_stats
66
76
  conn = ActiveRecord::Base.connection
67
77
  # Use conn.tables as authoritative list - never interpolate user input
68
78
  table_names = (conn.tables - SqliteVirtualTables.hidden(conn)).reject { |t| t.start_with?("ar_internal_metadata", "schema_migrations") }
69
79
 
70
80
  tables = table_names.map do |table|
71
- count = conn.select_value("SELECT COUNT(*) FROM #{conn.quote_table_name(table)}").to_i
72
- { table: table, approximate_rows: count }
73
- end.sort_by { |t| -t[:approximate_rows] }
81
+ count = conn.select_value("SELECT COUNT(*) FROM (SELECT 1 FROM #{conn.quote_table_name(table)} LIMIT #{SQLITE_COUNT_CAP + 1})").to_i
82
+ entry = { table: table, approximate_rows: [ count, SQLITE_COUNT_CAP ].min }
83
+ entry[:at_least] = true if count > SQLITE_COUNT_CAP
84
+ entry
85
+ end.sort_by { |t| [ -t[:approximate_rows], t[:table] ] }
74
86
 
75
87
  { adapter: "sqlite", tables: tables, total_tables: tables.size }
76
88
  end
@@ -72,11 +72,13 @@ module RailsAiContext
72
72
  path = File.join(root, filename)
73
73
  next unless File.exist?(path)
74
74
 
75
+ # A command leaves the app as source does, through Redaction: `rails ai:context` writes
76
+ # it into .ai-context.json, which is committed (`--requirepass ...`, `DB_PASSWORD=...`).
75
77
  entries = (RailsAiContext::SafeFile.read(path) || "").lines.filter_map do |line|
76
78
  line.strip!
77
79
  next if line.empty? || line.start_with?("#")
78
80
  parts = line.split(":", 2)
79
- { name: parts[0].strip, command: parts[1]&.strip } if parts.size == 2
81
+ { name: parts[0].strip, command: RailsAiContext::Redaction.redact_source_line(parts[1].strip, path: filename) } if parts.size == 2
80
82
  end
81
83
 
82
84
  { file: filename, entries: entries } if entries.any?
@@ -6,36 +6,120 @@ module RailsAiContext
6
6
  # did not eager load. eager_load_dir stops at the first unloadable file
7
7
  # and raises for a directory another loader owns, so both fall back to
8
8
  # loading one constant at a time; a file that cannot load costs itself.
9
+ #
10
+ # A file Ruby cannot compile is never required, and neither is one whose
11
+ # class body names its constant, since loading that one autoloads it:
12
+ # Prism reads each file the loader has not loaded yet first, about a
13
+ # tenth of a millisecond a file, and a file is read again only once its
14
+ # stat changes. Requiring one raised a SyntaxError out of Ruby's
15
+ # compiler, and with web-console in the bundle (bindex's raise hook)
16
+ # that crashed Ruby in about half the runs.
9
17
  module EagerLoad
18
+ # File => [stat, answer]: whether it compiles, and the constants its
19
+ # class body names. After a reload only the files edited since are
20
+ # read again.
21
+ SYNTAX = Concurrent::Map.new
22
+ CONSTANTS = Concurrent::Map.new
23
+
24
+ # File => [mtime, reason] for each file a note was written for, so a
25
+ # file left broken is noted once, not at every rebuild of the caches.
26
+ NOTED = Concurrent::Map.new
27
+
10
28
  module_function
11
29
 
30
+ # Returns the files left out, by real path, each with why.
12
31
  def dir(root, kind:)
13
- return if Rails.application.config.eager_load
32
+ return {} if Rails.application.config.eager_load
14
33
 
15
34
  dirs = PathResolver.dirs_for(root, kind) + PathResolver.enclosing_engine_roots(root).map { |engine| File.join(engine, kind) }
16
- dirs.select { |path| Dir.exist?(path) }.uniq.each { |path| load_dir(path) }
35
+ left_out = dirs.select { |path| Dir.exist?(path) }.uniq.each_with_object({}) { |path, found| found.merge!(load_dir(path)) }
36
+ RunCache.fetch([ :eager_load_skipped ]) { {} }.merge!(left_out)
37
+ left_out
17
38
  rescue StandardError, ScriptError => e
18
- RailsAiContext.debug_fail(e, nil, label: "eager load of #{kind}")
39
+ RailsAiContext.debug_fail(e, {}, label: "eager load of #{kind}")
40
+ end
41
+
42
+ # Why this run left a file unloaded, by its real path, or nil: it does
43
+ # not compile, or its class body names a constant from one that does
44
+ # not. Outside a run, or for a file no walk here reached, only the
45
+ # file's own syntax is asked, and only if it is not loaded already.
46
+ def skipped(file)
47
+ RunCache.fetch([ :eager_load_skipped ]) { {} }[file] ||
48
+ (syntax_error(file) unless $LOADED_FEATURES.include?(file))
49
+ end
50
+
51
+ # The constant a name resolves to, loading what it must as constantize
52
+ # does, or nil for a name nothing defines. A tool that looks an app class
53
+ # up by name keeps the rule the walks keep: a file this module leaves
54
+ # out is never required, and the answer is then the block's, given why,
55
+ # or nil.
56
+ def constant(name)
57
+ name.to_s.delete_prefix("::").split("::").reduce(Object) do |scope, segment|
58
+ return nil unless scope.is_a?(Module) && scope.const_defined?(segment, false)
59
+
60
+ file = scope.autoload?(segment, false)
61
+ reason = file && left_out(file)
62
+ return (yield reason if block_given?) if reason
63
+
64
+ scope.const_get(segment, false)
65
+ end
66
+ rescue NameError
67
+ nil
68
+ end
69
+
70
+ # Why requiring a file now would raise out of Ruby's compiler, or nil:
71
+ # what a walk this run found, the file's own syntax error, else what a
72
+ # walk of the autoload root holding it finds - a file its class body
73
+ # needs that does not compile, however far down.
74
+ def left_out(file)
75
+ found = skipped(file)
76
+ return found if found || $LOADED_FEATURES.include?(file)
77
+
78
+ loader = Rails.autoloaders.main
79
+ root = loader.dirs.select { |dir| file.start_with?("#{dir}/") }.max_by(&:length) if loader.respond_to?(:dirs)
80
+ root && unloadable(root, Dir.glob(File.join(root, "**/*.rb")).sort)[file]
81
+ end
82
+
83
+ # Prism's first error in a file, worded as Ruby words a SyntaxError
84
+ # ("app/models/post.rb:4: syntax error, unexpected 'end'; ..."), or nil
85
+ # when the running Ruby's grammar takes it - or when Prism cannot read
86
+ # it, and the require decides as it always did.
87
+ def syntax_error(file)
88
+ per_file(SYNTAX, file) do
89
+ next nil if AstCache.parse_success?(file, ruby: RUBY_VERSION)
90
+
91
+ error = AstCache.parse(file, ruby: RUBY_VERSION).errors.first
92
+ error && "#{relative(file)}:#{error.location.start_line}: syntax error, #{error.message}"
93
+ end
94
+ rescue StandardError
95
+ nil
19
96
  end
20
97
 
21
98
  # eager_load_dir returns silently for a directory the loader manages
22
99
  # but does not eager load, so the per-constant walk always follows: a
23
100
  # constant already loaded is a hash lookup, and one that is not gets
24
- # loaded here. A file that cannot load costs itself.
101
+ # loaded here. eager_load_dir would require the files left out, so it
102
+ # runs only when there are none.
25
103
  def load_dir(path)
104
+ files = Dir.glob(File.join(path, "**/*.rb")).sort
105
+ left_out = unloadable(path, files)
106
+ load_whole(path) if left_out.empty?
107
+ load_individually(path, files - left_out.keys)
108
+ left_out.transform_keys { |file| real(file) }
109
+ end
110
+
111
+ def load_whole(path)
26
112
  loader = Rails.autoloaders.main
27
113
  loader.eager_load_dir(path) if loader.respond_to?(:eager_load_dir)
28
114
  rescue StandardError, ScriptError => e
29
115
  RailsAiContext.debug_fail(e, label: "eager_load_dir #{path}")
30
- ensure
31
- load_individually(path)
32
116
  end
33
117
 
34
118
  # The loader names the constant, not `camelize`: an app inflection
35
119
  # makes the two disagree, and a wrong name silently loads nothing.
36
- def load_individually(path)
120
+ def load_individually(path, files)
37
121
  loader = Rails.autoloaders.main
38
- Dir.glob(File.join(path, "**/*.rb")).sort.each do |file|
122
+ files.each do |file|
39
123
  cpath_for(loader, path, file).constantize
40
124
  rescue StandardError, ScriptError
41
125
  next
@@ -48,6 +132,8 @@ module RailsAiContext
48
132
  def files(records)
49
133
  loader = Rails.autoloaders.main
50
134
  records.each do |record|
135
+ next if skipped(record.path)
136
+
51
137
  names = [ expected_cpath(loader, record.path) ].compact
52
138
  names = DeclaredConstant.declarations(record.source).map(&:name) if names.empty?
53
139
  names = [ record.path_name ] if names.empty?
@@ -57,6 +143,110 @@ module RailsAiContext
57
143
  end
58
144
  end
59
145
 
146
+ # The files under `path` the loader has not loaded and must not load
147
+ # now, each with why: one Prism finds an error in, then each whose
148
+ # class body names a constant from one of those - an included concern,
149
+ # a superclass - and so on down. A file already loaded is a constant
150
+ # lookup, which compiles nothing, and is not read.
151
+ def unloadable(path, files)
152
+ loaded = $LOADED_FEATURES.to_set
153
+ pending = files.reject { |file| loaded.include?(file) }
154
+ found = pending.each_with_object({}) do |file, broken|
155
+ error = syntax_error(file)
156
+ broken[file] = error if error
157
+ end
158
+ return found if found.empty?
159
+
160
+ add_dependents(path, pending, found)
161
+ found.each { |file, reason| note(file, reason) }
162
+ found
163
+ end
164
+
165
+ # Each reason names the file that does not compile, however far down.
166
+ def add_dependents(path, pending, found)
167
+ loader = Rails.autoloaders.main
168
+ blocked = found.keys.to_h { |file| [ cpath_for(loader, path, file), file ] }
169
+ cause = found.keys.to_h { |file| [ file, file ] }
170
+ names = (pending - found.keys).to_h { |file| [ file, load_time_constants(file) ] }
171
+ loop do
172
+ added = names.filter_map do |file, refs|
173
+ next if found.key?(file)
174
+
175
+ cpath = refs.lazy.filter_map { |ref| blocker(blocked, ref) }.first
176
+ next unless cpath
177
+
178
+ by = blocked[cpath]
179
+ cause[file] = cause[by]
180
+ how = by == cause[by] ? "which does not compile" : "which needs a file that does not compile"
181
+ found[file] = "#{relative(file)} needs #{cpath}, #{how}: #{found[cause[by]]}"
182
+ blocked[cpath_for(loader, path, file)] = file
183
+ end
184
+ break if added.empty?
185
+ end
186
+ end
187
+
188
+ # The constants loading a file evaluates: what its code names outside
189
+ # method bodies and lambdas - a superclass, an included concern, the
190
+ # namespace it opens - each with the namespaces it is written under.
191
+ # Read once per file while one is broken, so the parse is not cached.
192
+ def load_time_constants(file)
193
+ per_file(CONSTANTS, file) do
194
+ names = []
195
+ visit = lambda do |node|
196
+ case node
197
+ when Prism::DefNode, Prism::LambdaNode then nil
198
+ when Prism::ConstantReadNode then names << node.name.to_s
199
+ when Prism::ConstantPathNode
200
+ parts = (node.full_name rescue node.slice).delete_prefix("::").split("::")
201
+ parts.each_index { |i| names << parts[0..i].join("::") }
202
+ else node.compact_child_nodes.each(&visit)
203
+ end
204
+ end
205
+ visit.call(AstCache.parse_uncached(File.read(file)).value)
206
+ names.uniq
207
+ end
208
+ rescue StandardError
209
+ []
210
+ end
211
+
212
+ # The block's answer for a file, kept while the file's stat holds. A
213
+ # file younger than AstCache::RACY_WINDOW is read every time: its stat
214
+ # may not show a rewrite yet.
215
+ def per_file(store, file)
216
+ stat = File.stat(file)
217
+ stamp = [ stat.mtime.to_i, stat.mtime.nsec, stat.size, stat.ino ] if stat.mtime < Time.now - AstCache::RACY_WINDOW
218
+ kept = stamp && store[file]
219
+ return kept[1] if kept && kept[0] == stamp
220
+
221
+ answer = yield
222
+ store[file] = [ stamp, answer ] if stamp
223
+ answer
224
+ end
225
+
226
+ # A name written inside a namespace can mean that namespace's constant,
227
+ # so a broken Admin::BaseController blocks a `< BaseController`.
228
+ def blocker(blocked, ref)
229
+ blocked.keys.find { |cpath| cpath == ref || cpath.end_with?("::#{ref}") }
230
+ end
231
+
232
+ def note(file, reason)
233
+ stamp = [ (File.mtime(file) rescue nil), reason ]
234
+ return if NOTED[file] == stamp
235
+
236
+ NOTED[file] = stamp
237
+ RailsAiContext.log_warn("[rails-ai-context] Left unloaded: #{reason}")
238
+ end
239
+
240
+ def relative(file)
241
+ PortablePath.relativize_text(file, Rails.root)
242
+ end
243
+
244
+ def real(file)
245
+ File.realpath(file)
246
+ rescue SystemCallError
247
+ file
248
+ end
249
+
60
250
  # The loader declines a file it does not manage (a pack or an in-repo
61
251
  # engine runs its own), by nil or by raising, and Zeitwerk before 2.6.9
62
252
  # cannot answer; its inflector, segment by segment as Zeitwerk itself
@@ -75,7 +265,9 @@ module RailsAiContext
75
265
  rescue Zeitwerk::Error
76
266
  nil
77
267
  end
78
- private_class_method :load_dir, :load_individually, :cpath_for, :inflect, :expected_cpath
268
+ private_class_method :load_dir, :load_whole, :load_individually, :left_out, :unloadable, :add_dependents,
269
+ :load_time_constants, :per_file, :blocker, :note, :relative, :real,
270
+ :cpath_for, :inflect, :expected_cpath
79
271
  end
80
272
  end
81
273
  end
@@ -97,7 +97,7 @@ module RailsAiContext
97
97
  return "." if path == app.root.to_s
98
98
 
99
99
  relative = PortablePath.relativize("#{path}/", app.root).chomp("/")
100
- relative unless relative.start_with?(File::SEPARATOR)
100
+ relative unless SafePath.absolute?(relative)
101
101
  end
102
102
 
103
103
  def discover_rails_engines
@@ -27,7 +27,7 @@ module RailsAiContext
27
27
 
28
28
  # @return [Hash] per-environment config summary
29
29
  def call
30
- files = Dir.glob(File.join(root, "config", "environments", "*.rb")).sort.filter_map do |path|
30
+ files = Dir.glob(File.join(root, "config", "environments", "*.rb")).sort.reject { |path| PathResolver.linked_out?(path, root) }.filter_map do |path|
31
31
  summarize(path)
32
32
  end
33
33
 
@@ -36,7 +36,9 @@ module RailsAiContext
36
36
  next if !ruby?(path) && !source.include?("<%")
37
37
 
38
38
  refs = references(ruby_source(path, source))
39
- refs = refs.map { |ref| ref.except(:default).merge(default_unread: true) } if names_only
39
+ # Whether a site has a default is its shape, not its value: a site
40
+ # with none still raises or reads nil, and says so.
41
+ refs = refs.map { |ref| ref.key?(:default) ? ref.except(:default).merge(default_unread: true) : ref } if names_only
40
42
  found[path] = refs if refs.any?
41
43
  end
42
44
  rescue SystemCallError => e
@@ -77,10 +79,17 @@ module RailsAiContext
77
79
  script?(path) || path.end_with?(*RUBY_EXTENSIONS)
78
80
  end
79
81
 
82
+ # A YAML comment line still runs through ERB, but what its tag writes
83
+ # goes with the comment: database.yml's `# url: <%= ENV["MY_APP_DATABASE_URL"] %>`
84
+ # names a variable nothing uses.
85
+ YAML_COMMENT_LINE = /^[ \t]*#.*$/
86
+
80
87
  # ERB tags carry the Ruby of a `.yml` or `.erb` file.
81
88
  def ruby_source(path, source)
82
89
  return source if ruby?(path)
83
90
 
91
+ # Blanked, not dropped, so every line keeps its number.
92
+ source = source.gsub(YAML_COMMENT_LINE) { |line| " " * line.size } if path.end_with?(".yml", ".yaml")
84
93
  RailsAiContext::ErbSource.ruby_in_place(source)
85
94
  end
86
95
 
@@ -197,7 +197,7 @@ module RailsAiContext
197
197
  end
198
198
 
199
199
  def component_files(dir, matcher)
200
- FileWalk.each_file(dir, skip: scan_skip_dirs).select { |path| File.basename(path).match?(matcher) }
200
+ FileWalk.each_file(dir, skip: scan_skip_dirs, root: root.to_s).select { |path| File.basename(path).match?(matcher) }
201
201
  end
202
202
 
203
203
  def scan_skip_dirs
@@ -179,6 +179,18 @@ module RailsAiContext
179
179
  "webauthn" => { category: :auth, note: "WebAuthn/passkey registration and authentication." }
180
180
  }.freeze
181
181
 
182
+ # Notable gems that run inside the app whichever gem brings them in: one that ships a Rails::Railtie or
183
+ # Rails::Engine, and OmniAuth, the Rack middleware a gem pulls in only to mount it in the app.
184
+ IN_APP_GEMS = %w[
185
+ action_policy activeadmin administrate alba apipie-rails avo bullet carrierwave clearance cocoon cssbundling-rails
186
+ datadog delayed_job devise devise-jwt doorkeeper factory_bot_rails falcon figaro flipper good_job grape graphql
187
+ hotwire-native-rails importmap-rails inertia_rails jbuilder jsbundling-rails jsonapi-serializer kaminari kredis lockbox
188
+ lookbook madmin mission_control-jobs mongoid motor-admin neighbor newrelic_rpm noticed omniauth paper_trail pg_search
189
+ propshaft pundit rack-attack react-rails rodauth-rails rolify rspec-rails rswag-api rswag-ui sentry-rails sidekiq
190
+ simple_form skylight solid_cable solid_cache solid_errors solid_queue sorcery stimulus-rails
191
+ stripe tailwindcss-rails trestle turbo-rails view_component will_paginate
192
+ ].to_set.freeze
193
+
182
194
  # @return [Hash] gem analysis
183
195
  def call
184
196
  lock = RailsAiContext::GemLock.for(root)
@@ -220,7 +232,8 @@ module RailsAiContext
220
232
  if opts[:path]
221
233
  { name: entry[:name], source: "path", location: opts[:path].to_s }
222
234
  elsif opts[:git]
223
- { name: entry[:name], source: "git", location: opts[:git].to_s }
235
+ # A private repository's URL may carry its credentials (`https://user:token@host/...`).
236
+ { name: entry[:name], source: "git", location: RailsAiContext::Redaction.redact_source_line(opts[:git].to_s) }
224
237
  end
225
238
  end
226
239
  rescue => e
@@ -242,14 +255,9 @@ module RailsAiContext
242
255
  RailsAiContext.debug_fail(e, {}, label: "detect_gem_groups")
243
256
  end
244
257
 
245
- # Every Rails app resolves minitest through activesupport, so presence
246
- # in the lockfile said nothing about the app's own test suite.
247
- INDIRECT_BY_DEFAULT = %w[minitest].freeze
248
-
249
258
  def detect_notable_gems(lock)
250
259
  NOTABLE_GEMS.filter_map do |gem_name, info|
251
- next unless lock.present?(gem_name)
252
- next if INDIRECT_BY_DEFAULT.include?(gem_name) && !app_uses_indirect?(gem_name, lock)
260
+ next unless lock.present?(gem_name) && app_uses?(gem_name, lock)
253
261
 
254
262
  {
255
263
  name: gem_name,
@@ -260,10 +268,15 @@ module RailsAiContext
260
268
  end
261
269
  end
262
270
 
263
- def app_uses_indirect?(gem_name, lock)
264
- return true if lock.direct?(gem_name)
271
+ # A lockfile also holds what other gems depend on: Rails resolves every framework and
272
+ # loads only those it is asked for, and activesupport pulls in minitest.
273
+ def app_uses?(gem_name, lock)
274
+ return true if lock.direct?(gem_name) || IN_APP_GEMS.include?(gem_name)
275
+ return Dir.exist?(File.join(root, "test")) if gem_name == "minitest"
276
+ # Rails 7 left Sprockets out of rails/all and the rails gem: a gem that pulls it in since loads it.
277
+ return true if gem_name == "sprockets-rails" && lock.version("rails") && Gem::Version.new(lock.version("rails")) >= Gem::Version.new("7.0")
265
278
 
266
- gem_name == "minitest" ? Dir.exist?(File.join(root, "test")) : false
279
+ lock.present?("rails") && AppKind.loads_framework?(root, gem_name)
267
280
  end
268
281
 
269
282
  # What the app does with Redis is a question the app answers. The fixed
@@ -39,7 +39,7 @@ module RailsAiContext
39
39
  def read(root)
40
40
  classes = {}
41
41
  DIRS.each do |dir|
42
- FileWalk.each_file(File.join(root, dir)).select { |path| path.end_with?(".rb") }.sort.each do |path|
42
+ FileWalk.each_file(File.join(root, dir), root: root).select { |path| path.end_with?(".rb") }.sort.each do |path|
43
43
  relative = path.delete_prefix("#{root}/")
44
44
  source, = SafePath.read(relative, under: root)
45
45
  next if source.nil? || source.empty?
@@ -48,7 +48,7 @@ module RailsAiContext
48
48
 
49
49
  def ruby_files(root)
50
50
  ([ root ] + PathResolver.code_roots(root)).uniq.flat_map do |dir|
51
- %w[app lib].flat_map { |sub| FileWalk.each_file(File.join(dir, sub), skip: SKIP).select { |f| f.end_with?(".rb") } }
51
+ %w[app lib].flat_map { |sub| FileWalk.each_file(File.join(dir, sub), skip: SKIP, root: root).select { |f| f.end_with?(".rb") } }
52
52
  end.uniq
53
53
  end
54
54
 
@@ -15,7 +15,7 @@ module RailsAiContext
15
15
  names = Set.new
16
16
  included = []
17
17
  PathResolver.dirs_for(root, "app/helpers").each do |dir|
18
- FileWalk.each_file(dir).select { |path| path.end_with?(".rb") }.each do |path|
18
+ FileWalk.each_file(dir, root: root).select { |path| path.end_with?(".rb") }.each do |path|
19
19
  source = RailsAiContext::SafeFile.read(path)
20
20
  next if source && !view_helper?(source, path, dir)
21
21
 
@@ -140,7 +140,7 @@ module RailsAiContext
140
140
  # says nothing about this one. The rest are read only when the running
141
141
  # one is absent, where reading nothing would be the worse answer.
142
142
  env = ENV["RAILS_ENV"] || "development"
143
- all_environments = Dir.glob(File.join(root, "config", "environments", "*.rb")).sort
143
+ all_environments = Dir.glob(File.join(root, "config", "environments", "*.rb")).sort.reject { |path| PathResolver.linked_out?(path, root) }
144
144
  running = all_environments.select { |path| File.basename(path, ".rb") == env }
145
145
  environments = running.any? ? running.map { |path| [ path, false ] }
146
146
  : all_environments.map { |path| [ path, true ] }
@@ -174,7 +174,7 @@ module RailsAiContext
174
174
  .map { |entry| entry.merge(spelling: :config) } +
175
175
  walked[:i18n].select { |entry| I18N_AVAILABLE_LOCALES.include?(entry[:path]) }
176
176
  .map { |entry| entry.merge(spelling: :i18n) }
177
- entries.select { |entry| entry[:assignment] }.sort_by { |entry| entry[:location] }
177
+ entries.select { |entry| entry[:assignment] }.sort_by.with_index { |entry, i| [ entry[:location], i ] }
178
178
  end
179
179
 
180
180
  # One walk per config file for every i18n setting read from it, and none for
@@ -357,7 +357,7 @@ module RailsAiContext
357
357
  # The files Rails loads: config/locales by its glob, then what config.i18n.load_path adds.
358
358
  def locale_file_paths
359
359
  @locale_file_paths ||= begin
360
- defaults = locales_dir ? Dir.glob(File.join(locales_dir, RAILS_LOCALE_GLOB)) : []
360
+ defaults = locales_dir ? Dir.glob(File.join(locales_dir, RAILS_LOCALE_GLOB)).reject { |path| PathResolver.linked_out?(path, root) } : []
361
361
  (defaults + load_path_files).uniq.sort
362
362
  end
363
363
  end
@@ -177,7 +177,7 @@ module RailsAiContext
177
177
  @queue_as_walks[file] = begin
178
178
  real = File.realpath(file)
179
179
  root = app.root.to_s
180
- if SourceScan.under_root?(file, real, root, app_root_real)
180
+ if SourceScan.under_root?(file, real, root, app_root_real) && !PathResolver.linked_out?(file, root)
181
181
  relative = SourceScan.relative_file(file, real, root, app_root_real)
182
182
  job_candidates.values.find { |candidate| candidate.file == relative }&.ast ||
183
183
  (source = SafeFile.read(file)) && SourceIntrospector.walk_source(source, QUEUE_AS_LISTENERS)
@@ -965,7 +965,8 @@ module RailsAiContext
965
965
  end
966
966
 
967
967
  def app_config_files
968
- @app_config_files ||= MAILER_CONFIG_GLOBS.flat_map { |glob| Dir.glob(File.join(app.root.to_s, glob)).sort }.map { |path| path.delete_prefix("#{app.root}/") }
968
+ @app_config_files ||= MAILER_CONFIG_GLOBS.flat_map { |glob| Dir.glob(File.join(app.root.to_s, glob)).sort }
969
+ .reject { |path| PathResolver.linked_out?(path, app.root) }.map { |path| path.delete_prefix("#{app.root}/") }
969
970
  end
970
971
 
971
972
  def mailer_config_files
@@ -1253,15 +1254,16 @@ module RailsAiContext
1253
1254
  end
1254
1255
 
1255
1256
  # The file a loaded class was read from, root-relative; nil when the
1256
- # constant has no source location or it lies outside the app. Compared
1257
- # as real paths, the way app_defined? does, so a symlinked root keeps it.
1257
+ # constant has no source location, it lies outside the app, or a symlink
1258
+ # carries it out of the app. Compared as real paths, the way app_defined?
1259
+ # does, so a symlinked root keeps it.
1258
1260
  def source_file_for(klass)
1259
1261
  location = Object.const_source_location(klass.name)&.first
1260
1262
  return nil unless location
1261
1263
 
1262
1264
  real = File.realpath(location)
1263
1265
  root = app.root.to_s
1264
- return nil unless SourceScan.under_root?(location, real, root, app_root_real)
1266
+ return nil unless SourceScan.under_root?(location, real, root, app_root_real) && !PathResolver.linked_out?(location, root)
1265
1267
 
1266
1268
  SourceScan.relative_file(location, real, root, app_root_real)
1267
1269
  rescue NameError, ArgumentError, TypeError, SystemCallError
@@ -1301,7 +1303,7 @@ module RailsAiContext
1301
1303
 
1302
1304
  def channel_source(channel)
1303
1305
  path = channel_absolute_path(channel)
1304
- return nil unless path && File.exist?(path)
1306
+ return nil unless path && File.exist?(path) && !PathResolver.linked_out?(path, app.root)
1305
1307
  RailsAiContext::SafeFile.read(path)
1306
1308
  end
1307
1309
 
@@ -198,14 +198,11 @@ module RailsAiContext
198
198
  end
199
199
  end
200
200
 
201
- # The source of a call's default: option, for a value no literal carries.
202
- def default_source(node)
203
- value = default_node(node)
204
- value && NodeSource.text(value)
205
- end
206
-
207
- def proc_default?(node)
208
- proc_argument?(default_node(node))
201
+ # A call's default: for a value no literal carries: its source, and whether it is a proc.
202
+ # key: reads create_table's default for its implicit key.
203
+ def default_fields(node, key: false)
204
+ value = key ? key_default_node(node) : default_node(node)
205
+ { default_source: value && NodeSource.text(value), default_proc: proc_argument?(value) }
209
206
  end
210
207
 
211
208
  # `-> {}`, `lambda {}`, `proc {}` or `Proc.new {}`.
@@ -219,6 +216,13 @@ module RailsAiContext
219
216
  arg.name == :new && (receiver.is_a?(Prism::ConstantReadNode) || receiver.is_a?(Prism::ConstantPathNode)) && constant_path_string(receiver) == "Proc"
220
217
  end
221
218
 
219
+ # The default create_table gives its implicit key: an `id: { default: }` over its own `default:`.
220
+ def key_default_node(node)
221
+ id = extract_keyword_nodes(node)[:id]
222
+ inner = id.elements.find { |e| e.is_a?(Prism::AssocNode) && extract_key(e.key) == :default } if id.is_a?(Prism::HashNode)
223
+ inner&.value || default_node(node)
224
+ end
225
+
222
226
  def default_node(node)
223
227
  args = node.arguments&.arguments || []
224
228
  assoc = args.grep(Prism::KeywordHashNode)