rails-ai-context 5.35.0 → 5.35.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1024 -0
  3. data/CONTEXT.md +37 -7
  4. data/CONTRIBUTING.md +30 -1
  5. data/README.md +9 -9
  6. data/SECURITY.md +2 -2
  7. data/docs/ARCHITECTURE.md +12 -4
  8. data/docs/CLI.md +44 -15
  9. data/docs/COMPATIBILITY.md +34 -8
  10. data/docs/CONFIGURATION.md +28 -5
  11. data/docs/CUSTOM_TOOLS.md +8 -2
  12. data/docs/FAQ.md +5 -5
  13. data/docs/GUIDE.md +33 -65
  14. data/docs/INTROSPECTORS.md +4 -4
  15. data/docs/QUICKSTART.md +1 -1
  16. data/docs/RECIPES.md +5 -5
  17. data/docs/RELEASING.md +3 -1
  18. data/docs/SECURITY.md +255 -74
  19. data/docs/SETUP.md +1 -1
  20. data/docs/STANDALONE.md +2 -2
  21. data/docs/TOOLS.md +151 -90
  22. data/docs/TROUBLESHOOTING.md +123 -8
  23. data/docs/_config.yml +1 -1
  24. data/docs/adr/0008-query-runs-only-where-it-can-guard.md +38 -0
  25. data/docs/adr/0009-dbhub-shapes-in-ruby.md +26 -0
  26. data/docs/adr/0010-one-config-tool-with-sections.md +32 -0
  27. data/docs/index.md +2 -2
  28. data/exe/rails-ai-context +24 -9
  29. data/lib/generators/rails_ai_context/install/install_generator.rb +2 -0
  30. data/lib/rails_ai_context/app_databases.rb +377 -0
  31. data/lib/rails_ai_context/app_kind.rb +2 -0
  32. data/lib/rails_ai_context/cli/app_root.rb +1 -0
  33. data/lib/rails_ai_context/cli/tool_runner.rb +110 -99
  34. data/lib/rails_ai_context/configuration.rb +15 -8
  35. data/lib/rails_ai_context/database_family/mysql_runtime.rb +170 -0
  36. data/lib/rails_ai_context/database_family/postgresql_plan.rb +124 -0
  37. data/lib/rails_ai_context/database_family/postgresql_runtime.rb +211 -0
  38. data/lib/rails_ai_context/database_family/sqlite_runtime.rb +391 -0
  39. data/lib/rails_ai_context/database_family.rb +282 -0
  40. data/lib/rails_ai_context/database_file.rb +61 -7
  41. data/lib/rails_ai_context/database_yml.rb +7 -29
  42. data/lib/rails_ai_context/doctor.rb +15 -16
  43. data/lib/rails_ai_context/instrumentation.rb +1 -1
  44. data/lib/rails_ai_context/introspectors/connection_pool_introspector.rb +8 -23
  45. data/lib/rails_ai_context/introspectors/database_stats_introspector.rb +6 -70
  46. data/lib/rails_ai_context/introspectors/declared_constant.rb +19 -7
  47. data/lib/rails_ai_context/introspectors/env_introspector.rb +1 -1
  48. data/lib/rails_ai_context/introspectors/env_references.rb +1 -1
  49. data/lib/rails_ai_context/introspectors/job_introspector.rb +7 -7
  50. data/lib/rails_ai_context/introspectors/migration_introspector.rb +8 -11
  51. data/lib/rails_ai_context/introspectors/migration_replay/statements.rb +9 -9
  52. data/lib/rails_ai_context/introspectors/model_introspector.rb +33 -39
  53. data/lib/rails_ai_context/introspectors/multi_database_introspector.rb +33 -30
  54. data/lib/rails_ai_context/introspectors/node_source.rb +2 -0
  55. data/lib/rails_ai_context/introspectors/performance_introspector.rb +80 -24
  56. data/lib/rails_ai_context/introspectors/pg_partitions.rb +1 -4
  57. data/lib/rails_ai_context/introspectors/schema_conventions.rb +6 -8
  58. data/lib/rails_ai_context/introspectors/schema_dump_path.rb +12 -14
  59. data/lib/rails_ai_context/introspectors/schema_introspector.rb +47 -27
  60. data/lib/rails_ai_context/introspectors/schema_reader.rb +1 -1
  61. data/lib/rails_ai_context/introspectors/source_scan.rb +39 -4
  62. data/lib/rails_ai_context/migration_status.rb +18 -46
  63. data/lib/rails_ai_context/payload.rb +48 -1
  64. data/lib/rails_ai_context/pending_migrations.rb +3 -4
  65. data/lib/rails_ai_context/run_cache.rb +5 -0
  66. data/lib/rails_ai_context/schema_adapter.rb +13 -18
  67. data/lib/rails_ai_context/secret_columns.rb +506 -0
  68. data/lib/rails_ai_context/serializers/claude_rules_serializer.rb +2 -2
  69. data/lib/rails_ai_context/serializers/markdown_serializer.rb +14 -10
  70. data/lib/rails_ai_context/serializers/section_facts.rb +12 -2
  71. data/lib/rails_ai_context/serializers/stack_overview_helper.rb +5 -2
  72. data/lib/rails_ai_context/serializers/tool_guide_helper.rb +35 -7
  73. data/lib/rails_ai_context/server.rb +143 -29
  74. data/lib/rails_ai_context/sql_gate.rb +1007 -0
  75. data/lib/rails_ai_context/tasks/rails_ai_context.rake +1 -2
  76. data/lib/rails_ai_context/test_helper.rb +3 -1
  77. data/lib/rails_ai_context/tools/analyze_feature.rb +11 -7
  78. data/lib/rails_ai_context/tools/base_tool.rb +71 -48
  79. data/lib/rails_ai_context/tools/dependency_graph.rb +1 -1
  80. data/lib/rails_ai_context/tools/diagnose.rb +105 -42
  81. data/lib/rails_ai_context/tools/generate_test.rb +4 -6
  82. data/lib/rails_ai_context/tools/get_active_support.rb +1 -6
  83. data/lib/rails_ai_context/tools/get_api.rb +1 -6
  84. data/lib/rails_ai_context/tools/get_autoload.rb +1 -6
  85. data/lib/rails_ai_context/tools/get_callbacks.rb +5 -5
  86. data/lib/rails_ai_context/tools/get_component_catalog.rb +6 -7
  87. data/lib/rails_ai_context/tools/get_concern.rb +6 -6
  88. data/lib/rails_ai_context/tools/get_config.rb +189 -12
  89. data/lib/rails_ai_context/tools/get_context.rb +111 -105
  90. data/lib/rails_ai_context/tools/get_controllers.rb +8 -5
  91. data/lib/rails_ai_context/tools/get_conventions.rb +1 -6
  92. data/lib/rails_ai_context/tools/get_engines.rb +1 -6
  93. data/lib/rails_ai_context/tools/get_env.rb +1 -6
  94. data/lib/rails_ai_context/tools/get_env_config.rb +2 -8
  95. data/lib/rails_ai_context/tools/get_frontend_stack.rb +1 -6
  96. data/lib/rails_ai_context/tools/get_gems.rb +1 -6
  97. data/lib/rails_ai_context/tools/get_helper_methods.rb +4 -5
  98. data/lib/rails_ai_context/tools/get_i18n.rb +1 -7
  99. data/lib/rails_ai_context/tools/get_job_pattern.rb +1 -1
  100. data/lib/rails_ai_context/tools/get_mailers.rb +1 -1
  101. data/lib/rails_ai_context/tools/get_model_details.rb +5 -8
  102. data/lib/rails_ai_context/tools/get_partial_interface.rb +3 -4
  103. data/lib/rails_ai_context/tools/get_schema.rb +10 -8
  104. data/lib/rails_ai_context/tools/get_service_pattern.rb +60 -24
  105. data/lib/rails_ai_context/tools/get_turbo_map.rb +1 -1
  106. data/lib/rails_ai_context/tools/migration_advisor.rb +47 -32
  107. data/lib/rails_ai_context/tools/onboard.rb +24 -24
  108. data/lib/rails_ai_context/tools/performance_check.rb +15 -2
  109. data/lib/rails_ai_context/tools/query.rb +142 -1356
  110. data/lib/rails_ai_context/tools/read_logs.rb +1 -1
  111. data/lib/rails_ai_context/tools/review_changes.rb +6 -7
  112. data/lib/rails_ai_context/tools/runtime_info.rb +125 -88
  113. data/lib/rails_ai_context/tools/safe_call.rb +70 -28
  114. data/lib/rails_ai_context/tools/search_code.rb +0 -4
  115. data/lib/rails_ai_context/tools/search_docs.rb +3 -1
  116. data/lib/rails_ai_context/tools/security_scan.rb +9 -5
  117. data/lib/rails_ai_context/tools/session_context.rb +18 -12
  118. data/lib/rails_ai_context/tools/validate.rb +1 -0
  119. data/lib/rails_ai_context/tools/validate_semantics.rb +5 -5
  120. data/lib/rails_ai_context/version.rb +1 -1
  121. data/lib/rails_ai_context.rb +4 -7
  122. data/server.json +2 -2
  123. metadata +14 -3
@@ -224,9 +224,10 @@ module RailsAiContext
224
224
 
225
225
  next if mixin_path?(record.path_name.underscore, source)
226
226
 
227
- declarations = DeclaredConstant.declarations(source, path_name: record.path_name)
228
- declaration = DeclaredConstant.declaration_for(declarations, record.path_name)
229
- class_name = declaration&.name || record.path_name
227
+ path_name = SourceScan.concerns_name(record.path_name, source)
228
+ declarations = DeclaredConstant.declarations(source, path_name: path_name)
229
+ declaration = DeclaredConstant.declaration_for(declarations, path_name)
230
+ class_name = declaration&.name || path_name
230
231
  # Two files can declare one class (an app reopening a model to add methods);
231
232
  # the one that names a superclass defines the model.
232
233
  previous = found[class_name]
@@ -245,7 +246,7 @@ module RailsAiContext
245
246
  rescue => e
246
247
  # The file is known here whatever failed, and a consumer with none
247
248
  # derives app/models/<name>.rb, which a pack model does not have.
248
- found[record.path_name] = { error: e.message, file: record.file }.compact
249
+ found[SourceScan.concerns_name(record.path_name, source.to_s)] = { error: e.message, file: record.file }.compact
249
250
  end
250
251
  end
251
252
  end
@@ -391,10 +392,10 @@ module RailsAiContext
391
392
  # The tables every other database's dump holds; nil when one of them is not on disk to read.
392
393
  def secondary_dump_tables
393
394
  RunCache.fetch([ :secondary_dump_tables, app.root.to_s ]) do
394
- SchemaDumpPath.secondaries(app.root).values.each_with_object(Set.new) do |(format, path), tables|
395
- break nil unless File.exist?(path)
395
+ AppDatabases.for(app.root).reject(&:primary).filter_map(&:dump).each_with_object(Set.new) do |dump, tables|
396
+ break nil unless File.exist?(dump.path)
396
397
 
397
- found = format == :ruby ? SchemaReader.new(path).tables : StructureSqlReader.parse(SafeFile.read(path).to_s)[:tables]
398
+ found = dump.format == :ruby ? SchemaReader.new(dump.path).tables : StructureSqlReader.parse(SafeFile.read(dump.path).to_s)[:tables]
398
399
  tables.merge(found.keys)
399
400
  end
400
401
  end
@@ -550,16 +551,12 @@ module RailsAiContext
550
551
  source.match?(/^[^\S\n]*primary_abstract_class\b/)
551
552
  end
552
553
 
553
- # `concerns/` under app/models is the Zeitwerk root for mixins: it does
554
- # not namespace its files, so the path is not their name. A nested
555
- # concerns/ is an ordinary namespace - OpenProject fills one with mixins,
556
- # but a class declared there is a model like any other.
554
+ # A file under any concerns/ that is not named for a class. One that is
555
+ # may be a model: Whitehall keeps one in app/models/concerns itself.
557
556
  def mixin_path?(relative, source)
558
- segments = relative.split("/")
559
- return false unless segments.include?("concerns")
560
- return true if segments.first == "concerns"
557
+ return false unless relative.split("/").include?("concerns")
561
558
 
562
- !DeclaredConstant.declares_class?(source, path_name: relative.camelize)
559
+ SourceScan.concern_class(source, SourceScan.concerns_name(relative.camelize, source)).nil?
563
560
  end
564
561
 
565
562
  # A class whose file is inside an installed gem (PaperTrail::Version) is
@@ -596,15 +593,17 @@ module RailsAiContext
596
593
 
597
594
  known = models.map(&:name).to_set
598
595
  @outside_model_dirs = Set.new
599
- # Concerns stay in: a nested concerns directory is a namespace, so a
600
- # class declared under one is a model and constantize sorts the mixins
601
- # out. A top-level `app/models/concerns` is an autoload root instead,
602
- # so its files declare no `Concerns::` prefix and that path name never
603
- # constantizes.
596
+ # A nested concerns directory is a namespace, so a class under one is a model;
597
+ # under the top-level one only a file that declares a class is read on.
604
598
  SourceScan.model_paths(app.root, base_model: model_base_check).each do |record|
605
- next if record.path_name.start_with?("Concerns::")
606
- next if known.include?(record.path_name)
607
- next if config.excluded_models.include?(record.path_name)
599
+ path_name = record.path_name
600
+ if path_name.start_with?("Concerns::")
601
+ source = model_source(record.path).to_s
602
+ path_name = SourceScan.concerns_name(path_name, source)
603
+ end
604
+ next if known.include?(path_name)
605
+ next if config.excluded_models.include?(path_name)
606
+ next if path_name != record.path_name && mixin_path?(record.path_name.underscore, source)
608
607
 
609
608
  # The path does not name the class: an app inflection only changes
610
609
  # case, so activitypub/activity.rb camelizes to a constant the app
@@ -612,7 +611,7 @@ module RailsAiContext
612
611
  # load. A root pushed under a namespace declares a prefix the path
613
612
  # lacks. Read only where the camelized name is not already loaded, so
614
613
  # a booted run does not parse every model file to learn nothing.
615
- class_name = DeclaredConstant.named(model_source(record.path).to_s, record.path_name)
614
+ class_name = DeclaredConstant.named(model_source(record.path).to_s, path_name)
616
615
  next if known.include?(class_name)
617
616
  next if config.excluded_models.include?(class_name)
618
617
 
@@ -630,10 +629,9 @@ module RailsAiContext
630
629
  next unless klass < ActiveRecord::Base && !klass.abstract_class?
631
630
  models << klass
632
631
  known << class_name
633
- rescue NameError, LoadError, ScriptError => e
634
- # A syntax-broken file costs itself, not the whole listing, but
635
- # its name is recorded: a file that exists for a class reflection
636
- # lacks is not the same answer as no such model.
632
+ rescue StandardError, ScriptError => e
633
+ # A file that raises as it loads costs itself alone, and its name is recorded:
634
+ # a file that exists for a class reflection lacks is no missing model.
637
635
  @unloadable[class_name] = e.message.to_s.lines.first.to_s.strip
638
636
  @outside_model_dirs << class_name unless model_dir_file?(record.path)
639
637
  end
@@ -988,7 +986,7 @@ module RailsAiContext
988
986
  bases = abstract_bases(model)
989
987
  return bases unless model.superclass == ActiveRecord::Base
990
988
 
991
- bases | ActiveRecord::Base.subclasses.select { |klass| klass.name && klass.abstract_class? }
989
+ bases | DeclaredConstant.carried(ActiveRecord::Base.subclasses).select(&:abstract_class?)
992
990
  end
993
991
 
994
992
  def every_model_modules
@@ -1138,13 +1136,7 @@ module RailsAiContext
1138
1136
 
1139
1137
  return nil unless has_type_column
1140
1138
 
1141
- children = if model.respond_to?(:descendants)
1142
- model.descendants.map(&:name).compact.sort
1143
- elsif model.respond_to?(:subclasses)
1144
- model.subclasses.map(&:name).compact.sort
1145
- else
1146
- []
1147
- end
1139
+ children = DeclaredConstant.carried(model.descendants).map(&:name).sort
1148
1140
 
1149
1141
  parent = model.superclass
1150
1142
  sti_parent = if parent && parent != ActiveRecord::Base &&
@@ -2245,15 +2237,17 @@ module RailsAiContext
2245
2237
  source = model_source(path)
2246
2238
  next if source.nil? || mixin_path?(relative, source) || abstract_class?(source)
2247
2239
 
2240
+ path_name = SourceScan.concerns_name(relative.camelize, source)
2241
+
2248
2242
  # The static tier never loads the app's inflector, so camelizing alone
2249
2243
  # would invent `Activitypub::` for an app that declares `ActivityPub::`.
2250
- class_name = DeclaredConstant.resolve(source, relative.camelize)
2244
+ class_name = DeclaredConstant.resolve(source, path_name)
2251
2245
  next if found.key?(class_name) || config.excluded_models.include?(class_name)
2252
2246
 
2253
- declaration = DeclaredConstant.declaration_for(DeclaredConstant.declarations(source, path_name: relative.camelize), class_name)
2247
+ declaration = DeclaredConstant.declaration_for(DeclaredConstant.declarations(source, path_name: path_name), class_name)
2254
2248
  found[class_name] = { path: path, source: source, superclass: declaration&.superclass, nesting: declaration&.nesting }
2255
2249
  rescue => e
2256
- found[relative.camelize] ||= { error: e.message }
2250
+ found[path_name || relative.camelize] ||= { error: e.message }
2257
2251
  end
2258
2252
  end
2259
2253
  end
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "uri"
4
+
3
5
  module RailsAiContext
4
6
  module Introspectors
5
7
  # Discovers multi-database configuration: multiple databases, replicas,
@@ -13,22 +15,37 @@ module RailsAiContext
13
15
  dbs = discover_databases
14
16
  {
15
17
  databases: dbs,
16
- replicas: dbs.select { |d| d[:replica] }.map { |d| { name: d[:name], adapter: d[:adapter] } },
18
+ replicas: dbs.select { |d| d[:replica] }.map { |d| d.slice(:name, :adapter, :environments) },
17
19
  sharding: detect_sharding,
18
20
  model_connections: detect_model_connections,
19
21
  multi_db: dbs.size > 1
20
- }.merge({ example_adapters: example_adapters }.compact)
22
+ }.merge(from_configuration)
23
+ end
24
+
25
+ # What the section says of each database by name, read from configuration alone.
26
+ def listing
27
+ { databases: discover_databases }.merge(from_configuration)
21
28
  end
22
29
 
23
30
  private
24
31
 
32
+ def from_configuration
33
+ { example_adapters: example_adapters, primary_name: primary_name }.compact
34
+ end
35
+
36
+ # The primary's database.yml name where it is not "primary": a generated file tells it from a database that is.
37
+ def primary_name
38
+ name = RailsAiContext::AppDatabases.for(root).primary&.name.to_s
39
+ name unless name.empty? || name == "primary"
40
+ rescue => e
41
+ RailsAiContext.debug_fail(e, nil, label: "primary_name")
42
+ end
43
+
44
+ # Every app database; one the running environment does not configure names where it is configured.
25
45
  def discover_databases
26
- booted = booted_databases
27
- # A boot that failed leaves the handlers empty without raising, and an
28
- # app that declares two databases is not an app with none.
29
- booted.any? ? booted : file_databases
46
+ RailsAiContext::AppDatabases.for(root).map { |db| database_entry(db) }
30
47
  rescue => e
31
- RailsAiContext.debug_fail(e, file_databases, label: "discover_databases")
48
+ RailsAiContext.debug_fail(e, [], label: "discover_databases")
32
49
  end
33
50
 
34
51
  # Adapters in the example database files of an app that commits none, read by line since
@@ -113,30 +130,16 @@ module RailsAiContext
113
130
  end
114
131
  end
115
132
 
116
- def booted_databases
117
- return [] unless defined?(ActiveRecord::Base)
118
-
119
- ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, include_hidden: true).map do |config|
120
- info = { name: config.name, adapter: config.adapter }
121
- info[:database] = anonymize_db_name(config.database) if config.database
122
- info[:replica] = true if config.respond_to?(:replica?) && config.replica?
123
- info
133
+ def database_entry(db)
134
+ info = { name: db.name, adapter: db.adapter }
135
+ info[:adapter_default] = true if db.adapter_default
136
+ database = db.config.database if db.config.respond_to?(:database)
137
+ info[:database] = anonymize_db_name(database) if database
138
+ info[:replica] = true if db.replica
139
+ # Configured only in another environment, or only a committed dump names it.
140
+ unless db.here
141
+ db.environments.any? ? info[:environments] = db.environments : info[:dump_only] = true
124
142
  end
125
- end
126
-
127
- def file_databases
128
- databases = RailsAiContext::DatabaseYml.databases(root)
129
- # Rails builds the primary from DATABASE_URL when the file has no entry for the env.
130
- return ENV["DATABASE_URL"].to_s.empty? ? [] : [ database_entry("primary", {}) ] if databases.empty?
131
-
132
- databases.map { |name, entry| database_entry(name, entry) }
133
- end
134
-
135
- def database_entry(name, entry)
136
- adapter, from_default = RailsAiContext::DatabaseYml.adapter(name, entry)
137
- info = { name: name.to_s, adapter: adapter }
138
- info[:adapter_default] = true if from_default
139
- info[:replica] = true if entry["replica"] == true
140
143
  info
141
144
  end
142
145
 
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "prism"
4
+
3
5
  module RailsAiContext
4
6
  module Introspectors
5
7
  # A node's source with its heredoc bodies: Prism's `slice` stops at the node's end, so
@@ -18,27 +18,68 @@ module RailsAiContext
18
18
  attr_writer :context
19
19
 
20
20
  def call
21
- schema_data = load_schema_data
22
21
  model_data = load_model_data
22
+ schemas, unchecked = database_schemas
23
+ counter_cache = []
24
+ fk_indexes = []
25
+ schemas.each do |name, tables, foreign_keys|
26
+ tag = ->(finding) { name ? finding.merge(database: name) : finding }
27
+ counter_cache.concat(detect_missing_counter_cache(model_data, tables).map(&tag))
28
+ fk_indexes.concat(detect_missing_fk_indexes(tables, model_data, foreign_keys).map(&tag))
29
+ end
23
30
 
24
31
  {
25
32
  n_plus_one_risks: detect_n_plus_one,
26
- missing_counter_cache: detect_missing_counter_cache(model_data, schema_data),
27
- missing_fk_indexes: detect_missing_fk_indexes(schema_data, model_data, load_foreign_keys),
33
+ missing_counter_cache: counter_cache.uniq { |f| f.except(:database) },
34
+ missing_fk_indexes: fk_indexes,
28
35
  model_all_in_controllers: detect_model_all_in_controllers(model_data),
29
36
  eager_load_candidates: detect_eager_load_candidates(model_data),
30
37
  summary: nil # populated below
31
- }.tap { |result| result[:summary] = build_summary(result) }
38
+ }.tap do |result|
39
+ result[:unchecked_databases] = unchecked if unchecked.any?
40
+ result[:summary] = build_summary(result)
41
+ end
32
42
  end
33
43
 
34
44
  private
35
45
 
36
- def load_schema_data
37
- SchemaReader.for(root).tables
46
+ # [[database name (nil for the primary), tables, foreign keys]], and { name:, reason: } for each database not read.
47
+ def database_schemas
48
+ reader = SchemaReader.for(root)
49
+ schemas = [ [ nil, reader.tables, reader.foreign_keys ] ]
50
+ unchecked = []
51
+ databases = AppDatabases.for(root)
52
+ # Rails names one file for the first database and the one named primary. It is read once, above.
53
+ shared = databases.primary&.dump&.path
54
+ databases.reject { |db| db.primary || db.replica || (shared && db.dump&.path == shared) }.each do |db|
55
+ read = read_dump(db)
56
+ read[:reason] ? unchecked << { name: db.name, reason: read[:reason] } : schemas << [ db.name, read[:tables], read[:foreign_keys] ]
57
+ end
58
+ [ schemas, unchecked ]
59
+ end
60
+
61
+ # { tables:, foreign_keys: } from the database's dump, or { reason: } saying why there is none to read.
62
+ def read_dump(db)
63
+ return { reason: no_dump_reason(db) } unless db.dump
64
+
65
+ path = db.dump.path
66
+ shown = path.delete_prefix("#{root.to_s.chomp('/')}/")
67
+ return { reason: "#{shown} is not on disk" } unless File.exist?(path)
68
+
69
+ content = SafeFile.read(path, max_size: RailsAiContext.configuration.max_schema_file_size)
70
+ return { reason: "#{shown} is over max_schema_file_size" } unless content
71
+ return SchemaReader.new(path).then { |schema| { tables: schema.tables, foreign_keys: schema.foreign_keys } } if db.dump.format == :ruby
72
+
73
+ tables = StructureSqlReader.parse(content)[:tables]
74
+ { tables: tables,
75
+ foreign_keys: tables.values.flat_map { |table| Array(table[:foreign_keys]).map { |fk| { from: fk[:from_table], to: fk[:to_table], column: fk[:column] } } } }
38
76
  end
39
77
 
40
- def load_foreign_keys
41
- SchemaReader.for(root).foreign_keys
78
+ def no_dump_reason(db)
79
+ return "its database.yml entry turns `database_tasks` off, so Rails writes it no dump" unless db.tasks
80
+ return "its database.yml entry turns `schema_dump` off, so Rails writes it no dump" unless db.settings["schema_dump"]
81
+
82
+ "`schema_dump` in its database.yml entry names no file this check reads"
42
83
  end
43
84
 
44
85
  # The models section when the run has one; otherwise the model tier is
@@ -77,7 +118,7 @@ module RailsAiContext
77
118
  ast = SourceIntrospector.walk_source(record.source, {
78
119
  classes: Listeners::ClassDefinitionListener,
79
120
  associations: Listeners::AssociationsListener,
80
- tree: TREE_LISTENER,
121
+ column_macros: COLUMN_MACRO_LISTENER,
81
122
  mixins: Listeners::MixinsListener,
82
123
  counter_culture: -> { Listeners::GenericMacroListener.new(:counter_culture) }
83
124
  })
@@ -106,7 +147,7 @@ module RailsAiContext
106
147
  table_name: model_table(class_name),
107
148
  has_many: has_many,
108
149
  belongs_to: belongs_to,
109
- tree_parent_keys: tree_parent_keys(ast, class_name),
150
+ macro_columns: macro_columns(ast, class_name),
110
151
  counter_cultures: ast[:counter_culture].map { |c| counter_culture_record(c) }
111
152
  }
112
153
  rescue => e
@@ -114,24 +155,39 @@ module RailsAiContext
114
155
  end
115
156
  end
116
157
 
117
- # Gem macros that declare a parent key, with the option that renames it.
118
- # ponytail: a fixed list; a gem outside it declares keys the static walk cannot see.
158
+ # Tree gem macros, which declare a parent key, with the option that renames it.
119
159
  TREE_MACROS = { acts_as_tree: :foreign_key, acts_as_nested_set: :parent_column,
120
160
  has_closure_tree: :parent_column_name }.freeze
121
- TREE_LISTENER = -> { Listeners::GenericMacroListener.new(*TREE_MACROS.keys) }
122
- TREE_CONCERN_LISTENERS = { tree: TREE_LISTENER, mixins: Listeners::MixinsListener }.freeze
123
-
124
- # The parent key a tree macro declares, in the model or its concerns.
161
+ # paper_trail adds `has_many :versions, as: :item` when the app boots.
162
+ VERSIONS_MACRO = :has_paper_trail
163
+ # Gem macros that declare a column.
164
+ # ponytail: a fixed list; a gem outside it declares columns the static walk cannot see.
165
+ COLUMN_MACROS = [ *TREE_MACROS.keys, VERSIONS_MACRO ].freeze
166
+ COLUMN_MACRO_LISTENER = -> { Listeners::GenericMacroListener.new(*COLUMN_MACROS) }
167
+ COLUMN_MACRO_CONCERN_LISTENERS = { column_macros: COLUMN_MACRO_LISTENER, mixins: Listeners::MixinsListener }.freeze
168
+
169
+ # The [table, column] a gem macro declares, in the model or its concerns.
125
170
  # The models section cannot carry it statically: the macro is a gem's.
126
- def tree_parent_keys(ast, class_name)
127
- collected = ConcernMacros.collect(root, ast[:mixins], keys: %i[tree], prefer: "model",
171
+ def macro_columns(ast, class_name)
172
+ collected = ConcernMacros.collect(root, ast[:mixins], keys: %i[column_macros], prefer: "model",
128
173
  within: class_name, cache: (@concern_cache ||= {}),
129
- listeners: TREE_CONCERN_LISTENERS).collected
130
- (ast[:tree] + Array(collected[:tree])).map do |macro|
131
- ((macro[:options] || {})[TREE_MACROS[macro[:macro]]] || "parent_id").to_s
174
+ listeners: COLUMN_MACRO_CONCERN_LISTENERS).collected
175
+ (ast[:column_macros] + Array(collected[:column_macros])).map do |macro|
176
+ options = macro[:options] || {}
177
+ next [ versions_table(options, class_name), "item_id" ] if macro[:macro] == VERSIONS_MACRO
178
+
179
+ [ model_table(class_name), (options[TREE_MACROS[macro[:macro]]] || "parent_id").to_s ]
132
180
  end
133
181
  end
134
182
 
183
+ # PaperTrail::Version reads `versions`; `versions: { class_name: }`, or the older `class_name:`, names another class.
184
+ def versions_table(options, owner)
185
+ written = options[:versions].is_a?(Hash) ? options[:versions][:class_name] : options[:class_name]
186
+ return "versions" unless written.is_a?(String)
187
+
188
+ table_of(TableName.resolve_class(written, owner) { |candidate| candidate if resolved_models.key?(candidate) } || written, resolved_models)
189
+ end
190
+
135
191
  # The scan reads the run models section, so an association a concern or a
136
192
  # base declares is one it can follow.
137
193
  def detect_n_plus_one
@@ -271,9 +327,9 @@ module RailsAiContext
271
327
  end
272
328
  end
273
329
 
274
- def tree_parent_columns(model_data)
330
+ def macro_declared_columns(model_data)
275
331
  Array(model_data).each_with_object(Set.new) do |model, found|
276
- Array(model[:tree_parent_keys]).each { |key| found << [ model[:table_name].to_s, key ] }
332
+ Array(model[:macro_columns]).each { |column| found << column }
277
333
  end
278
334
  end
279
335
 
@@ -285,7 +341,7 @@ module RailsAiContext
285
341
  def detect_missing_fk_indexes(schema_data, model_data = [], foreign_keys = [])
286
342
  missing = []
287
343
  declared = foreign_key_columns(foreign_keys) | association_columns(resolved_models) |
288
- tree_parent_columns(model_data)
344
+ macro_declared_columns(model_data)
289
345
 
290
346
  schema_data.each do |table_name, table|
291
347
  columns = table[:columns]
@@ -7,9 +7,6 @@ module RailsAiContext
7
7
  module PgPartitions
8
8
  module_function
9
9
 
10
- # activerecord-postgis-adapter reports "PostGIS" for what is a PostgreSQL connection.
11
- POSTGRES_ADAPTER = /postg/i
12
-
13
10
  # Scoped the way connection.tables is.
14
11
  NAMES_SQL = <<~SQL.squish.freeze
15
12
  SELECT c.relname FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
@@ -19,7 +16,7 @@ module RailsAiContext
19
16
  def names(conn)
20
17
  # relispartition arrived with partitions in 10; querying it before then
21
18
  # fails, and a failed query aborts any open transaction.
22
- return [] unless conn.adapter_name.match?(POSTGRES_ADAPTER) && conn.database_version >= 10_00_00
19
+ return [] unless RailsAiContext::DatabaseFamily.of(conn).postgresql? && conn.database_version >= 10_00_00
23
20
 
24
21
  conn.select_values(NAMES_SQL)
25
22
  rescue => e
@@ -196,17 +196,15 @@ module RailsAiContext
196
196
  # primary, sqlite queue) and each dump must be typed by its own. dump_path
197
197
  # names a secondary database's dump, nil the primary's; database: names the database outright.
198
198
  def implicit_pk_type(root, dump_path = nil, database: SchemaDumpPath.database_name(root, dump_path))
199
- adapter = database_adapter_for(root, database)
200
- adapter&.start_with?("sqlite") ? "integer" : "bigint"
199
+ RailsAiContext::DatabaseFamily.of(database_adapter_for(root, database)).implicit_pk_type
201
200
  end
202
201
 
203
- # The running environment's adapter for one database, as database.yml and a merged URL name it.
204
- # A database it does not configure (Rails 8's queue outside production) takes the adapter of the
205
- # environment that does, else the running primary's.
202
+ # One app database's adapter, the primary's for no name. A database only another environment configures (Rails 8's queue
203
+ # outside production) takes that environment's; one only a dump names, the primary's.
206
204
  def database_adapter_for(root, db_name)
207
- yml = RailsAiContext::DatabaseYml
208
- entry = yml.entry(root, db_name) || yml.elsewhere(root, db_name)&.last
209
- yml.adapter(db_name, entry).first || (yml.adapter("primary", yml.primary(root)).first unless entry)
205
+ databases = RailsAiContext::AppDatabases.for(root)
206
+ found = databases.of_dump_name(db_name)
207
+ found&.config ? found.adapter : databases.primary&.adapter
210
208
  end
211
209
 
212
210
  # A primary key as connection.primary_key gives it: the column's name, or
@@ -20,24 +20,22 @@ module RailsAiContext
20
20
  end
21
21
  end
22
22
 
23
- # Each secondary database's dump by its database.yml name, as schema_dump_path names it:
24
- # its schema_dump, else <name>_schema.rb or <name>_structure.sql in its format.
25
- def secondaries(root)
26
- root = root.to_s
27
- RailsAiContext::RunCache.fetch([ :secondary_schema_dumps, root ]) do
28
- RailsAiContext::DatabaseYml.task_secondaries(root).each_with_object({}) do |(name, entry), found|
29
- format = schema_format(root, entry)
30
- dump = entry.key?("schema_dump") ? configured(root, entry, format) : [ format, File.join(root, "db", "#{name}_#{FILE_NAMES[format]}") ]
31
- found[name] = dump if dump
32
- end
33
- end
23
+ # A secondary database's dump: the file schema_dump_path names (its schema_dump, else <name>_schema.rb or
24
+ # <name>_structure.sql in its format) when on disk, else a file under its name in either format. nil for none.
25
+ def secondary(root, name, config)
26
+ format = schema_format(root, config)
27
+ own = configured(root, config, format) if config.key?("schema_dump")
28
+ # Rails gives an entry named primary the unprefixed file wherever it stands.
29
+ named = [ format, *(FILE_NAMES.keys - [ format ]) ].map { |f| [ f, File.join(root, "db", "#{"#{name}_" unless name == "primary"}#{FILE_NAMES[f]}") ] }
30
+ [ own, *named ].compact.find { |_, path| File.exist?(path) } || (config.key?("schema_dump") ? own : named.first)
34
31
  end
35
32
 
36
- # The database a dump belongs to: the secondary whose dump it is, else the one its file name says.
33
+ # The database a dump belongs to: the app database whose dump it is, else the one its file name says.
34
+ # No dump is the primary's.
37
35
  def database_name(root, dump_path)
38
- return "primary" if dump_path.nil?
36
+ return RailsAiContext::AppDatabases.for(root).primary&.name || "primary" if dump_path.nil?
39
37
 
40
- named = secondaries(root).find { |_, (_, path)| path == dump_path.to_s }&.first
38
+ named = RailsAiContext::AppDatabases.for(root).find_by_dump(dump_path)&.name
41
39
  named || File.basename(dump_path.to_s).sub(/\.(rb|sql)\z/, "").sub(/_?(schema|structure)\z/, "").then { |name| name.empty? ? "primary" : name }
42
40
  end
43
41
 
@@ -143,17 +143,20 @@ module RailsAiContext
143
143
  RailsAiContext.debug_fail(e, "unknown", label: "adapter_name")
144
144
  end
145
145
 
146
+ def family
147
+ RailsAiContext::DatabaseFamily.of(adapter_name)
148
+ end
149
+
146
150
  def sqlite?
147
- adapter_name.to_s.match?(/sqlite/i)
151
+ family.sqlite?
148
152
  end
149
153
 
150
154
  def postgres?
151
- adapter_name.to_s.match?(/postg/i)
155
+ family.postgresql?
152
156
  end
153
157
 
154
- # Every MySQL adapter inherits mariadb? from AbstractMysqlAdapter.
155
158
  def mysql?
156
- connection.respond_to?(:mariadb?)
159
+ RailsAiContext::DatabaseFamily.of(connection).mysql?
157
160
  end
158
161
 
159
162
  def connection
@@ -593,31 +596,41 @@ module RailsAiContext
593
596
  @secondary_database_dumps ||= read_secondary_database_dumps
594
597
  end
595
598
 
596
- # [name, format, path] for each secondary database's dump on disk, the configured first.
599
+ def app_databases
600
+ RailsAiContext::AppDatabases.for(app.root)
601
+ end
602
+
603
+ # [name, format, path] for each other database's dump on disk, in the app databases' order.
597
604
  def secondary_dump_files
598
605
  primary = dump_candidates.map(&:last)
599
- configured = SchemaDumpPath.secondaries(app.root).map { |name, (format, path)| [ name, format, path ] }
600
- taken = primary + configured.map(&:last)
601
- globbed = SchemaDumpPath::FILE_NAMES.flat_map do |format, file|
602
- kind, ext = file.split(".")
603
- Dir.glob(File.join(app.root.to_s, "db", "*_#{kind}.#{ext}")).sort
604
- .reject { |path| taken.include?(path) }
605
- .map { |path| [ File.basename(path, ".#{ext}").delete_suffix("_#{kind}"), format, path ] }
606
+ app_databases.filter_map do |db|
607
+ path = db.dump&.path
608
+ [ db.name, db.dump.format, path ] if path && !db.primary && !primary.include?(path) && File.exist?(path)
606
609
  end
607
- (configured + globbed).reject { |_, _, path| primary.include?(path) || !File.exist?(path) }
608
610
  end
609
611
 
612
+ NO_TABLES = "declares no tables"
613
+
614
+ # A dump too large to read, or holding no tables, keeps an entry that says so.
610
615
  def read_secondary_database_dumps
616
+ unread = {}
611
617
  dumps = secondary_dump_files.each_with_object({}) do |(name, format, path), found|
612
- next if found.key?(name)
613
-
614
618
  parsed = parse_dump(format, path)
615
- next if parsed[:tables].blank?
619
+ if parsed[:tables].blank?
620
+ unread[name] = [ path, parsed[:error]&.delete_prefix("#{relative_dump_path(path)} ") || NO_TABLES ]
621
+ next
622
+ end
616
623
 
617
624
  parsed[:note] = "Parsed from #{relative_dump_path(path)} (from committed dump, not a live connection)"
618
625
  found[name] = parsed.merge(database_files(path, migrate_dir_for_dump(path)))
619
626
  end
620
- replay_secondary_migrations(dumps)
627
+ replay_secondary_migrations(dumps, unread)
628
+ unread.each do |name, (path, state)|
629
+ dumps[name] ||= { tables: {}, total_tables: 0, note: "#{relative_dump_path(path)} #{state}, so its tables are not listed" }
630
+ .merge(database_files(path, migrate_dir_for_dump(path)), state == NO_TABLES ? {} : { unread: true })
631
+ end
632
+ names = app_databases.names
633
+ dumps.sort_by { |name, _| names.index(name) || names.size }.to_h
621
634
  end
622
635
 
623
636
  # Where a secondary database's schema lives, app-relative, so a rule
@@ -627,22 +640,29 @@ module RailsAiContext
627
640
  { dump: (relative_dump_path(dump) if dump), migrations_paths: (dirs if dirs.any?) }.compact
628
641
  end
629
642
 
630
- # A database whose dump is not written yet answers from its own migrations_paths.
631
- def replay_secondary_migrations(dumps)
632
- RailsAiContext::DatabaseYml.task_secondaries(app.root).each do |name, entry|
633
- next if dumps.key?(name)
643
+ # A database whose dump gives no tables answers from its own migrations_paths.
644
+ def replay_secondary_migrations(dumps, unread)
645
+ app_databases.secondaries.each do |db|
646
+ name = db.name
647
+ dirs = db.migrate_dirs
648
+ next if dumps.key?(name) || dirs.empty? || dirs == [ migrations_dir ]
634
649
 
635
- dirs = MigrationReplay.configured_dirs(app.root.to_s, entry) or next
636
650
  pk_type = SchemaConventions.implicit_pk_type(app.root.to_s, database: name)
637
651
  tables = MigrationReplay.tables(dirs, pk_type: pk_type, root: app.root.to_s,
638
652
  adapter: SchemaConventions.database_adapter_for(app.root.to_s, name))
639
653
  next if tables.empty?
640
654
 
641
655
  tables.each_value { |table| SchemaConventions.mark_primary_key(table) }
642
- _, dump = SchemaDumpPath.secondaries(app.root)[name]
656
+ dump = db.dump&.path
657
+ path, state = unread[name]
658
+ said = if path then "#{relative_dump_path(path)} #{state}"
659
+ elsif dump && File.exist?(dump) && secondary_dump(dump).nil?
660
+ "#{relative_dump_path(dump)} is read as the #{app_databases.primary.name} database's dump"
661
+ else "no #{dump ? relative_dump_path(dump) : "#{name}_schema.rb"}"
662
+ end
643
663
  dumps[name] = {
644
664
  adapter: "static_parse", tables: tables, total_tables: SchemaConventions.table_count(tables),
645
- note: "Reconstructed from the migrations in #{dirs.map { |dir| relative_dump_path(dir) }.join(', ')} (#{connection_state}, no #{dump ? relative_dump_path(dump) : "#{name}_schema.rb"})"
665
+ note: "Reconstructed from the migrations in #{dirs.map { |dir| relative_dump_path(dir) }.join(', ')} (#{connection_state}, #{said})"
646
666
  }.merge(database_files(dump, dirs))
647
667
  end
648
668
  dumps
@@ -721,8 +741,8 @@ module RailsAiContext
721
741
  end
722
742
 
723
743
  def dump_lookup(database)
724
- adapter = SchemaConventions.database_adapter_for(app.root.to_s, database || "primary")
725
- return if adapter && !adapter.start_with?("postg")
744
+ adapter = SchemaConventions.database_adapter_for(app.root.to_s, database)
745
+ return if adapter && !RailsAiContext::DatabaseFamily.of(adapter).postgresql?
726
746
 
727
747
  _, format, path = database ? secondary_dump_files.find { |name, _, _| name == database } : [ nil, *present_dump ]
728
748
  return unless path && File.exist?(path)
@@ -878,7 +898,7 @@ module RailsAiContext
878
898
  def parse_migrations
879
899
  pk_type = SchemaConventions.implicit_pk_type(app.root.to_s)
880
900
  replayed = MigrationReplay.replayed(migrations_dirs, pk_type: pk_type, root: app.root.to_s,
881
- adapter: SchemaConventions.database_adapter_for(app.root.to_s, "primary"))
901
+ adapter: SchemaConventions.database_adapter_for(app.root.to_s, nil))
882
902
  tables = replayed.tables
883
903
  tables.each_value { |table| SchemaConventions.mark_primary_key(table) }
884
904