rails-ai-context 5.35.0 → 5.35.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1024 -0
- data/CONTEXT.md +37 -7
- data/CONTRIBUTING.md +30 -1
- data/README.md +9 -9
- data/SECURITY.md +2 -2
- data/docs/ARCHITECTURE.md +12 -4
- data/docs/CLI.md +44 -15
- data/docs/COMPATIBILITY.md +34 -8
- data/docs/CONFIGURATION.md +28 -5
- data/docs/CUSTOM_TOOLS.md +8 -2
- data/docs/FAQ.md +5 -5
- data/docs/GUIDE.md +33 -65
- data/docs/INTROSPECTORS.md +4 -4
- data/docs/QUICKSTART.md +1 -1
- data/docs/RECIPES.md +5 -5
- data/docs/RELEASING.md +3 -1
- data/docs/SECURITY.md +255 -74
- data/docs/SETUP.md +1 -1
- data/docs/STANDALONE.md +2 -2
- data/docs/TOOLS.md +151 -90
- data/docs/TROUBLESHOOTING.md +123 -8
- data/docs/_config.yml +1 -1
- data/docs/adr/0008-query-runs-only-where-it-can-guard.md +38 -0
- data/docs/adr/0009-dbhub-shapes-in-ruby.md +26 -0
- data/docs/adr/0010-one-config-tool-with-sections.md +32 -0
- data/docs/index.md +2 -2
- data/exe/rails-ai-context +24 -9
- data/lib/generators/rails_ai_context/install/install_generator.rb +2 -0
- data/lib/rails_ai_context/app_databases.rb +377 -0
- data/lib/rails_ai_context/app_kind.rb +2 -0
- data/lib/rails_ai_context/cli/app_root.rb +1 -0
- data/lib/rails_ai_context/cli/tool_runner.rb +110 -99
- data/lib/rails_ai_context/configuration.rb +15 -8
- data/lib/rails_ai_context/database_family/mysql_runtime.rb +170 -0
- data/lib/rails_ai_context/database_family/postgresql_plan.rb +124 -0
- data/lib/rails_ai_context/database_family/postgresql_runtime.rb +211 -0
- data/lib/rails_ai_context/database_family/sqlite_runtime.rb +391 -0
- data/lib/rails_ai_context/database_family.rb +282 -0
- data/lib/rails_ai_context/database_file.rb +61 -7
- data/lib/rails_ai_context/database_yml.rb +7 -29
- data/lib/rails_ai_context/doctor.rb +15 -16
- data/lib/rails_ai_context/instrumentation.rb +1 -1
- data/lib/rails_ai_context/introspectors/connection_pool_introspector.rb +8 -23
- data/lib/rails_ai_context/introspectors/database_stats_introspector.rb +6 -70
- data/lib/rails_ai_context/introspectors/declared_constant.rb +19 -7
- data/lib/rails_ai_context/introspectors/env_introspector.rb +1 -1
- data/lib/rails_ai_context/introspectors/env_references.rb +1 -1
- data/lib/rails_ai_context/introspectors/job_introspector.rb +7 -7
- data/lib/rails_ai_context/introspectors/migration_introspector.rb +8 -11
- data/lib/rails_ai_context/introspectors/migration_replay/statements.rb +9 -9
- data/lib/rails_ai_context/introspectors/model_introspector.rb +33 -39
- data/lib/rails_ai_context/introspectors/multi_database_introspector.rb +33 -30
- data/lib/rails_ai_context/introspectors/node_source.rb +2 -0
- data/lib/rails_ai_context/introspectors/performance_introspector.rb +80 -24
- data/lib/rails_ai_context/introspectors/pg_partitions.rb +1 -4
- data/lib/rails_ai_context/introspectors/schema_conventions.rb +6 -8
- data/lib/rails_ai_context/introspectors/schema_dump_path.rb +12 -14
- data/lib/rails_ai_context/introspectors/schema_introspector.rb +47 -27
- data/lib/rails_ai_context/introspectors/schema_reader.rb +1 -1
- data/lib/rails_ai_context/introspectors/source_scan.rb +39 -4
- data/lib/rails_ai_context/migration_status.rb +18 -46
- data/lib/rails_ai_context/payload.rb +48 -1
- data/lib/rails_ai_context/pending_migrations.rb +3 -4
- data/lib/rails_ai_context/run_cache.rb +5 -0
- data/lib/rails_ai_context/schema_adapter.rb +13 -18
- data/lib/rails_ai_context/secret_columns.rb +506 -0
- data/lib/rails_ai_context/serializers/claude_rules_serializer.rb +2 -2
- data/lib/rails_ai_context/serializers/markdown_serializer.rb +14 -10
- data/lib/rails_ai_context/serializers/section_facts.rb +12 -2
- data/lib/rails_ai_context/serializers/stack_overview_helper.rb +5 -2
- data/lib/rails_ai_context/serializers/tool_guide_helper.rb +35 -7
- data/lib/rails_ai_context/server.rb +143 -29
- data/lib/rails_ai_context/sql_gate.rb +1007 -0
- data/lib/rails_ai_context/tasks/rails_ai_context.rake +1 -2
- data/lib/rails_ai_context/test_helper.rb +3 -1
- data/lib/rails_ai_context/tools/analyze_feature.rb +11 -7
- data/lib/rails_ai_context/tools/base_tool.rb +71 -48
- data/lib/rails_ai_context/tools/dependency_graph.rb +1 -1
- data/lib/rails_ai_context/tools/diagnose.rb +105 -42
- data/lib/rails_ai_context/tools/generate_test.rb +4 -6
- data/lib/rails_ai_context/tools/get_active_support.rb +1 -6
- data/lib/rails_ai_context/tools/get_api.rb +1 -6
- data/lib/rails_ai_context/tools/get_autoload.rb +1 -6
- data/lib/rails_ai_context/tools/get_callbacks.rb +5 -5
- data/lib/rails_ai_context/tools/get_component_catalog.rb +6 -7
- data/lib/rails_ai_context/tools/get_concern.rb +6 -6
- data/lib/rails_ai_context/tools/get_config.rb +189 -12
- data/lib/rails_ai_context/tools/get_context.rb +111 -105
- data/lib/rails_ai_context/tools/get_controllers.rb +8 -5
- data/lib/rails_ai_context/tools/get_conventions.rb +1 -6
- data/lib/rails_ai_context/tools/get_engines.rb +1 -6
- data/lib/rails_ai_context/tools/get_env.rb +1 -6
- data/lib/rails_ai_context/tools/get_env_config.rb +2 -8
- data/lib/rails_ai_context/tools/get_frontend_stack.rb +1 -6
- data/lib/rails_ai_context/tools/get_gems.rb +1 -6
- data/lib/rails_ai_context/tools/get_helper_methods.rb +4 -5
- data/lib/rails_ai_context/tools/get_i18n.rb +1 -7
- data/lib/rails_ai_context/tools/get_job_pattern.rb +1 -1
- data/lib/rails_ai_context/tools/get_mailers.rb +1 -1
- data/lib/rails_ai_context/tools/get_model_details.rb +5 -8
- data/lib/rails_ai_context/tools/get_partial_interface.rb +3 -4
- data/lib/rails_ai_context/tools/get_schema.rb +10 -8
- data/lib/rails_ai_context/tools/get_service_pattern.rb +60 -24
- data/lib/rails_ai_context/tools/get_turbo_map.rb +1 -1
- data/lib/rails_ai_context/tools/migration_advisor.rb +47 -32
- data/lib/rails_ai_context/tools/onboard.rb +24 -24
- data/lib/rails_ai_context/tools/performance_check.rb +15 -2
- data/lib/rails_ai_context/tools/query.rb +142 -1356
- data/lib/rails_ai_context/tools/read_logs.rb +1 -1
- data/lib/rails_ai_context/tools/review_changes.rb +6 -7
- data/lib/rails_ai_context/tools/runtime_info.rb +125 -88
- data/lib/rails_ai_context/tools/safe_call.rb +70 -28
- data/lib/rails_ai_context/tools/search_code.rb +0 -4
- data/lib/rails_ai_context/tools/search_docs.rb +3 -1
- data/lib/rails_ai_context/tools/security_scan.rb +9 -5
- data/lib/rails_ai_context/tools/session_context.rb +18 -12
- data/lib/rails_ai_context/tools/validate.rb +1 -0
- data/lib/rails_ai_context/tools/validate_semantics.rb +5 -5
- data/lib/rails_ai_context/version.rb +1 -1
- data/lib/rails_ai_context.rb +4 -7
- data/server.json +2 -2
- metadata +14 -3
|
@@ -224,9 +224,10 @@ module RailsAiContext
|
|
|
224
224
|
|
|
225
225
|
next if mixin_path?(record.path_name.underscore, source)
|
|
226
226
|
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
227
|
+
path_name = SourceScan.concerns_name(record.path_name, source)
|
|
228
|
+
declarations = DeclaredConstant.declarations(source, path_name: path_name)
|
|
229
|
+
declaration = DeclaredConstant.declaration_for(declarations, path_name)
|
|
230
|
+
class_name = declaration&.name || path_name
|
|
230
231
|
# Two files can declare one class (an app reopening a model to add methods);
|
|
231
232
|
# the one that names a superclass defines the model.
|
|
232
233
|
previous = found[class_name]
|
|
@@ -245,7 +246,7 @@ module RailsAiContext
|
|
|
245
246
|
rescue => e
|
|
246
247
|
# The file is known here whatever failed, and a consumer with none
|
|
247
248
|
# derives app/models/<name>.rb, which a pack model does not have.
|
|
248
|
-
found[record.path_name] = { error: e.message, file: record.file }.compact
|
|
249
|
+
found[SourceScan.concerns_name(record.path_name, source.to_s)] = { error: e.message, file: record.file }.compact
|
|
249
250
|
end
|
|
250
251
|
end
|
|
251
252
|
end
|
|
@@ -391,10 +392,10 @@ module RailsAiContext
|
|
|
391
392
|
# The tables every other database's dump holds; nil when one of them is not on disk to read.
|
|
392
393
|
def secondary_dump_tables
|
|
393
394
|
RunCache.fetch([ :secondary_dump_tables, app.root.to_s ]) do
|
|
394
|
-
|
|
395
|
-
break nil unless File.exist?(path)
|
|
395
|
+
AppDatabases.for(app.root).reject(&:primary).filter_map(&:dump).each_with_object(Set.new) do |dump, tables|
|
|
396
|
+
break nil unless File.exist?(dump.path)
|
|
396
397
|
|
|
397
|
-
found = format == :ruby ? SchemaReader.new(path).tables : StructureSqlReader.parse(SafeFile.read(path).to_s)[:tables]
|
|
398
|
+
found = dump.format == :ruby ? SchemaReader.new(dump.path).tables : StructureSqlReader.parse(SafeFile.read(dump.path).to_s)[:tables]
|
|
398
399
|
tables.merge(found.keys)
|
|
399
400
|
end
|
|
400
401
|
end
|
|
@@ -550,16 +551,12 @@ module RailsAiContext
|
|
|
550
551
|
source.match?(/^[^\S\n]*primary_abstract_class\b/)
|
|
551
552
|
end
|
|
552
553
|
|
|
553
|
-
#
|
|
554
|
-
#
|
|
555
|
-
# concerns/ is an ordinary namespace - OpenProject fills one with mixins,
|
|
556
|
-
# but a class declared there is a model like any other.
|
|
554
|
+
# A file under any concerns/ that is not named for a class. One that is
|
|
555
|
+
# may be a model: Whitehall keeps one in app/models/concerns itself.
|
|
557
556
|
def mixin_path?(relative, source)
|
|
558
|
-
|
|
559
|
-
return false unless segments.include?("concerns")
|
|
560
|
-
return true if segments.first == "concerns"
|
|
557
|
+
return false unless relative.split("/").include?("concerns")
|
|
561
558
|
|
|
562
|
-
|
|
559
|
+
SourceScan.concern_class(source, SourceScan.concerns_name(relative.camelize, source)).nil?
|
|
563
560
|
end
|
|
564
561
|
|
|
565
562
|
# A class whose file is inside an installed gem (PaperTrail::Version) is
|
|
@@ -596,15 +593,17 @@ module RailsAiContext
|
|
|
596
593
|
|
|
597
594
|
known = models.map(&:name).to_set
|
|
598
595
|
@outside_model_dirs = Set.new
|
|
599
|
-
#
|
|
600
|
-
#
|
|
601
|
-
# out. A top-level `app/models/concerns` is an autoload root instead,
|
|
602
|
-
# so its files declare no `Concerns::` prefix and that path name never
|
|
603
|
-
# constantizes.
|
|
596
|
+
# A nested concerns directory is a namespace, so a class under one is a model;
|
|
597
|
+
# under the top-level one only a file that declares a class is read on.
|
|
604
598
|
SourceScan.model_paths(app.root, base_model: model_base_check).each do |record|
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
599
|
+
path_name = record.path_name
|
|
600
|
+
if path_name.start_with?("Concerns::")
|
|
601
|
+
source = model_source(record.path).to_s
|
|
602
|
+
path_name = SourceScan.concerns_name(path_name, source)
|
|
603
|
+
end
|
|
604
|
+
next if known.include?(path_name)
|
|
605
|
+
next if config.excluded_models.include?(path_name)
|
|
606
|
+
next if path_name != record.path_name && mixin_path?(record.path_name.underscore, source)
|
|
608
607
|
|
|
609
608
|
# The path does not name the class: an app inflection only changes
|
|
610
609
|
# case, so activitypub/activity.rb camelizes to a constant the app
|
|
@@ -612,7 +611,7 @@ module RailsAiContext
|
|
|
612
611
|
# load. A root pushed under a namespace declares a prefix the path
|
|
613
612
|
# lacks. Read only where the camelized name is not already loaded, so
|
|
614
613
|
# a booted run does not parse every model file to learn nothing.
|
|
615
|
-
class_name = DeclaredConstant.named(model_source(record.path).to_s,
|
|
614
|
+
class_name = DeclaredConstant.named(model_source(record.path).to_s, path_name)
|
|
616
615
|
next if known.include?(class_name)
|
|
617
616
|
next if config.excluded_models.include?(class_name)
|
|
618
617
|
|
|
@@ -630,10 +629,9 @@ module RailsAiContext
|
|
|
630
629
|
next unless klass < ActiveRecord::Base && !klass.abstract_class?
|
|
631
630
|
models << klass
|
|
632
631
|
known << class_name
|
|
633
|
-
rescue
|
|
634
|
-
# A
|
|
635
|
-
#
|
|
636
|
-
# lacks is not the same answer as no such model.
|
|
632
|
+
rescue StandardError, ScriptError => e
|
|
633
|
+
# A file that raises as it loads costs itself alone, and its name is recorded:
|
|
634
|
+
# a file that exists for a class reflection lacks is no missing model.
|
|
637
635
|
@unloadable[class_name] = e.message.to_s.lines.first.to_s.strip
|
|
638
636
|
@outside_model_dirs << class_name unless model_dir_file?(record.path)
|
|
639
637
|
end
|
|
@@ -988,7 +986,7 @@ module RailsAiContext
|
|
|
988
986
|
bases = abstract_bases(model)
|
|
989
987
|
return bases unless model.superclass == ActiveRecord::Base
|
|
990
988
|
|
|
991
|
-
bases | ActiveRecord::Base.subclasses.select
|
|
989
|
+
bases | DeclaredConstant.carried(ActiveRecord::Base.subclasses).select(&:abstract_class?)
|
|
992
990
|
end
|
|
993
991
|
|
|
994
992
|
def every_model_modules
|
|
@@ -1138,13 +1136,7 @@ module RailsAiContext
|
|
|
1138
1136
|
|
|
1139
1137
|
return nil unless has_type_column
|
|
1140
1138
|
|
|
1141
|
-
children =
|
|
1142
|
-
model.descendants.map(&:name).compact.sort
|
|
1143
|
-
elsif model.respond_to?(:subclasses)
|
|
1144
|
-
model.subclasses.map(&:name).compact.sort
|
|
1145
|
-
else
|
|
1146
|
-
[]
|
|
1147
|
-
end
|
|
1139
|
+
children = DeclaredConstant.carried(model.descendants).map(&:name).sort
|
|
1148
1140
|
|
|
1149
1141
|
parent = model.superclass
|
|
1150
1142
|
sti_parent = if parent && parent != ActiveRecord::Base &&
|
|
@@ -2245,15 +2237,17 @@ module RailsAiContext
|
|
|
2245
2237
|
source = model_source(path)
|
|
2246
2238
|
next if source.nil? || mixin_path?(relative, source) || abstract_class?(source)
|
|
2247
2239
|
|
|
2240
|
+
path_name = SourceScan.concerns_name(relative.camelize, source)
|
|
2241
|
+
|
|
2248
2242
|
# The static tier never loads the app's inflector, so camelizing alone
|
|
2249
2243
|
# would invent `Activitypub::` for an app that declares `ActivityPub::`.
|
|
2250
|
-
class_name = DeclaredConstant.resolve(source,
|
|
2244
|
+
class_name = DeclaredConstant.resolve(source, path_name)
|
|
2251
2245
|
next if found.key?(class_name) || config.excluded_models.include?(class_name)
|
|
2252
2246
|
|
|
2253
|
-
declaration = DeclaredConstant.declaration_for(DeclaredConstant.declarations(source, path_name:
|
|
2247
|
+
declaration = DeclaredConstant.declaration_for(DeclaredConstant.declarations(source, path_name: path_name), class_name)
|
|
2254
2248
|
found[class_name] = { path: path, source: source, superclass: declaration&.superclass, nesting: declaration&.nesting }
|
|
2255
2249
|
rescue => e
|
|
2256
|
-
found[relative.camelize] ||= { error: e.message }
|
|
2250
|
+
found[path_name || relative.camelize] ||= { error: e.message }
|
|
2257
2251
|
end
|
|
2258
2252
|
end
|
|
2259
2253
|
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "uri"
|
|
4
|
+
|
|
3
5
|
module RailsAiContext
|
|
4
6
|
module Introspectors
|
|
5
7
|
# Discovers multi-database configuration: multiple databases, replicas,
|
|
@@ -13,22 +15,37 @@ module RailsAiContext
|
|
|
13
15
|
dbs = discover_databases
|
|
14
16
|
{
|
|
15
17
|
databases: dbs,
|
|
16
|
-
replicas: dbs.select { |d| d[:replica] }.map { |d|
|
|
18
|
+
replicas: dbs.select { |d| d[:replica] }.map { |d| d.slice(:name, :adapter, :environments) },
|
|
17
19
|
sharding: detect_sharding,
|
|
18
20
|
model_connections: detect_model_connections,
|
|
19
21
|
multi_db: dbs.size > 1
|
|
20
|
-
}.merge(
|
|
22
|
+
}.merge(from_configuration)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# What the section says of each database by name, read from configuration alone.
|
|
26
|
+
def listing
|
|
27
|
+
{ databases: discover_databases }.merge(from_configuration)
|
|
21
28
|
end
|
|
22
29
|
|
|
23
30
|
private
|
|
24
31
|
|
|
32
|
+
def from_configuration
|
|
33
|
+
{ example_adapters: example_adapters, primary_name: primary_name }.compact
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# The primary's database.yml name where it is not "primary": a generated file tells it from a database that is.
|
|
37
|
+
def primary_name
|
|
38
|
+
name = RailsAiContext::AppDatabases.for(root).primary&.name.to_s
|
|
39
|
+
name unless name.empty? || name == "primary"
|
|
40
|
+
rescue => e
|
|
41
|
+
RailsAiContext.debug_fail(e, nil, label: "primary_name")
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Every app database; one the running environment does not configure names where it is configured.
|
|
25
45
|
def discover_databases
|
|
26
|
-
|
|
27
|
-
# A boot that failed leaves the handlers empty without raising, and an
|
|
28
|
-
# app that declares two databases is not an app with none.
|
|
29
|
-
booted.any? ? booted : file_databases
|
|
46
|
+
RailsAiContext::AppDatabases.for(root).map { |db| database_entry(db) }
|
|
30
47
|
rescue => e
|
|
31
|
-
RailsAiContext.debug_fail(e,
|
|
48
|
+
RailsAiContext.debug_fail(e, [], label: "discover_databases")
|
|
32
49
|
end
|
|
33
50
|
|
|
34
51
|
# Adapters in the example database files of an app that commits none, read by line since
|
|
@@ -113,30 +130,16 @@ module RailsAiContext
|
|
|
113
130
|
end
|
|
114
131
|
end
|
|
115
132
|
|
|
116
|
-
def
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
133
|
+
def database_entry(db)
|
|
134
|
+
info = { name: db.name, adapter: db.adapter }
|
|
135
|
+
info[:adapter_default] = true if db.adapter_default
|
|
136
|
+
database = db.config.database if db.config.respond_to?(:database)
|
|
137
|
+
info[:database] = anonymize_db_name(database) if database
|
|
138
|
+
info[:replica] = true if db.replica
|
|
139
|
+
# Configured only in another environment, or only a committed dump names it.
|
|
140
|
+
unless db.here
|
|
141
|
+
db.environments.any? ? info[:environments] = db.environments : info[:dump_only] = true
|
|
124
142
|
end
|
|
125
|
-
end
|
|
126
|
-
|
|
127
|
-
def file_databases
|
|
128
|
-
databases = RailsAiContext::DatabaseYml.databases(root)
|
|
129
|
-
# Rails builds the primary from DATABASE_URL when the file has no entry for the env.
|
|
130
|
-
return ENV["DATABASE_URL"].to_s.empty? ? [] : [ database_entry("primary", {}) ] if databases.empty?
|
|
131
|
-
|
|
132
|
-
databases.map { |name, entry| database_entry(name, entry) }
|
|
133
|
-
end
|
|
134
|
-
|
|
135
|
-
def database_entry(name, entry)
|
|
136
|
-
adapter, from_default = RailsAiContext::DatabaseYml.adapter(name, entry)
|
|
137
|
-
info = { name: name.to_s, adapter: adapter }
|
|
138
|
-
info[:adapter_default] = true if from_default
|
|
139
|
-
info[:replica] = true if entry["replica"] == true
|
|
140
143
|
info
|
|
141
144
|
end
|
|
142
145
|
|
|
@@ -18,27 +18,68 @@ module RailsAiContext
|
|
|
18
18
|
attr_writer :context
|
|
19
19
|
|
|
20
20
|
def call
|
|
21
|
-
schema_data = load_schema_data
|
|
22
21
|
model_data = load_model_data
|
|
22
|
+
schemas, unchecked = database_schemas
|
|
23
|
+
counter_cache = []
|
|
24
|
+
fk_indexes = []
|
|
25
|
+
schemas.each do |name, tables, foreign_keys|
|
|
26
|
+
tag = ->(finding) { name ? finding.merge(database: name) : finding }
|
|
27
|
+
counter_cache.concat(detect_missing_counter_cache(model_data, tables).map(&tag))
|
|
28
|
+
fk_indexes.concat(detect_missing_fk_indexes(tables, model_data, foreign_keys).map(&tag))
|
|
29
|
+
end
|
|
23
30
|
|
|
24
31
|
{
|
|
25
32
|
n_plus_one_risks: detect_n_plus_one,
|
|
26
|
-
missing_counter_cache:
|
|
27
|
-
missing_fk_indexes:
|
|
33
|
+
missing_counter_cache: counter_cache.uniq { |f| f.except(:database) },
|
|
34
|
+
missing_fk_indexes: fk_indexes,
|
|
28
35
|
model_all_in_controllers: detect_model_all_in_controllers(model_data),
|
|
29
36
|
eager_load_candidates: detect_eager_load_candidates(model_data),
|
|
30
37
|
summary: nil # populated below
|
|
31
|
-
}.tap
|
|
38
|
+
}.tap do |result|
|
|
39
|
+
result[:unchecked_databases] = unchecked if unchecked.any?
|
|
40
|
+
result[:summary] = build_summary(result)
|
|
41
|
+
end
|
|
32
42
|
end
|
|
33
43
|
|
|
34
44
|
private
|
|
35
45
|
|
|
36
|
-
|
|
37
|
-
|
|
46
|
+
# [[database name (nil for the primary), tables, foreign keys]], and { name:, reason: } for each database not read.
|
|
47
|
+
def database_schemas
|
|
48
|
+
reader = SchemaReader.for(root)
|
|
49
|
+
schemas = [ [ nil, reader.tables, reader.foreign_keys ] ]
|
|
50
|
+
unchecked = []
|
|
51
|
+
databases = AppDatabases.for(root)
|
|
52
|
+
# Rails names one file for the first database and the one named primary. It is read once, above.
|
|
53
|
+
shared = databases.primary&.dump&.path
|
|
54
|
+
databases.reject { |db| db.primary || db.replica || (shared && db.dump&.path == shared) }.each do |db|
|
|
55
|
+
read = read_dump(db)
|
|
56
|
+
read[:reason] ? unchecked << { name: db.name, reason: read[:reason] } : schemas << [ db.name, read[:tables], read[:foreign_keys] ]
|
|
57
|
+
end
|
|
58
|
+
[ schemas, unchecked ]
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# { tables:, foreign_keys: } from the database's dump, or { reason: } saying why there is none to read.
|
|
62
|
+
def read_dump(db)
|
|
63
|
+
return { reason: no_dump_reason(db) } unless db.dump
|
|
64
|
+
|
|
65
|
+
path = db.dump.path
|
|
66
|
+
shown = path.delete_prefix("#{root.to_s.chomp('/')}/")
|
|
67
|
+
return { reason: "#{shown} is not on disk" } unless File.exist?(path)
|
|
68
|
+
|
|
69
|
+
content = SafeFile.read(path, max_size: RailsAiContext.configuration.max_schema_file_size)
|
|
70
|
+
return { reason: "#{shown} is over max_schema_file_size" } unless content
|
|
71
|
+
return SchemaReader.new(path).then { |schema| { tables: schema.tables, foreign_keys: schema.foreign_keys } } if db.dump.format == :ruby
|
|
72
|
+
|
|
73
|
+
tables = StructureSqlReader.parse(content)[:tables]
|
|
74
|
+
{ tables: tables,
|
|
75
|
+
foreign_keys: tables.values.flat_map { |table| Array(table[:foreign_keys]).map { |fk| { from: fk[:from_table], to: fk[:to_table], column: fk[:column] } } } }
|
|
38
76
|
end
|
|
39
77
|
|
|
40
|
-
def
|
|
41
|
-
|
|
78
|
+
def no_dump_reason(db)
|
|
79
|
+
return "its database.yml entry turns `database_tasks` off, so Rails writes it no dump" unless db.tasks
|
|
80
|
+
return "its database.yml entry turns `schema_dump` off, so Rails writes it no dump" unless db.settings["schema_dump"]
|
|
81
|
+
|
|
82
|
+
"`schema_dump` in its database.yml entry names no file this check reads"
|
|
42
83
|
end
|
|
43
84
|
|
|
44
85
|
# The models section when the run has one; otherwise the model tier is
|
|
@@ -77,7 +118,7 @@ module RailsAiContext
|
|
|
77
118
|
ast = SourceIntrospector.walk_source(record.source, {
|
|
78
119
|
classes: Listeners::ClassDefinitionListener,
|
|
79
120
|
associations: Listeners::AssociationsListener,
|
|
80
|
-
|
|
121
|
+
column_macros: COLUMN_MACRO_LISTENER,
|
|
81
122
|
mixins: Listeners::MixinsListener,
|
|
82
123
|
counter_culture: -> { Listeners::GenericMacroListener.new(:counter_culture) }
|
|
83
124
|
})
|
|
@@ -106,7 +147,7 @@ module RailsAiContext
|
|
|
106
147
|
table_name: model_table(class_name),
|
|
107
148
|
has_many: has_many,
|
|
108
149
|
belongs_to: belongs_to,
|
|
109
|
-
|
|
150
|
+
macro_columns: macro_columns(ast, class_name),
|
|
110
151
|
counter_cultures: ast[:counter_culture].map { |c| counter_culture_record(c) }
|
|
111
152
|
}
|
|
112
153
|
rescue => e
|
|
@@ -114,24 +155,39 @@ module RailsAiContext
|
|
|
114
155
|
end
|
|
115
156
|
end
|
|
116
157
|
|
|
117
|
-
#
|
|
118
|
-
# ponytail: a fixed list; a gem outside it declares keys the static walk cannot see.
|
|
158
|
+
# Tree gem macros, which declare a parent key, with the option that renames it.
|
|
119
159
|
TREE_MACROS = { acts_as_tree: :foreign_key, acts_as_nested_set: :parent_column,
|
|
120
160
|
has_closure_tree: :parent_column_name }.freeze
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
#
|
|
161
|
+
# paper_trail adds `has_many :versions, as: :item` when the app boots.
|
|
162
|
+
VERSIONS_MACRO = :has_paper_trail
|
|
163
|
+
# Gem macros that declare a column.
|
|
164
|
+
# ponytail: a fixed list; a gem outside it declares columns the static walk cannot see.
|
|
165
|
+
COLUMN_MACROS = [ *TREE_MACROS.keys, VERSIONS_MACRO ].freeze
|
|
166
|
+
COLUMN_MACRO_LISTENER = -> { Listeners::GenericMacroListener.new(*COLUMN_MACROS) }
|
|
167
|
+
COLUMN_MACRO_CONCERN_LISTENERS = { column_macros: COLUMN_MACRO_LISTENER, mixins: Listeners::MixinsListener }.freeze
|
|
168
|
+
|
|
169
|
+
# The [table, column] a gem macro declares, in the model or its concerns.
|
|
125
170
|
# The models section cannot carry it statically: the macro is a gem's.
|
|
126
|
-
def
|
|
127
|
-
collected = ConcernMacros.collect(root, ast[:mixins], keys: %i[
|
|
171
|
+
def macro_columns(ast, class_name)
|
|
172
|
+
collected = ConcernMacros.collect(root, ast[:mixins], keys: %i[column_macros], prefer: "model",
|
|
128
173
|
within: class_name, cache: (@concern_cache ||= {}),
|
|
129
|
-
listeners:
|
|
130
|
-
(ast[:
|
|
131
|
-
|
|
174
|
+
listeners: COLUMN_MACRO_CONCERN_LISTENERS).collected
|
|
175
|
+
(ast[:column_macros] + Array(collected[:column_macros])).map do |macro|
|
|
176
|
+
options = macro[:options] || {}
|
|
177
|
+
next [ versions_table(options, class_name), "item_id" ] if macro[:macro] == VERSIONS_MACRO
|
|
178
|
+
|
|
179
|
+
[ model_table(class_name), (options[TREE_MACROS[macro[:macro]]] || "parent_id").to_s ]
|
|
132
180
|
end
|
|
133
181
|
end
|
|
134
182
|
|
|
183
|
+
# PaperTrail::Version reads `versions`; `versions: { class_name: }`, or the older `class_name:`, names another class.
|
|
184
|
+
def versions_table(options, owner)
|
|
185
|
+
written = options[:versions].is_a?(Hash) ? options[:versions][:class_name] : options[:class_name]
|
|
186
|
+
return "versions" unless written.is_a?(String)
|
|
187
|
+
|
|
188
|
+
table_of(TableName.resolve_class(written, owner) { |candidate| candidate if resolved_models.key?(candidate) } || written, resolved_models)
|
|
189
|
+
end
|
|
190
|
+
|
|
135
191
|
# The scan reads the run models section, so an association a concern or a
|
|
136
192
|
# base declares is one it can follow.
|
|
137
193
|
def detect_n_plus_one
|
|
@@ -271,9 +327,9 @@ module RailsAiContext
|
|
|
271
327
|
end
|
|
272
328
|
end
|
|
273
329
|
|
|
274
|
-
def
|
|
330
|
+
def macro_declared_columns(model_data)
|
|
275
331
|
Array(model_data).each_with_object(Set.new) do |model, found|
|
|
276
|
-
Array(model[:
|
|
332
|
+
Array(model[:macro_columns]).each { |column| found << column }
|
|
277
333
|
end
|
|
278
334
|
end
|
|
279
335
|
|
|
@@ -285,7 +341,7 @@ module RailsAiContext
|
|
|
285
341
|
def detect_missing_fk_indexes(schema_data, model_data = [], foreign_keys = [])
|
|
286
342
|
missing = []
|
|
287
343
|
declared = foreign_key_columns(foreign_keys) | association_columns(resolved_models) |
|
|
288
|
-
|
|
344
|
+
macro_declared_columns(model_data)
|
|
289
345
|
|
|
290
346
|
schema_data.each do |table_name, table|
|
|
291
347
|
columns = table[:columns]
|
|
@@ -7,9 +7,6 @@ module RailsAiContext
|
|
|
7
7
|
module PgPartitions
|
|
8
8
|
module_function
|
|
9
9
|
|
|
10
|
-
# activerecord-postgis-adapter reports "PostGIS" for what is a PostgreSQL connection.
|
|
11
|
-
POSTGRES_ADAPTER = /postg/i
|
|
12
|
-
|
|
13
10
|
# Scoped the way connection.tables is.
|
|
14
11
|
NAMES_SQL = <<~SQL.squish.freeze
|
|
15
12
|
SELECT c.relname FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
@@ -19,7 +16,7 @@ module RailsAiContext
|
|
|
19
16
|
def names(conn)
|
|
20
17
|
# relispartition arrived with partitions in 10; querying it before then
|
|
21
18
|
# fails, and a failed query aborts any open transaction.
|
|
22
|
-
return [] unless conn.
|
|
19
|
+
return [] unless RailsAiContext::DatabaseFamily.of(conn).postgresql? && conn.database_version >= 10_00_00
|
|
23
20
|
|
|
24
21
|
conn.select_values(NAMES_SQL)
|
|
25
22
|
rescue => e
|
|
@@ -196,17 +196,15 @@ module RailsAiContext
|
|
|
196
196
|
# primary, sqlite queue) and each dump must be typed by its own. dump_path
|
|
197
197
|
# names a secondary database's dump, nil the primary's; database: names the database outright.
|
|
198
198
|
def implicit_pk_type(root, dump_path = nil, database: SchemaDumpPath.database_name(root, dump_path))
|
|
199
|
-
|
|
200
|
-
adapter&.start_with?("sqlite") ? "integer" : "bigint"
|
|
199
|
+
RailsAiContext::DatabaseFamily.of(database_adapter_for(root, database)).implicit_pk_type
|
|
201
200
|
end
|
|
202
201
|
|
|
203
|
-
#
|
|
204
|
-
#
|
|
205
|
-
# environment that does, else the running primary's.
|
|
202
|
+
# One app database's adapter, the primary's for no name. A database only another environment configures (Rails 8's queue
|
|
203
|
+
# outside production) takes that environment's; one only a dump names, the primary's.
|
|
206
204
|
def database_adapter_for(root, db_name)
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
205
|
+
databases = RailsAiContext::AppDatabases.for(root)
|
|
206
|
+
found = databases.of_dump_name(db_name)
|
|
207
|
+
found&.config ? found.adapter : databases.primary&.adapter
|
|
210
208
|
end
|
|
211
209
|
|
|
212
210
|
# A primary key as connection.primary_key gives it: the column's name, or
|
|
@@ -20,24 +20,22 @@ module RailsAiContext
|
|
|
20
20
|
end
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
-
#
|
|
24
|
-
# its
|
|
25
|
-
def
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
found[name] = dump if dump
|
|
32
|
-
end
|
|
33
|
-
end
|
|
23
|
+
# A secondary database's dump: the file schema_dump_path names (its schema_dump, else <name>_schema.rb or
|
|
24
|
+
# <name>_structure.sql in its format) when on disk, else a file under its name in either format. nil for none.
|
|
25
|
+
def secondary(root, name, config)
|
|
26
|
+
format = schema_format(root, config)
|
|
27
|
+
own = configured(root, config, format) if config.key?("schema_dump")
|
|
28
|
+
# Rails gives an entry named primary the unprefixed file wherever it stands.
|
|
29
|
+
named = [ format, *(FILE_NAMES.keys - [ format ]) ].map { |f| [ f, File.join(root, "db", "#{"#{name}_" unless name == "primary"}#{FILE_NAMES[f]}") ] }
|
|
30
|
+
[ own, *named ].compact.find { |_, path| File.exist?(path) } || (config.key?("schema_dump") ? own : named.first)
|
|
34
31
|
end
|
|
35
32
|
|
|
36
|
-
# The database a dump belongs to: the
|
|
33
|
+
# The database a dump belongs to: the app database whose dump it is, else the one its file name says.
|
|
34
|
+
# No dump is the primary's.
|
|
37
35
|
def database_name(root, dump_path)
|
|
38
|
-
return "primary" if dump_path.nil?
|
|
36
|
+
return RailsAiContext::AppDatabases.for(root).primary&.name || "primary" if dump_path.nil?
|
|
39
37
|
|
|
40
|
-
named =
|
|
38
|
+
named = RailsAiContext::AppDatabases.for(root).find_by_dump(dump_path)&.name
|
|
41
39
|
named || File.basename(dump_path.to_s).sub(/\.(rb|sql)\z/, "").sub(/_?(schema|structure)\z/, "").then { |name| name.empty? ? "primary" : name }
|
|
42
40
|
end
|
|
43
41
|
|
|
@@ -143,17 +143,20 @@ module RailsAiContext
|
|
|
143
143
|
RailsAiContext.debug_fail(e, "unknown", label: "adapter_name")
|
|
144
144
|
end
|
|
145
145
|
|
|
146
|
+
def family
|
|
147
|
+
RailsAiContext::DatabaseFamily.of(adapter_name)
|
|
148
|
+
end
|
|
149
|
+
|
|
146
150
|
def sqlite?
|
|
147
|
-
|
|
151
|
+
family.sqlite?
|
|
148
152
|
end
|
|
149
153
|
|
|
150
154
|
def postgres?
|
|
151
|
-
|
|
155
|
+
family.postgresql?
|
|
152
156
|
end
|
|
153
157
|
|
|
154
|
-
# Every MySQL adapter inherits mariadb? from AbstractMysqlAdapter.
|
|
155
158
|
def mysql?
|
|
156
|
-
connection.
|
|
159
|
+
RailsAiContext::DatabaseFamily.of(connection).mysql?
|
|
157
160
|
end
|
|
158
161
|
|
|
159
162
|
def connection
|
|
@@ -593,31 +596,41 @@ module RailsAiContext
|
|
|
593
596
|
@secondary_database_dumps ||= read_secondary_database_dumps
|
|
594
597
|
end
|
|
595
598
|
|
|
596
|
-
|
|
599
|
+
def app_databases
|
|
600
|
+
RailsAiContext::AppDatabases.for(app.root)
|
|
601
|
+
end
|
|
602
|
+
|
|
603
|
+
# [name, format, path] for each other database's dump on disk, in the app databases' order.
|
|
597
604
|
def secondary_dump_files
|
|
598
605
|
primary = dump_candidates.map(&:last)
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
kind, ext = file.split(".")
|
|
603
|
-
Dir.glob(File.join(app.root.to_s, "db", "*_#{kind}.#{ext}")).sort
|
|
604
|
-
.reject { |path| taken.include?(path) }
|
|
605
|
-
.map { |path| [ File.basename(path, ".#{ext}").delete_suffix("_#{kind}"), format, path ] }
|
|
606
|
+
app_databases.filter_map do |db|
|
|
607
|
+
path = db.dump&.path
|
|
608
|
+
[ db.name, db.dump.format, path ] if path && !db.primary && !primary.include?(path) && File.exist?(path)
|
|
606
609
|
end
|
|
607
|
-
(configured + globbed).reject { |_, _, path| primary.include?(path) || !File.exist?(path) }
|
|
608
610
|
end
|
|
609
611
|
|
|
612
|
+
NO_TABLES = "declares no tables"
|
|
613
|
+
|
|
614
|
+
# A dump too large to read, or holding no tables, keeps an entry that says so.
|
|
610
615
|
def read_secondary_database_dumps
|
|
616
|
+
unread = {}
|
|
611
617
|
dumps = secondary_dump_files.each_with_object({}) do |(name, format, path), found|
|
|
612
|
-
next if found.key?(name)
|
|
613
|
-
|
|
614
618
|
parsed = parse_dump(format, path)
|
|
615
|
-
|
|
619
|
+
if parsed[:tables].blank?
|
|
620
|
+
unread[name] = [ path, parsed[:error]&.delete_prefix("#{relative_dump_path(path)} ") || NO_TABLES ]
|
|
621
|
+
next
|
|
622
|
+
end
|
|
616
623
|
|
|
617
624
|
parsed[:note] = "Parsed from #{relative_dump_path(path)} (from committed dump, not a live connection)"
|
|
618
625
|
found[name] = parsed.merge(database_files(path, migrate_dir_for_dump(path)))
|
|
619
626
|
end
|
|
620
|
-
replay_secondary_migrations(dumps)
|
|
627
|
+
replay_secondary_migrations(dumps, unread)
|
|
628
|
+
unread.each do |name, (path, state)|
|
|
629
|
+
dumps[name] ||= { tables: {}, total_tables: 0, note: "#{relative_dump_path(path)} #{state}, so its tables are not listed" }
|
|
630
|
+
.merge(database_files(path, migrate_dir_for_dump(path)), state == NO_TABLES ? {} : { unread: true })
|
|
631
|
+
end
|
|
632
|
+
names = app_databases.names
|
|
633
|
+
dumps.sort_by { |name, _| names.index(name) || names.size }.to_h
|
|
621
634
|
end
|
|
622
635
|
|
|
623
636
|
# Where a secondary database's schema lives, app-relative, so a rule
|
|
@@ -627,22 +640,29 @@ module RailsAiContext
|
|
|
627
640
|
{ dump: (relative_dump_path(dump) if dump), migrations_paths: (dirs if dirs.any?) }.compact
|
|
628
641
|
end
|
|
629
642
|
|
|
630
|
-
# A database whose dump
|
|
631
|
-
def replay_secondary_migrations(dumps)
|
|
632
|
-
|
|
633
|
-
|
|
643
|
+
# A database whose dump gives no tables answers from its own migrations_paths.
|
|
644
|
+
def replay_secondary_migrations(dumps, unread)
|
|
645
|
+
app_databases.secondaries.each do |db|
|
|
646
|
+
name = db.name
|
|
647
|
+
dirs = db.migrate_dirs
|
|
648
|
+
next if dumps.key?(name) || dirs.empty? || dirs == [ migrations_dir ]
|
|
634
649
|
|
|
635
|
-
dirs = MigrationReplay.configured_dirs(app.root.to_s, entry) or next
|
|
636
650
|
pk_type = SchemaConventions.implicit_pk_type(app.root.to_s, database: name)
|
|
637
651
|
tables = MigrationReplay.tables(dirs, pk_type: pk_type, root: app.root.to_s,
|
|
638
652
|
adapter: SchemaConventions.database_adapter_for(app.root.to_s, name))
|
|
639
653
|
next if tables.empty?
|
|
640
654
|
|
|
641
655
|
tables.each_value { |table| SchemaConventions.mark_primary_key(table) }
|
|
642
|
-
|
|
656
|
+
dump = db.dump&.path
|
|
657
|
+
path, state = unread[name]
|
|
658
|
+
said = if path then "#{relative_dump_path(path)} #{state}"
|
|
659
|
+
elsif dump && File.exist?(dump) && secondary_dump(dump).nil?
|
|
660
|
+
"#{relative_dump_path(dump)} is read as the #{app_databases.primary.name} database's dump"
|
|
661
|
+
else "no #{dump ? relative_dump_path(dump) : "#{name}_schema.rb"}"
|
|
662
|
+
end
|
|
643
663
|
dumps[name] = {
|
|
644
664
|
adapter: "static_parse", tables: tables, total_tables: SchemaConventions.table_count(tables),
|
|
645
|
-
note: "Reconstructed from the migrations in #{dirs.map { |dir| relative_dump_path(dir) }.join(', ')} (#{connection_state},
|
|
665
|
+
note: "Reconstructed from the migrations in #{dirs.map { |dir| relative_dump_path(dir) }.join(', ')} (#{connection_state}, #{said})"
|
|
646
666
|
}.merge(database_files(dump, dirs))
|
|
647
667
|
end
|
|
648
668
|
dumps
|
|
@@ -721,8 +741,8 @@ module RailsAiContext
|
|
|
721
741
|
end
|
|
722
742
|
|
|
723
743
|
def dump_lookup(database)
|
|
724
|
-
adapter = SchemaConventions.database_adapter_for(app.root.to_s, database
|
|
725
|
-
return if adapter && !adapter.
|
|
744
|
+
adapter = SchemaConventions.database_adapter_for(app.root.to_s, database)
|
|
745
|
+
return if adapter && !RailsAiContext::DatabaseFamily.of(adapter).postgresql?
|
|
726
746
|
|
|
727
747
|
_, format, path = database ? secondary_dump_files.find { |name, _, _| name == database } : [ nil, *present_dump ]
|
|
728
748
|
return unless path && File.exist?(path)
|
|
@@ -878,7 +898,7 @@ module RailsAiContext
|
|
|
878
898
|
def parse_migrations
|
|
879
899
|
pk_type = SchemaConventions.implicit_pk_type(app.root.to_s)
|
|
880
900
|
replayed = MigrationReplay.replayed(migrations_dirs, pk_type: pk_type, root: app.root.to_s,
|
|
881
|
-
adapter: SchemaConventions.database_adapter_for(app.root.to_s,
|
|
901
|
+
adapter: SchemaConventions.database_adapter_for(app.root.to_s, nil))
|
|
882
902
|
tables = replayed.tables
|
|
883
903
|
tables.each_value { |table| SchemaConventions.mark_primary_key(table) }
|
|
884
904
|
|