rails-ai-context 5.35.0 → 5.35.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1024 -0
  3. data/CONTEXT.md +37 -7
  4. data/CONTRIBUTING.md +30 -1
  5. data/README.md +9 -9
  6. data/SECURITY.md +2 -2
  7. data/docs/ARCHITECTURE.md +12 -4
  8. data/docs/CLI.md +44 -15
  9. data/docs/COMPATIBILITY.md +34 -8
  10. data/docs/CONFIGURATION.md +28 -5
  11. data/docs/CUSTOM_TOOLS.md +8 -2
  12. data/docs/FAQ.md +5 -5
  13. data/docs/GUIDE.md +33 -65
  14. data/docs/INTROSPECTORS.md +4 -4
  15. data/docs/QUICKSTART.md +1 -1
  16. data/docs/RECIPES.md +5 -5
  17. data/docs/RELEASING.md +3 -1
  18. data/docs/SECURITY.md +255 -74
  19. data/docs/SETUP.md +1 -1
  20. data/docs/STANDALONE.md +2 -2
  21. data/docs/TOOLS.md +151 -90
  22. data/docs/TROUBLESHOOTING.md +123 -8
  23. data/docs/_config.yml +1 -1
  24. data/docs/adr/0008-query-runs-only-where-it-can-guard.md +38 -0
  25. data/docs/adr/0009-dbhub-shapes-in-ruby.md +26 -0
  26. data/docs/adr/0010-one-config-tool-with-sections.md +32 -0
  27. data/docs/index.md +2 -2
  28. data/exe/rails-ai-context +24 -9
  29. data/lib/generators/rails_ai_context/install/install_generator.rb +2 -0
  30. data/lib/rails_ai_context/app_databases.rb +377 -0
  31. data/lib/rails_ai_context/app_kind.rb +2 -0
  32. data/lib/rails_ai_context/cli/app_root.rb +1 -0
  33. data/lib/rails_ai_context/cli/tool_runner.rb +110 -99
  34. data/lib/rails_ai_context/configuration.rb +15 -8
  35. data/lib/rails_ai_context/database_family/mysql_runtime.rb +170 -0
  36. data/lib/rails_ai_context/database_family/postgresql_plan.rb +124 -0
  37. data/lib/rails_ai_context/database_family/postgresql_runtime.rb +211 -0
  38. data/lib/rails_ai_context/database_family/sqlite_runtime.rb +391 -0
  39. data/lib/rails_ai_context/database_family.rb +282 -0
  40. data/lib/rails_ai_context/database_file.rb +61 -7
  41. data/lib/rails_ai_context/database_yml.rb +7 -29
  42. data/lib/rails_ai_context/doctor.rb +15 -16
  43. data/lib/rails_ai_context/instrumentation.rb +1 -1
  44. data/lib/rails_ai_context/introspectors/connection_pool_introspector.rb +8 -23
  45. data/lib/rails_ai_context/introspectors/database_stats_introspector.rb +6 -70
  46. data/lib/rails_ai_context/introspectors/declared_constant.rb +19 -7
  47. data/lib/rails_ai_context/introspectors/env_introspector.rb +1 -1
  48. data/lib/rails_ai_context/introspectors/env_references.rb +1 -1
  49. data/lib/rails_ai_context/introspectors/job_introspector.rb +7 -7
  50. data/lib/rails_ai_context/introspectors/migration_introspector.rb +8 -11
  51. data/lib/rails_ai_context/introspectors/migration_replay/statements.rb +9 -9
  52. data/lib/rails_ai_context/introspectors/model_introspector.rb +33 -39
  53. data/lib/rails_ai_context/introspectors/multi_database_introspector.rb +33 -30
  54. data/lib/rails_ai_context/introspectors/node_source.rb +2 -0
  55. data/lib/rails_ai_context/introspectors/performance_introspector.rb +80 -24
  56. data/lib/rails_ai_context/introspectors/pg_partitions.rb +1 -4
  57. data/lib/rails_ai_context/introspectors/schema_conventions.rb +6 -8
  58. data/lib/rails_ai_context/introspectors/schema_dump_path.rb +12 -14
  59. data/lib/rails_ai_context/introspectors/schema_introspector.rb +47 -27
  60. data/lib/rails_ai_context/introspectors/schema_reader.rb +1 -1
  61. data/lib/rails_ai_context/introspectors/source_scan.rb +39 -4
  62. data/lib/rails_ai_context/migration_status.rb +18 -46
  63. data/lib/rails_ai_context/payload.rb +48 -1
  64. data/lib/rails_ai_context/pending_migrations.rb +3 -4
  65. data/lib/rails_ai_context/run_cache.rb +5 -0
  66. data/lib/rails_ai_context/schema_adapter.rb +13 -18
  67. data/lib/rails_ai_context/secret_columns.rb +506 -0
  68. data/lib/rails_ai_context/serializers/claude_rules_serializer.rb +2 -2
  69. data/lib/rails_ai_context/serializers/markdown_serializer.rb +14 -10
  70. data/lib/rails_ai_context/serializers/section_facts.rb +12 -2
  71. data/lib/rails_ai_context/serializers/stack_overview_helper.rb +5 -2
  72. data/lib/rails_ai_context/serializers/tool_guide_helper.rb +35 -7
  73. data/lib/rails_ai_context/server.rb +143 -29
  74. data/lib/rails_ai_context/sql_gate.rb +1007 -0
  75. data/lib/rails_ai_context/tasks/rails_ai_context.rake +1 -2
  76. data/lib/rails_ai_context/test_helper.rb +3 -1
  77. data/lib/rails_ai_context/tools/analyze_feature.rb +11 -7
  78. data/lib/rails_ai_context/tools/base_tool.rb +71 -48
  79. data/lib/rails_ai_context/tools/dependency_graph.rb +1 -1
  80. data/lib/rails_ai_context/tools/diagnose.rb +105 -42
  81. data/lib/rails_ai_context/tools/generate_test.rb +4 -6
  82. data/lib/rails_ai_context/tools/get_active_support.rb +1 -6
  83. data/lib/rails_ai_context/tools/get_api.rb +1 -6
  84. data/lib/rails_ai_context/tools/get_autoload.rb +1 -6
  85. data/lib/rails_ai_context/tools/get_callbacks.rb +5 -5
  86. data/lib/rails_ai_context/tools/get_component_catalog.rb +6 -7
  87. data/lib/rails_ai_context/tools/get_concern.rb +6 -6
  88. data/lib/rails_ai_context/tools/get_config.rb +189 -12
  89. data/lib/rails_ai_context/tools/get_context.rb +111 -105
  90. data/lib/rails_ai_context/tools/get_controllers.rb +8 -5
  91. data/lib/rails_ai_context/tools/get_conventions.rb +1 -6
  92. data/lib/rails_ai_context/tools/get_engines.rb +1 -6
  93. data/lib/rails_ai_context/tools/get_env.rb +1 -6
  94. data/lib/rails_ai_context/tools/get_env_config.rb +2 -8
  95. data/lib/rails_ai_context/tools/get_frontend_stack.rb +1 -6
  96. data/lib/rails_ai_context/tools/get_gems.rb +1 -6
  97. data/lib/rails_ai_context/tools/get_helper_methods.rb +4 -5
  98. data/lib/rails_ai_context/tools/get_i18n.rb +1 -7
  99. data/lib/rails_ai_context/tools/get_job_pattern.rb +1 -1
  100. data/lib/rails_ai_context/tools/get_mailers.rb +1 -1
  101. data/lib/rails_ai_context/tools/get_model_details.rb +5 -8
  102. data/lib/rails_ai_context/tools/get_partial_interface.rb +3 -4
  103. data/lib/rails_ai_context/tools/get_schema.rb +10 -8
  104. data/lib/rails_ai_context/tools/get_service_pattern.rb +60 -24
  105. data/lib/rails_ai_context/tools/get_turbo_map.rb +1 -1
  106. data/lib/rails_ai_context/tools/migration_advisor.rb +47 -32
  107. data/lib/rails_ai_context/tools/onboard.rb +24 -24
  108. data/lib/rails_ai_context/tools/performance_check.rb +15 -2
  109. data/lib/rails_ai_context/tools/query.rb +142 -1356
  110. data/lib/rails_ai_context/tools/read_logs.rb +1 -1
  111. data/lib/rails_ai_context/tools/review_changes.rb +6 -7
  112. data/lib/rails_ai_context/tools/runtime_info.rb +125 -88
  113. data/lib/rails_ai_context/tools/safe_call.rb +70 -28
  114. data/lib/rails_ai_context/tools/search_code.rb +0 -4
  115. data/lib/rails_ai_context/tools/search_docs.rb +3 -1
  116. data/lib/rails_ai_context/tools/security_scan.rb +9 -5
  117. data/lib/rails_ai_context/tools/session_context.rb +18 -12
  118. data/lib/rails_ai_context/tools/validate.rb +1 -0
  119. data/lib/rails_ai_context/tools/validate_semantics.rb +5 -5
  120. data/lib/rails_ai_context/version.rb +1 -1
  121. data/lib/rails_ai_context.rb +4 -7
  122. data/server.json +2 -2
  123. metadata +14 -3
@@ -0,0 +1,1007 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "set"
4
+ require "strscan"
5
+ require "timeout"
6
+
7
+ module RailsAiContext
8
+ # Decides whether rails_query runs a statement: checks on its text, and on PostgreSQL on its plan.
9
+ # The read-only transaction behind it is a second guard (ADR-0008).
10
+ class SqlGate
11
+ # sql: the comment-free text to run, or nil with the refusal. plan: the
12
+ # PostgreSQL plan the checks read, for the result redaction to reuse.
13
+ Verdict = Data.define(:sql, :refusal, :plan) do
14
+ def refused?
15
+ !refusal.nil?
16
+ end
17
+ end
18
+
19
+ # The seconds a group of reads has between them. Each read is charged the time it took itself.
20
+ class Budget
21
+ def initialize(seconds)
22
+ @left = seconds
23
+ end
24
+
25
+ def spent?
26
+ !@left.positive?
27
+ end
28
+
29
+ # Runs the block with the seconds left.
30
+ # @raise [DatabaseFamily::TimedOut] when earlier reads used them all
31
+ def spend
32
+ raise DatabaseFamily::TimedOut, "the reads beside the statement used the whole timeout" if spent?
33
+
34
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
35
+ yield @left
36
+ ensure
37
+ @left -= Process.clock_gettime(Process::CLOCK_MONOTONIC) - started if started
38
+ end
39
+ end
40
+
41
+ # The longest statement written by hand is far under it: 100,000 ids in an IN list are 700 KB. The checks on a
42
+ # statement's text have text_timeout seconds to read one.
43
+ MAX_BYTES = 1_000_000
44
+
45
+ # The text checks ran past the time they have.
46
+ class TextChecksTimedOut < RailsAiContext::Error; end
47
+
48
+ # @return [String, nil] the refusal of a call that holds no statement
49
+ def self.blank_refusal(sql)
50
+ "SQL query is required." if sql.to_s.strip.empty?
51
+ end
52
+
53
+ # @return [String, nil] the refusal of a statement longer than the checks read
54
+ def self.length_refusal(sql)
55
+ return nil unless sql.to_s.bytesize > MAX_BYTES
56
+
57
+ "Blocked: the statement is #{sql.bytesize} bytes long, and rails_query checks a statement of #{MAX_BYTES} bytes at most."
58
+ end
59
+
60
+ # REPLACE is the MySQL statement; `replace(` is the string function every
61
+ # database has, so a REPLACE followed by an opening parenthesis is let be.
62
+ BLOCKED_KEYWORDS = /\b(INSERT|UPDATE|DELETE|DROP|ALTER|TRUNCATE|CREATE|GRANT|REVOKE|SET|COPY|MERGE|REPLACE(?!\s*\())\b/i
63
+ # SHOW CREATE TABLE / VIEW print a definition; every other SHOW CREATE is
64
+ # refused by the CREATE keyword - SHOW CREATE USER prints credential hashes.
65
+ SHOW_CREATE_DEFINITION = /\A\s*SHOW\s+CREATE\s+(?:TABLE|VIEW)\b/i
66
+ BLOCKED_CLAUSES = /\bFOR\s+(?:UPDATE|SHARE|NO\s+KEY\s+UPDATE|KEY\s+SHARE)\b|\bLOCK\s+IN\s+SHARE\s+MODE\b/i
67
+ # A list keyed by family holds each name under the family that has it, so a name is refused where it can be read.
68
+
69
+ # Other sessions' statements, replication hosts and users, and the row data a log holds.
70
+ BLOCKED_SHOWS = { mysql: /\bSHOW\s+(?:(?:FULL|ALL)\s+)?(?:GRANTS|PROCESSLIST|BINLOG|BINARY|RELAYLOG|SLAVES?|MASTER|REPLICAS?|ENGINE|
71
+ PROFILES?|OPEN\s+TABLES|EXPLAIN|ANALYZE|USER_STATISTICS|CLIENT_STATISTICS|WSREP_\w+)\b/ix }.freeze
72
+ # The plan of the statement another session is running.
73
+ CONNECTION_PLAN = { mysql: /\A\s*(?:EXPLAIN|DESCRIBE|DESC)\b.*\bFOR\s+CONNECTION\b/mi }.freeze
74
+ # A SHOW filter runs an expression for each row, and MySQL's statement timeout holds SELECT only.
75
+ SHOW_WHERE = /\A\s*SHOW\b.*\bWHERE\b/mi
76
+ MULTI_STATEMENT = /;\s*\S/
77
+ ALLOWED_PREFIX = /\A\s*(SELECT|WITH|SHOW|EXPLAIN|DESCRIBE|DESC)\b/i
78
+
79
+ # SELECT-callable functions that reach files or the network through the database process.
80
+ # To the database they are reads, so a read-only transaction lets them through.
81
+ BLOCKED_FUNCTIONS = {
82
+ postgresql: /\b(
83
+ pg_read_binary_file | pg_read_file |
84
+ pg_ls_dir | pg_ls_logdir | pg_ls_tmpdir | pg_ls_waldir | pg_ls_archive_statusdir |
85
+ pg_stat_file | pg_current_logfile |
86
+ lo_import | lo_export |
87
+ dblink[a-z_]*
88
+ )\b/ix,
89
+ mysql: /\b(LOAD\s+DATA | load_file)\b/ix,
90
+ sqlite: /\b(load_extension)\b/i
91
+ }.freeze
92
+
93
+ # PostgreSQL functions that return a whole table as one value, its secret columns included.
94
+ BLOCKED_DUMPS = { postgresql: /\b((?:query|table|schema|database|cursor)_to_xml(?:schema|_and_xmlschema)?)\s*\(/i }.freeze
95
+
96
+ # Relations that hold values sampled from other tables' columns, other sessions' statements or stored
97
+ # credentials, where no check on a column's name applies.
98
+ BLOCKED_RELATIONS = {
99
+ postgresql: /\b(
100
+ pg_stats(?:_ext(?:_exprs)?)? | pg_statistic(?:_ext(?:_data)?)? | pg_largeobject |
101
+ pg_stat_activity | pg_stat_get_activity | pg_stat_get_backend_activity |
102
+ pg_stat_statements | pg_prepared_statements | pg_cursors |
103
+ pg_user_mappings? | user_mapping_options | pg_subscription |
104
+ pg_foreign_server | pg_foreign_data_wrapper | pg_foreign_table |
105
+ pg_settings | pg_file_settings | pg_show_all_settings | pg_show_all_file_settings | pg_hba_file_rules | pg_ident_file_mappings
106
+ )\b/ix,
107
+ mysql: /\b(?:(column_statistics | column_stats | global_priv | processlist | performance_schema | general_log | slow_log)\b|
108
+ information_schema[`"]?\s*\.\s*[`"]?(innodb_trx | innodb_lock\w* | innodb_ft_\w+ |
109
+ user_statistics | client_statistics | profiling | optimizer_trace)\b)/ix,
110
+ sqlite: /\b(sqlite_stat[134] | sqlite_stmt)\b/ix
111
+ }.freeze
112
+
113
+ # MySQL's own schemas, read with string literals masked. An alias cannot be told from a schema without a
114
+ # parser, so an alias of either name is refused with them.
115
+ SYSTEM_SCHEMAS = { mysql: /\b(sys|mysql)[`"]?\s*\./i }.freeze
116
+
117
+ # The SQLite pragma functions that describe the schema or the database file. Every other one, and the
118
+ # counters under it, answer for the connection that runs the statement, which is rails_query's own.
119
+ SQLITE_FILE_PRAGMAS = %w[
120
+ table_info table_xinfo table_list index_list index_info index_xinfo foreign_key_list foreign_key_check
121
+ integrity_check quick_check user_version application_id schema_version page_size page_count freelist_count
122
+ encoding auto_vacuum journal_mode compile_options pragma_list
123
+ ].to_set.freeze
124
+ SQLITE_CONNECTION_STATE = /\bpragma_(\w+)|\b(changes|total_changes|last_insert_rowid)\s*\(/i
125
+
126
+ # Catalogs refused when a statement names them. A view over one runs, and the plan checks hold the
127
+ # column that carries the credential (SecretColumns::CATALOG_CREDENTIALS) as a secret column.
128
+ BLOCKED_CATALOGS = { postgresql: /\b(#{(SecretColumns::CATALOG_CREDENTIALS.keys + %w[pg_shadow]).join("|")})\b/i }.freeze
129
+
130
+ # PostgreSQL settings whose value can be a connection string with its password, or a command line.
131
+ SECRET_SETTINGS = /\b(primary_conninfo|ssl_passphrase_command|archive_command|restore_command|archive_cleanup_command|recovery_end_command)\b/i
132
+ # A call is matched with the name it reads when that name is one plain string, and with none otherwise.
133
+ SETTING_CALL = /\bcurrent_setting\s*\((?:\s*'([^'\\]*)'\s*(?=[,)]))?/i
134
+ PLAN_SETTING_CALL = /\bcurrent_setting\((?:'((?:''|[^'])*)'::text(?=[,)]))?/
135
+ SHOWN_SETTING = /\A\s*SHOW\b(.*?);?\s*\z/mi
136
+
137
+ # Checked before the SELECT INTO check, which also matches, so the refusal names the disk write.
138
+ BLOCKED_OUTPUT = /\bINTO\s+(OUTFILE|DUMPFILE)\b/i
139
+
140
+ # PostgreSQL functions whose session or server effect outlives a read-only
141
+ # transaction. The plan's VOLATILE check refuses the ones not named here.
142
+ BLOCKED_ADMIN = { postgresql: /\b(
143
+ pg_terminate_backend | pg_cancel_backend |
144
+ pg_reload_conf | pg_rotate_logfile[a-z_]* |
145
+ pg_stat_reset[a-z_]* |
146
+ pg_switch_wal | pg_switch_xlog | pg_create_restore_point |
147
+ pg_create_(?:physical|logical)_replication_slot | pg_drop_replication_slot |
148
+ pg_replication_origin_[a-z_]* | pg_logical_emit_message |
149
+ pg_advisory_lock[a-z_]* | pg_advisory_unlock[a-z_]* |
150
+ pg_advisory_xact_lock[a-z_]* | pg_try_advisory_lock[a-z_]* |
151
+ pg_promote | pg_wal_replay_pause | pg_wal_replay_resume |
152
+ pg_xlog_replay_pause | pg_xlog_replay_resume |
153
+ pg_notify | set_config | txid_current[a-z_]* | pg_current_xact_id[a-z_]* |
154
+ pg_import_system_collations
155
+ )\s*\(/ix }.freeze
156
+
157
+ # GET_LOCK leaves a named lock on the pooled connection for a later caller to inherit, and the
158
+ # pos_wait functions block on replication. MySQL has no volatility catalogue, so these are named.
159
+ BLOCKED_SESSION = { mysql: /\b(
160
+ get_lock | release_lock | release_all_locks | is_free_lock | is_used_lock |
161
+ master_pos_wait | source_pos_wait
162
+ )\s*\(/ix }.freeze
163
+
164
+ # A value either one sets stays on the pooled connection, in or out of a read-only transaction.
165
+ VARIABLE_ASSIGNMENT = { mysql: /:=/ }.freeze
166
+ SETS_LAST_INSERT_ID = { mysql: /\blast_insert_id\s*\(\s*[^\s)]/i }.freeze
167
+
168
+ # VOLATILE built-ins that only read: pg_proc marks a reader of live server
169
+ # state VOLATILE too. Any other VOLATILE function in the plan is refused.
170
+ PG_VOLATILE_ALLOWLIST = %w[
171
+ random clock_timestamp gen_random_uuid timeofday uuid_generate_v4
172
+ pg_sleep pg_sleep_for pg_sleep_until
173
+ pg_total_relation_size pg_relation_size pg_table_size pg_indexes_size
174
+ pg_database_size pg_tablespace_size
175
+ pg_lock_status pg_blocking_pids pg_safe_snapshot_blocking_pids
176
+ pg_is_in_recovery pg_is_wal_replay_paused pg_get_wal_replay_pause_state
177
+ pg_current_wal_lsn pg_current_wal_insert_lsn pg_current_wal_flush_lsn
178
+ pg_last_wal_receive_lsn pg_last_wal_replay_lsn pg_last_xact_replay_timestamp
179
+ ].freeze
180
+
181
+ # Any character past ASCII is an identifier character to SQLite and MySQL, a no-break space included.
182
+ BARE_WORD = /(?:[\p{Word}$]|[^\x00-\x7F])+/
183
+ NUMBER = /\A(?:\d+(?:e\d+)?|0x\h+|0b[01]+)\z/i
184
+ # Words a parenthesis can follow that name no table and no alias. MySQL, whose derived table
185
+ # takes a bare alias and a column list, reserves each one, so none can stand for an alias there.
186
+ NO_ALIAS = %w[
187
+ AND AS BETWEEN CROSS DIV ELSE EXCEPT FOR FROM GROUP HAVING IN INNER INTERSECT INTO IS JOIN LATERAL LEFT LIKE LIMIT
188
+ MOD NATURAL NOT ON OR ORDER PARTITION REGEXP RIGHT RLIKE STRAIGHT_JOIN THEN UNION USING WHEN WHERE XOR
189
+ ].to_set.freeze
190
+ SUBQUERY_START = %w[SELECT WITH VALUES TABLE].freeze
191
+ # What a VALUES body may not hold if its column-alias list is to rename constants only.
192
+ ROW_SOURCES = %w[SELECT TABLE WITH UNION EXCEPT INTERSECT].freeze
193
+
194
+ IN_SHORTHAND = "`x IN t` is SQLite's shorthand for `x IN (SELECT * FROM t)`, and a value read through such a wildcard can be " \
195
+ "compared past the redaction, so write the subquery with its columns named."
196
+
197
+ # A UNION names its columns after its first SELECT, so another table's rows leave under those names.
198
+ # INTERSECT and EXCEPT return only rows of their first SELECT, so they stay.
199
+ SET_UNION = /\bUNION\b/i
200
+
201
+ # A quoted span by the quote that opens it. A doubled quote stays inside the span.
202
+ PLAIN_SPAN = { "'" => /'(?:''|[^'])*'?/m, '"' => /"(?:""|[^"])*"?/m, "`" => /`(?:``|[^`])*`?/m }.freeze
203
+ # A backslash escapes the next character: every MySQL string, and a PostgreSQL E'...' one.
204
+ ESCAPED_SPAN = { "'" => /'(?:\\.|''|[^'\\])*'?/m, '"' => /"(?:\\.|""|[^"\\])*"?/m }.freeze
205
+ # The quote of an E'...' string: the E stands right before it, as a word of its own.
206
+ E_QUOTE = /(?<=(?<![\p{Word}$]|[^\x00-\x7F])[eE])'/
207
+ # SQLite reads [name] as an identifier.
208
+ BRACKETED = /\[[^\]\n]*\]/
209
+ DOLLAR_QUOTED = /(?<![\p{Word}$])\$(\p{Word}*)\$.*?\$\1\$/m
210
+ DOLLAR_TAG = /(?<![\p{Word}$])\$\p{Word}*\$/
211
+ UNQUOTED = /[^'"`$\/\-#*\[]+/
212
+
213
+ # A constant in a plan expression. PostgreSQL's deparser always doubles
214
+ # quotes and never emits an E'' string, so this matches one exactly.
215
+ PLAN_LITERAL = /'(?:''|[^'])*'/
216
+
217
+ # Conditions and keys, which are never a plain pass-through of a column.
218
+ PLAN_COND_KEYS = [
219
+ "Filter", "Index Cond", "Recheck Cond", "Hash Cond", "Merge Cond",
220
+ "Join Filter", "One-Time Filter", "TID Cond", "Function Call",
221
+ "Sort Key", "Group Key", "Presorted Key", "Order By", "Cache Key"
222
+ ].freeze
223
+
224
+ PLAN_IDENT = /(?:[A-Za-z_]\w*|"(?:""|[^"])*")/
225
+ PASSTHROUGH_OUTPUT = /\A\(?\s*(?:#{PLAN_IDENT}\.)?#{PLAN_IDENT}\s*\)?\z/
226
+
227
+ # SHOW reads a setting and PostgreSQL has no DESCRIBE, so these have no plan.
228
+ # Every other statement is planned, as is whatever a leading EXPLAIN would run.
229
+ NO_PLAN = /\A\s*(?:SHOW|DESCRIBE|DESC)\b/i
230
+ EXPLAIN_PREFIX = /\A\s*EXPLAIN\s+(?:\([^)]*\)\s*)?(?:ANALYZE\s+)?(?:VERBOSE\s+)?/i
231
+
232
+ # A statement's text under a pointer. A lookbehind sees the text before the pointer, and where each dollar
233
+ # tag last stands is read in one pass: a search for its closer from every opener is quadratic.
234
+ class Scanner < StringScanner
235
+ def initialize(text)
236
+ super(text, fixed_anchor: true)
237
+ end
238
+
239
+ # The [name] at the pointer. A line with no closing bracket is read once, however many brackets open on it.
240
+ def bracketed
241
+ return nil if @unclosed_to && pos < @unclosed_to
242
+
243
+ span = scan(BRACKETED)
244
+ @unclosed_to = pos + check(/[^\]\n]*/).bytesize unless span
245
+ span
246
+ end
247
+
248
+ # Whether the dollar tag at the pointer stands again after itself, to close the quote it opens.
249
+ def closed?(tag)
250
+ @last ||= {}.tap do |last|
251
+ tags = StringScanner.new(string)
252
+ last["$#{tags[1]}"] = tags.pos - 1 while tags.skip_until(/\$(?=(\p{Word}*\$))/)
253
+ end
254
+ @last.fetch(tag, -1) >= pos + tag.bytesize
255
+ end
256
+ end
257
+
258
+ # budget: the seconds the reads the gate sends (the plan, the catalog lookups) share with the caller's other reads.
259
+ # text_timeout: the seconds the checks on the statement's text have to themselves.
260
+ def initialize(family:, secrets:, connection:, budget:, text_timeout: RailsAiContext.configuration.query_timeout)
261
+ @text_timeout = text_timeout
262
+ @family = family
263
+ @secrets = secrets
264
+ @connection = connection
265
+ @budget = budget
266
+ @backslash = family.backslash_escapes?
267
+ end
268
+
269
+ def check(sql)
270
+ if (refusal = self.class.blank_refusal(sql) || self.class.length_refusal(sql))
271
+ return refuse(refusal)
272
+ end
273
+
274
+ text, refusal = text_verdict(sql)
275
+ return refuse(refusal) if refusal
276
+ planned = @family.plan_checks? && !text.match?(NO_PLAN)
277
+ plan = @budget.spend { |left| @family.plan(@connection, text.sub(EXPLAIN_PREFIX, ""), timeout: left) } if planned
278
+ if planned && plan.nil?
279
+ return refuse("Blocked: the query is not run because its plan could not be read. On PostgreSQL " \
280
+ "the plan shows which columns a query returns, so no query runs without one.")
281
+ end
282
+
283
+ refusal = plan_refusal(plan, text) || unvouched_literal_refusal(text, plan)
284
+ refusal ? refuse(refusal) : Verdict.new(sql: text, refusal: nil, plan: plan)
285
+ end
286
+
287
+ # The text with its comments out and every quoted span copied whole, as the family reads them.
288
+ def self.comment_free(sql, family, backslash: family.backslash_escapes?)
289
+ scanner = Scanner.new(sql)
290
+ out = +""
291
+ executable = false
292
+
293
+ until scanner.eos?
294
+ if (span = quoted(scanner, family, backslash: backslash)&.last || scanner.scan(UNQUOTED))
295
+ out << span
296
+ elsif family.executable_comments? && scanner.scan(/\/\*!\d*/)
297
+ # Its content stays for the checks to read: `SELECT /*!50000 LOAD_FILE(...) */`.
298
+ executable = true
299
+ out << " "
300
+ elsif executable && scanner.scan(/\*\//)
301
+ executable = false
302
+ out << " "
303
+ elsif comment(scanner, family)
304
+ out << " "
305
+ else
306
+ out << scanner.getch
307
+ end
308
+ end
309
+
310
+ out.strip
311
+ end
312
+
313
+ # The quoted span that opens where the scanner stands, as the family reads it: [:string or :quoted, the span], or nil.
314
+ # brackets: whether [name] quotes a name in this text. backslash: whether one escapes the next character of a string.
315
+ def self.quoted(scanner, family, brackets: family.sqlite?, backslash: family.backslash_escapes?)
316
+ case scanner.peek(1)
317
+ when "'" then [ :string, scanner.scan(backslash || scanner.check(E_QUOTE) ? ESCAPED_SPAN["'"] : PLAIN_SPAN["'"]) ]
318
+ when '"' then [ :quoted, scanner.scan(backslash ? ESCAPED_SPAN['"'] : PLAIN_SPAN['"']) ]
319
+ when "`" then [ :quoted, scanner.scan(PLAIN_SPAN["`"]) ]
320
+ when "[" then brackets && (span = scanner.bracketed) ? [ :quoted, span ] : nil
321
+ when "$"
322
+ tag = scanner.check(DOLLAR_TAG) if family.dollar_quoting?
323
+ [ :string, scanner.scan(DOLLAR_QUOTED) ] if tag && scanner.closed?(tag)
324
+ end
325
+ end
326
+
327
+ # Skips the comment that opens where the scanner stands, as the family reads it. One that never closes runs to the end.
328
+ # @return [Boolean] whether one opened there
329
+ def self.comment(scanner, family)
330
+ return !scanner.skip(family.line_comment).nil? unless scanner.skip(/\/\*/)
331
+
332
+ depth = 1
333
+ marks = family.nested_comments? ? /\/\*|\*\// : /\*\//
334
+ depth += scanner.matched == "/*" ? 1 : -1 while depth.positive? && scanner.skip_until(marks)
335
+ scanner.terminate if depth.positive?
336
+ true
337
+ end
338
+
339
+ private
340
+
341
+ def refuse(reason)
342
+ Verdict.new(sql: nil, refusal: reason, plan: nil)
343
+ end
344
+
345
+ # [the comment-free text, the refusal the text checks give or nil]. The checks send nothing and hold nothing,
346
+ # so ending them from outside leaves nothing half done.
347
+ def text_verdict(sql)
348
+ Timeout.timeout(@text_timeout, TextChecksTimedOut) do
349
+ refusal = text_refusal(sql)
350
+ next [ nil, refusal ] if refusal
351
+
352
+ text = comment_free(sql)
353
+ # With no backslash in it the text reads the same both ways, and the pass above decided.
354
+ if @family.backslash_escapes? && text.include?("\\") && (refusal = without_backslash_escapes { text_refusal(text) })
355
+ refusal = "#{refusal.delete_suffix(".")}. That is how MySQL reads the statement when the session's sql_mode holds NO_BACKSLASH_ESCAPES, " \
356
+ "and rails_query does not read the session's mode. Write a quote inside a string doubled, so that both readings agree."
357
+ end
358
+ [ text, refusal ]
359
+ end
360
+ rescue TextChecksTimedOut
361
+ [ nil, "Blocked: the checks on the statement's text ran longer than query_timeout (#{@text_timeout} seconds), so the statement " \
362
+ "was not run. Send a shorter statement." ]
363
+ end
364
+
365
+ # MySQL reads a backslash in a string as an ordinary character under NO_BACKSLASH_ESCAPES. The block's
366
+ # checks read the text that way.
367
+ def without_backslash_escapes
368
+ @backslash = false
369
+ yield
370
+ ensure
371
+ @backslash = @family.backslash_escapes?
372
+ end
373
+
374
+ def text_refusal(sql)
375
+ # Read on the raw text too, in case a novel comment form defeats the stripping.
376
+ if (m = listed(BLOCKED_FUNCTIONS, sql))
377
+ return "Blocked: dangerous function #{m[0]} (filesystem/network primitive)"
378
+ end
379
+
380
+ cleaned = comment_free(sql).squeeze(" ")
381
+ # Statement checks read quote-masked text, so `LIKE '%update%'` is data. A disk write and the
382
+ # file functions stay on the raw text: no read-only transaction stops them.
383
+ masked = mask_quoted(cleaned)
384
+ return "Blocked: multiple statements (no semicolons)" if masked.match?(MULTI_STATEMENT)
385
+ # MariaDB in its MSSQL mode reads [name] as a quoted name, and no statement of this family holds a bracket otherwise.
386
+ if @family.mysql? && masked.include?("[")
387
+ return "Blocked: a `[` outside a string or a quoted name is not run. MariaDB reads it as the start of a quoted name when the " \
388
+ "session's sql_mode holds MSSQL, and rails_query does not read the session's mode. Quote a name in backticks."
389
+ end
390
+ # PostgreSQL's read-only transaction refuses a locking read, so a literal there is data.
391
+ # MySQL's runs FOR SHARE and LOCK IN SHARE MODE, so there the raw text decides.
392
+ locking = @family.refuses_locking_reads? ? masked : cleaned
393
+ return "Blocked: locking read (FOR UPDATE/SHARE or LOCK IN SHARE MODE clause)" if locking.match?(BLOCKED_CLAUSES)
394
+ return "Blocked: sensitive SHOW command" if listed(BLOCKED_SHOWS, cleaned)
395
+ if listed(CONNECTION_PLAN, masked)
396
+ return "Blocked: EXPLAIN FOR CONNECTION is not run. It shows the plan of the statement another session is running."
397
+ end
398
+ if masked.match?(SHOW_WHERE)
399
+ return "Blocked: SHOW with a WHERE clause is not run, because MySQL's statement timeout does not hold it. " \
400
+ "SHOW ... LIKE runs. For any other filter, SELECT from information_schema, " \
401
+ "for example `SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE()`."
402
+ end
403
+ return "Blocked: SELECT INTO OUTFILE / DUMPFILE writes to disk" if cleaned.match?(BLOCKED_OUTPUT)
404
+ # No second statement is left by here, so an INTO after a SELECT belongs to it.
405
+ select = masked.index(/\bSELECT\b/i)
406
+ if select && masked.rindex(/\bINTO\b/i).to_i > select
407
+ # After a WITH clause the INTO can be a write's own.
408
+ write = masked[/\b(?:INSERT|MERGE|REPLACE(?!\s*\())\b(?=[^;]*\bINTO\b)/i]
409
+ return write ? "Blocked: contains #{write}" : "Blocked: SELECT INTO creates a table"
410
+ end
411
+
412
+ if (m = listed(BLOCKED_FUNCTIONS, cleaned))
413
+ return "Blocked: dangerous function #{m[0]} (filesystem/network primitive)"
414
+ end
415
+ if (m = listed(BLOCKED_DUMPS, cleaned))
416
+ return dump_refusal(m[1])
417
+ end
418
+ # Matched again as MySQL folds a name, so a letter it reads as a plain one names the same relation.
419
+ if (m = [ cleaned, SecretColumns.fold(cleaned) ].filter_map { |read| listed(BLOCKED_RELATIONS, read) || listed(BLOCKED_CATALOGS, read) }.first)
420
+ return never_read_refusal(m.captures.compact.first)
421
+ end
422
+ if (m = listed(SYSTEM_SCHEMAS, mask_quoted(cleaned, :string)))
423
+ text_only = ->(kind, span) { kind == :string || (span.start_with?('"') && !span.match?(/\A"\w+"\z/)) }
424
+ return "#{never_read_refusal(m[1])} A name before a dot is read as that schema, so an alias named `#{m[1]}` needs another name." if
425
+ listed(SYSTEM_SCHEMAS, mask_quoted(cleaned, &text_only))
426
+
427
+ return "#{never_read_refusal(m[1])} The name stands before a dot inside double-quoted text, which MySQL reads as a name when the " \
428
+ "session's sql_mode holds ANSI_QUOTES, and rails_query does not read the session's mode. Write the text in single quotes."
429
+ end
430
+ table, why = @secrets.unread_table_in(cleaned)
431
+ return "Blocked: `#{table}` is not read: #{why}." if table
432
+ if (state = connection_state(cleaned))
433
+ return "Blocked: `#{state}` tells of the connection that runs it. rails_query runs a statement on a connection of its own and " \
434
+ "sets some of that connection's values itself, so the answer would not be the app's. A pragma function that describes the " \
435
+ "schema or the database file runs, for example `pragma_table_info`."
436
+ end
437
+
438
+ if (refusal = setting_refusal(cleaned))
439
+ return refusal
440
+ end
441
+ if (m = listed(BLOCKED_ADMIN, cleaned))
442
+ return "Blocked: administrative function #{m[0].sub(/\s*\(\z/, "")} has a session or server-wide " \
443
+ "effect that a read-only transaction does not prevent. rails_query runs " \
444
+ "read-only queries for inspection; it will not run server-control functions."
445
+ end
446
+ if (m = listed(BLOCKED_SESSION, cleaned))
447
+ return "Blocked: session function #{m[0].sub(/\s*\(\z/, "")} leaves state (a named lock) on the pooled " \
448
+ "connection after the query returns. rails_query will not run it."
449
+ end
450
+ if listed(VARIABLE_ASSIGNMENT, masked)
451
+ return "Blocked: a user variable assignment (`:=`) is not run. It leaves a value on the pooled connection after the query " \
452
+ "returns, which a read-only transaction does not stop."
453
+ end
454
+ if listed(SETS_LAST_INSERT_ID, cleaned)
455
+ return "Blocked: LAST_INSERT_ID with an argument is not run. It sets a value that stays on the pooled connection after the " \
456
+ "query returns, which a read-only transaction does not stop. LAST_INSERT_ID() with no argument runs."
457
+ end
458
+
459
+ tokens = tokens(cleaned)
460
+ if renaming_alias_list?(tokens)
461
+ return "Blocked: a column-alias list (e.g. `t(a, b, c)`) renames the columns a " \
462
+ "wildcard returns, so secret columns would leave under harmless names. " \
463
+ "Name the columns you want in the SELECT list instead."
464
+ end
465
+ if quoted_call?(tokens)
466
+ return "Blocked: a quoted name is followed by a parenthesis, the way a function is called. rails_query reads " \
467
+ "a function's name to decide whether it runs, so write the name without quotes."
468
+ end
469
+
470
+ # On PostgreSQL a literal is data, since the plan checks read the query itself. MySQL's quote
471
+ # modes and SQLite's t.'col' identifiers could fool a masker, so there the raw text decides.
472
+ literal_free = @family.masks_literals? ? mask_sql_literals(cleaned) : cleaned
473
+
474
+ if literal_free.match?(SET_UNION)
475
+ return "Blocked: UNION is not run. A UNION takes its column names from its first SELECT, " \
476
+ "so rows of another table would come out under those names, past the redaction that " \
477
+ "reads them. Query each table on its own; a recursive CTE needs UNION, so it is not run either."
478
+ end
479
+
480
+ if (m = masked.sub(SHOW_CREATE_DEFINITION, "SHOW").match(BLOCKED_KEYWORDS))
481
+ return "Blocked: contains #{m[0]}"
482
+ end
483
+
484
+ return "Only SELECT, WITH, SHOW, EXPLAIN, DESCRIBE allowed" unless cleaned.match?(ALLOWED_PREFIX)
485
+
486
+ if (offending = @secrets.named_in(literal_free))
487
+ return secret_column_refusal(offending)
488
+ end
489
+
490
+ # With no plan to read, a value a subquery's wildcard returns can be compared or renamed unseen. Which
491
+ # relation the wildcard reads is not told without a parser, so the rule is the two together.
492
+ wildcard = subquery_wildcard(tokens) unless @family.plan_checks?
493
+ if wildcard && (relation = @secrets.relation_named_in(names_text(tokens)))
494
+ holds = "is not run in a statement that names `#{relation}`, which holds the secret column `#{@secrets.of_relation(relation).first}`."
495
+ return "Blocked: `TABLE` inside a subquery #{holds} `TABLE t` reads every column of `t`, and on #{@family.name} a value read that " \
496
+ "way can be compared or renamed past the redaction, so SELECT the columns you need in the subquery." if wildcard == :table
497
+
498
+ return "Blocked: `IN` before a table's name #{holds} #{IN_SHORTHAND}" if wildcard == :in
499
+
500
+ return "Blocked: a `*` inside a subquery #{holds} On #{@family.name} a value read through such a wildcard can be compared or " \
501
+ "renamed past the redaction, so name the columns in the subquery."
502
+ end
503
+ if wildcard && wildcard != :table && (relation = string_naming_relation(tokens))
504
+ return "Blocked: #{wildcard == :in ? "`IN` before a table's name" : "a `*` inside a subquery"} is not run in a statement that holds " \
505
+ "the string '#{relation}'. SQLite reads a string as a table's name where a name can stand, and the table `#{relation}` holds the " \
506
+ "secret column `#{@secrets.of_relation(relation).first}`. " + (wildcard == :in ? IN_SHORTHAND :
507
+ "A value read through such a wildcard can be compared or renamed past the redaction, so name the columns in the subquery.")
508
+ end
509
+
510
+ # A definition is schema text, so SHOW CREATE VIEW still shows which tables a refused view reads.
511
+ # With a plan the view is told there, by its schema and name: the text does not say which schema a name is in.
512
+ return nil if cleaned.match?(SHOW_CREATE_DEFINITION) || @family.plan_checks?
513
+
514
+ view, why = @secrets.refused_view_in(literal_free)
515
+ view_refusal(view, why) if view
516
+ end
517
+
518
+ def view_refusal(view, why)
519
+ "Blocked: the view `#{view}` is not queried: #{why}. A view can rename a column past the redaction " \
520
+ "on #{@family.name}, so query the tables it reads."
521
+ end
522
+
523
+ # The plan names every relation a statement reads, through any view and in any schema. A view that stores
524
+ # its rows holds what its definition read, so the rule follows that definition as it follows a statement.
525
+ def unread_relation_refusal(plan)
526
+ relations = plan.relations.map(&:last)
527
+ unknown = plan.relations.reject { |_schema, relation| @secrets.knows?(relation) }.map(&:first)
528
+ beside = ->(&reads) { @budget.spend { |left| @family.bounded(@connection, timeout: left, &reads) } }
529
+ beside.call { @secrets.read_schemas(unknown) } if unknown.any?
530
+ stored = relations.select { |relation| @secrets.stored_view?(relation) }
531
+ beside.call { @secrets.reach(stored.join(" ")) } if stored.any?
532
+
533
+ if (relation = relations.find { |name| !@secrets.knows?(name) })
534
+ return "Blocked: `#{relation}` is not queried: its columns could not be read, so rails_query cannot tell which of them are secret."
535
+ end
536
+
537
+ relations.lazy.filter_map do |placed|
538
+ why = @secrets.placed_view_refusal(placed, plan.name(placed))
539
+ view_refusal(plan.name(placed), why) if why
540
+ end.first
541
+ end
542
+
543
+ # The family's own part of a list, matched against the text.
544
+ def listed(list, text)
545
+ list[@family.key]&.match(text)
546
+ end
547
+
548
+ def never_read_refusal(name)
549
+ "Blocked: `#{name}` is not read. It holds values sampled from other tables, other sessions' statements " \
550
+ "or stored credentials, and rails_query cannot tell which of them are secret." \
551
+ "#{" To read one setting, run SHOW with its name." if name.match?(/settings/i)}"
552
+ end
553
+
554
+ def connection_state(cleaned)
555
+ return nil unless @family.sqlite?
556
+
557
+ cleaned.to_enum(:scan, SQLITE_CONNECTION_STATE).filter_map do
558
+ pragma, counter = Regexp.last_match.captures
559
+ name = counter&.downcase || ("pragma_#{pragma.downcase}" unless SQLITE_FILE_PRAGMAS.include?(pragma.downcase))
560
+ name unless name.nil? || @secrets.knows?(name)
561
+ end.first
562
+ end
563
+
564
+ def dump_refusal(function)
565
+ "Blocked: function #{function} returns a whole table as one value, secret columns included. " \
566
+ "SELECT the columns you need from the table itself."
567
+ end
568
+
569
+ # A setting is read by SHOW and by current_setting. SHOW has no plan, so on PostgreSQL its name is read
570
+ # here and has to be plain: a quoted name can spell any setting.
571
+ def setting_refusal(cleaned)
572
+ return nil unless @family.postgresql?
573
+
574
+ if (m = cleaned.match(SECRET_SETTINGS))
575
+ return secret_setting_refusal(m[1])
576
+ end
577
+
578
+ called = cleaned.to_enum(:scan, SETTING_CALL).map { Regexp.last_match[1] }
579
+ return unread_setting_name_refusal if called.include?(nil)
580
+
581
+ shown = cleaned.match(SHOWN_SETTING)&.then { |m| m[1].strip }
582
+ if shown && !shown.match?(/\A[\w. ]+\z/)
583
+ return "Blocked: SHOW takes a setting's plain name on PostgreSQL, with no quotes around it, for example `SHOW work_mem`."
584
+ end
585
+ if shown&.casecmp?("ALL")
586
+ return "Blocked: SHOW ALL is not run. It lists every setting, the ones that can hold a password among them. " \
587
+ "SHOW one setting by its name."
588
+ end
589
+
590
+ secret = (called + [ shown ]).compact.find { |name| secret_setting?(name) }
591
+ secret_setting_refusal(secret) if secret
592
+ end
593
+
594
+ # An app's own setting (`app.jwt_secret`) is told by its name, as a column is.
595
+ def secret_setting?(name)
596
+ name.match?(SECRET_SETTINGS) || name.split(/[.\s]+/).any? { |part| @secrets.secret?(part) }
597
+ end
598
+
599
+ def secret_setting_refusal(name)
600
+ "Blocked: the setting `#{name}` is not read. It can hold a password, a key or a command line that carries one."
601
+ end
602
+
603
+ def unread_setting_name_refusal
604
+ "Blocked: current_setting runs only with the setting's name written as one plain string, " \
605
+ "for example `current_setting('work_mem')`, so that rails_query can read which setting it returns."
606
+ end
607
+
608
+ # An alias or an expression renames a column past the redaction that
609
+ # reads output names, so a statement naming a secret column is refused whole.
610
+ def secret_column_refusal(offending)
611
+ source = @secrets.generated_from(offending)
612
+ reads = "`#{offending}` is a generated column that reads `#{source}`" if source
613
+ way_out = if @secrets.encrypted?(offending)
614
+ if source
615
+ "#{reads}, an `encrypts` value, so config.query_allowed_columns does not release it."
616
+ else
617
+ "A model declares `#{offending}` with `encrypts`, so config.query_allowed_columns does not release it."
618
+ end
619
+ else
620
+ "#{"#{reads}. " if source}If `#{offending}` is not secret in your app, add \"#{offending}\" to query_allowed_columns in " \
621
+ ".rails-ai-context.yml (or in config/initializers/rails_ai_context.rb where the app bundles the gem)."
622
+ end
623
+ "Blocked: query references secret column `#{offending}`. " \
624
+ "Name the columns you need; `#{offending}` is secret and never " \
625
+ "returned, and an alias or expression over it cannot be redacted after " \
626
+ "the query runs, so the whole query is refused. #{way_out}"
627
+ end
628
+
629
+ def comment_free(sql)
630
+ self.class.comment_free(sql, @family, backslash: @backslash)
631
+ end
632
+
633
+ # Every quoted and dollar-quoted span blanked to spaces of the same length, so a check reads structure
634
+ # only and positions still line up. only: the one kind to blank, :string or :quoted. With a block, the spans it picks.
635
+ def mask_quoted(text, only = nil, &picks)
636
+ picks ||= ->(kind, _span) { only.nil? || kind == only }
637
+ scanner = Scanner.new(text)
638
+ out = +""
639
+
640
+ until scanner.eos?
641
+ kind, span = self.class.quoted(scanner, @family, backslash: @backslash)
642
+ out << (span.nil? ? scanner.scan(UNQUOTED) || scanner.getch : picks.call(kind, span) ? " " * span.length : span)
643
+ end
644
+
645
+ out
646
+ end
647
+
648
+ # [kind, text] for each token of comment-free text: :word (bare, a keyword or a name), :quoted (a name in
649
+ # double quotes, backticks or brackets), :string, :number, or :punct, one character.
650
+ def tokens(text)
651
+ scanner = Scanner.new(text)
652
+ found = []
653
+ until scanner.eos?
654
+ next if scanner.skip(/\s+/) || scanner.skip(/[uU]&(?=["'])/)
655
+
656
+ # A bracketed name is read on every family: an alias list is told by its shape, whatever quotes its names.
657
+ found << (self.class.quoted(scanner, @family, brackets: true, backslash: @backslash) ||
658
+ scanner.scan(BARE_WORD)&.then { |span| [ span.match?(NUMBER) ? :number : :word, span ] } || [ :punct, scanner.getch ])
659
+ end
660
+ found
661
+ end
662
+
663
+ # PostgreSQL's string literals emptied, with the E, B, X or U& that stands before one. A quoted name stays.
664
+ def mask_sql_literals(text)
665
+ scanner = Scanner.new(text)
666
+ out = +""
667
+
668
+ until scanner.eos?
669
+ kind, span = self.class.quoted(scanner, @family)
670
+ if kind == :string
671
+ out.chop! if out.end_with?("&")
672
+ # Read as a byte: a character past ASCII ends in one from 0x80 up, and it is part of a name.
673
+ before = out.getbyte(-2)
674
+ out.chop! if out.end_with?(*%w[e E b B x X u U]) && !(before && (before >= 0x80 || before.chr.match?(/[\w$]/)))
675
+ out << "''"
676
+ else
677
+ out << (span || scanner.scan(UNQUOTED) || scanner.getch)
678
+ end
679
+ end
680
+
681
+ out
682
+ end
683
+
684
+ # A column-alias list renames the columns a wildcard produced, past the redaction that reads names.
685
+ # It is told by its shape: a parenthesised list of names after a CTE's name, or after a table's alias.
686
+ def renaming_alias_list?(tokens)
687
+ pairs = paren_pairs(tokens)
688
+ tokens.each_index.any? do |open|
689
+ close = pairs[open] if tokens[open] == [ :punct, "(" ]
690
+ kinds = name_list_kinds(tokens, open, close) if close
691
+ kinds && (cte_list?(tokens, pairs, open, close) || alias_list?(tokens, pairs, open, close, kinds))
692
+ end
693
+ end
694
+
695
+ # "(" index => its ")" index, and back.
696
+ def paren_pairs(tokens)
697
+ pairs = {}
698
+ stack = []
699
+ tokens.each_with_index do |token, i|
700
+ if token == [ :punct, "(" ] then stack << i
701
+ elsif token == [ :punct, ")" ] && (open = stack.pop)
702
+ pairs[open] = i
703
+ pairs[i] = open
704
+ end
705
+ end
706
+ pairs
707
+ end
708
+
709
+ # The kinds of the names between the parentheses, or nil when they hold anything but names and commas.
710
+ # A type modifier holds numbers and a record function's list holds types, so neither is one.
711
+ def name_list_kinds(tokens, open, close)
712
+ return nil unless (close - open).even?
713
+
714
+ # Read in place and left at the first token that is no name: a copy for each parenthesis of a deep nest is quadratic.
715
+ kinds = []
716
+ (open + 1).step(close - 1, 2) do |i|
717
+ return nil unless %i[word quoted string].include?(tokens[i][0]) && (i + 1 == close || tokens[i + 1] == [ :punct, "," ])
718
+
719
+ kinds << tokens[i][0]
720
+ end
721
+ kinds.uniq
722
+ end
723
+
724
+ # name (a, b) AS [NOT] [MATERIALIZED] (body). SQLite takes a string for a name here.
725
+ def cte_list?(tokens, pairs, open, close)
726
+ return false unless open.positive? && %i[word quoted string].include?(tokens[open - 1][0]) && word?(tokens[close + 1], "AS")
727
+
728
+ body = close + 2
729
+ body += 1 if word?(tokens[body], "NOT")
730
+ body += 1 if word?(tokens[body], "MATERIALIZED")
731
+ tokens[body] == [ :punct, "(" ] && !values_only?(tokens, body, pairs[body])
732
+ end
733
+
734
+ # AS t (a, b) after anything, and with no AS: (subquery) t (a, b) and FROM table t (a, b).
735
+ def alias_list?(tokens, pairs, open, close, kinds)
736
+ return false unless open >= 2 && %i[word quoted string].include?(tokens[open - 1][0])
737
+
738
+ before = open - 2
739
+ if word?(tokens[before], "AS")
740
+ source = pairs[before - 1] if before.positive? && tokens[before - 1] == [ :punct, ")" ]
741
+ return true unless source
742
+ return false if values_only?(tokens, source, before - 1)
743
+
744
+ # One name after a function call names the one column the function returns.
745
+ close - open > 2 || subquery_at?(tokens, source)
746
+ elsif kinds.include?(:string) || !name?(tokens[open - 1])
747
+ false
748
+ elsif tokens[before] == [ :punct, ")" ]
749
+ source = pairs[before]
750
+ !source.nil? && subquery_at?(tokens, source) && !values_only?(tokens, source, before)
751
+ else
752
+ word?(tokens[before], "ORDINALITY") || table_at?(tokens, before)
753
+ end
754
+ end
755
+
756
+ # Whether the token at `last` ends a table name in a FROM clause: [FROM|JOIN] [ONLY] schema.table [*].
757
+ def table_at?(tokens, last)
758
+ last -= 1 if tokens[last] == [ :punct, "*" ]
759
+ return false unless last.positive? && name?(tokens[last])
760
+
761
+ last -= 2 while last >= 2 && tokens[last - 1] == [ :punct, "." ] && name?(tokens[last - 2])
762
+ last -= 1 if last.positive? && word?(tokens[last - 1], "ONLY")
763
+ last.positive? && %w[FROM JOIN STRAIGHT_JOIN].any? { |anchor| word?(tokens[last - 1], anchor) }
764
+ end
765
+
766
+ def word?(token, keyword)
767
+ !token.nil? && token[0] == :word && token[1].casecmp?(keyword)
768
+ end
769
+
770
+ # A quoted name, or a bare word that can name a table or an alias.
771
+ def name?(token)
772
+ token[0] == :quoted || (token[0] == :word && !NO_ALIAS.include?(token[1].upcase))
773
+ end
774
+
775
+ # A quoted name can spell any function, past every check that reads a function's name.
776
+ def quoted_call?(tokens)
777
+ tokens.each_index.any? do |i|
778
+ following = i + (word?(tokens[i + 1], "UESCAPE") ? 3 : 1)
779
+ tokens[i][0] == :quoted && tokens[following] == [ :punct, "(" ]
780
+ end
781
+ end
782
+
783
+ def subquery_at?(tokens, open)
784
+ following = tokens[open + 1]
785
+ following == [ :punct, "(" ] || SUBQUERY_START.any? { |start| word?(following, start) }
786
+ end
787
+
788
+ # A list over a body of VALUES rows renames constants, so it is left alone.
789
+ def values_only?(tokens, open, close)
790
+ return false unless close && word?(tokens[open + 1], "VALUES")
791
+
792
+ before = row_sources_before(tokens)
793
+ before[close] == before[open + 2]
794
+ end
795
+
796
+ # How many row-source words stand before each token, so that no body is scanned for them on its own.
797
+ def row_sources_before(tokens)
798
+ (@row_sources ||= {}.compare_by_identity)[tokens] ||= tokens.each_with_object([ 0 ]) do |token, counts|
799
+ counts << counts.last + (ROW_SOURCES.any? { |source| word?(token, source) } ? 1 : 0)
800
+ end
801
+ end
802
+
803
+ # :star for a select-list wildcard inside parentheses, :table for `TABLE t` there off SQLite, :in for SQLite's
804
+ # `x IN t` anywhere, else nil. `count(*)` reads no column and EXISTS returns none, so both are left alone.
805
+ def subquery_wildcard(tokens)
806
+ enclosing = []
807
+ tokens.each_with_index do |token, i|
808
+ following = tokens[i + 1]
809
+ return :in if @family.sqlite? && word?(token, "IN") && following && following != [ :punct, "(" ]
810
+
811
+ enclosing << i if token == [ :punct, "(" ]
812
+ enclosing.pop if token == [ :punct, ")" ]
813
+ next if enclosing.empty? || (enclosing.last.positive? && word?(tokens[enclosing.last - 1], "EXISTS"))
814
+
815
+ star = token == [ :punct, "*" ] && (following.nil? || [ ",", ")" ].include?(following[1]) || word?(following, "FROM")) &&
816
+ !(tokens[i - 1] == [ :punct, "(" ] && following == [ :punct, ")" ])
817
+ return :star if star
818
+ return :table if !@family.sqlite? && word?(token, "TABLE") && tokens[i - 1] != [ :punct, "." ]
819
+ end
820
+ nil
821
+ end
822
+
823
+ # The statement with each string emptied: where a string names a relation, string_naming_relation reads it.
824
+ def names_text(tokens)
825
+ tokens.map { |kind, text| kind == :string ? "''" : text }.join(" ")
826
+ end
827
+
828
+ # SQLite reads a string as a name wherever a name can stand, so a string equal to such a relation's name is one.
829
+ def string_naming_relation(tokens)
830
+ return nil unless @family.sqlite?
831
+
832
+ tokens.each do |kind, text|
833
+ name = text[1..].delete_suffix("'").gsub("''", "'") if kind == :string
834
+ return name if name && @secrets.relation_secret?(name)
835
+ end
836
+ nil
837
+ end
838
+
839
+ def plan_refusal(top, text)
840
+ return nil unless top
841
+
842
+ plan = DatabaseFamily::PostgresqlPlan.new(top)
843
+ refusal = unread_relation_refusal(plan)
844
+ return refusal if refusal
845
+
846
+ raw = plan.nodes.flat_map { |node| plan.expressions(node) }
847
+ outputs = []
848
+ conds = []
849
+ plan.nodes.each do |node|
850
+ outputs.concat(Array(node["Output"]).map { |e| e.to_s.gsub(PLAN_LITERAL, "''") })
851
+ PLAN_COND_KEYS.each { |k| conds.concat(Array(node[k]).map { |e| e.to_s.gsub(PLAN_LITERAL, "''") }) }
852
+ end
853
+ foreign = plan.nodes.select { |node| node["Remote SQL"] }
854
+ conds.concat(foreign.map { |node| remote_uses(node) }, local_names(foreign))
855
+ # EXPLAIN prints no row of a VALUES list and no LIMIT, so the functions the text names are read too.
856
+ called = function_names(raw.map { |expr| expr.gsub(PLAN_LITERAL, "''") } + [ mask_sql_literals(text) ])
857
+
858
+ # A view reaches a relation or a function the text check did not see.
859
+ sources = plan.nodes.flat_map { |node| node.values_at("Relation Name", "Function Name") }.compact + called.to_a
860
+ if (name = sources.find { |source| source.match?(/\A#{BLOCKED_RELATIONS.fetch(:postgresql)}\z/) })
861
+ return never_read_refusal(name)
862
+ end
863
+ if (name = called.find { |function| "#{function}(".match?(/\A#{BLOCKED_DUMPS.fetch(:postgresql)}\z/) })
864
+ return dump_refusal(name)
865
+ end
866
+ settings = raw.flat_map { |expr| expr.to_enum(:scan, PLAN_SETTING_CALL).map { Regexp.last_match[1] } }
867
+ return unread_setting_name_refusal if settings.include?(nil)
868
+ if (name = settings.find { |setting| secret_setting?(setting) })
869
+ return secret_setting_refusal(name)
870
+ end
871
+ if (rel = whole_row_leak(outputs + conds, plan.aliases))
872
+ return "Blocked: the query serialises a whole row of `#{plan.name(rel)}`, which carries every " \
873
+ "column including the secret ones. Name the columns you need in the SELECT list."
874
+ end
875
+ if (col = secret_column_in_plan(outputs, conds))
876
+ return "Blocked: the query's plan uses the secret column `#{col}` (expanded from a " \
877
+ "wildcard or a row reference). Name the other columns you need; `#{col}` is " \
878
+ "never returned."
879
+ end
880
+ if (col = laundered_secret_column(plan))
881
+ return "Blocked: the query passes the secret column `#{col}` through a CTE, subquery " \
882
+ "or set operation whose output the tool cannot map to a column, so it could leave " \
883
+ "unredacted. Name the other columns you need; `#{col}` is never returned."
884
+ end
885
+ if (fn = volatile_function(called))
886
+ "Blocked: function #{fn} is VOLATILE (pg_proc.provolatile = 'v') and not on " \
887
+ "rails_query's list of read-only VOLATILE functions, so it is not run. Some VOLATILE " \
888
+ "functions change server or session state in ways a read-only transaction does not " \
889
+ "stop; only those known to just read state (sizes, lock status, recovery and WAL " \
890
+ "position, random values) are allowed."
891
+ end
892
+ end
893
+
894
+ # What the statement a foreign scan sends does with a column past fetching it: its joins, conditions and sorts.
895
+ # GROUP BY names a key by its place in the select list, so a grouped statement is read whole.
896
+ def remote_uses(node)
897
+ remote = node["Remote SQL"].to_s.gsub(PLAN_LITERAL, "''")
898
+ past_select = remote.sub(/\ASELECT\b.*?\bFROM\b/m, "")
899
+ past_select.match?(/\bGROUP\s+BY\b/i) ? remote : past_select
900
+ end
901
+
902
+ # The foreign columns a remote statement uses under the name their column_name option gives them, by their own
903
+ # names. A join or an aggregate the remote server runs names its tables under "Relations".
904
+ def local_names(foreign)
905
+ return [] if foreign.empty?
906
+
907
+ renamed = @budget.spend do |left|
908
+ @family.bounded(@connection, timeout: left) do
909
+ @connection.select_rows(
910
+ "SELECT c.relname, a.attname, o.option_value FROM pg_attribute a JOIN pg_class c ON c.oid = a.attrelid " \
911
+ "CROSS JOIN LATERAL pg_options_to_table(a.attfdwoptions) o WHERE o.option_name = 'column_name' AND NOT a.attisdropped"
912
+ )
913
+ end
914
+ end
915
+ renamed.filter_map do |table, local, far|
916
+ local if foreign.any? { |node| names?(node.values_at("Relation Name", "Relations").map(&:to_s), table) && names?([ remote_uses(node) ], far) }
917
+ end
918
+ end
919
+
920
+ # A secret name the text check passed as a literal runs only when a plan cleared the checks,
921
+ # so the plan backs a masker that a quote trick fooled.
922
+ def unvouched_literal_refusal(text, plan)
923
+ return nil unless @family.plan_checks?
924
+
925
+ offending = @secrets.named_in(text) or return nil
926
+ return nil if plan && @secrets.named_in(mask_sql_literals(text)).nil?
927
+
928
+ "Blocked: query references secret column `#{offending}`. It is read as data " \
929
+ "only when the query plan confirms it, and the plan could not be obtained, so the " \
930
+ "query is refused. Name the columns you need; `#{offending}` is never returned."
931
+ end
932
+
933
+ # A whole-row `alias.*` where any relation the alias stands for holds a secret column. An alias the
934
+ # plan does not resolve is refused when any relation in it is secret.
935
+ def whole_row_leak(exprs, aliases)
936
+ any_secret = aliases.values.flatten.any? { |rel| @secrets.relation_secret?(rel) }
937
+ exprs.each do |expr|
938
+ expr.scan(/(?:([A-Za-z_]\w*)|"((?:""|[^"])+)")\.\*/) do |bare, quoted|
939
+ al = bare || quoted.gsub('""', '"')
940
+ rels = aliases[al]
941
+ secret = rels&.find { |rel| @secrets.relation_secret?(rel) }
942
+ return secret if secret
943
+ return al if rels.nil? && any_secret
944
+ end
945
+ end
946
+ nil
947
+ end
948
+
949
+ # A secret column anywhere but a plain pass-through output. The text check refused any statement
950
+ # naming one, so a name here came from a wildcard, an expanded row reference or a view.
951
+ def secret_column_in_plan(outputs, conds)
952
+ secret = @secrets.in_schema
953
+ return nil if secret.empty?
954
+
955
+ named = ->(expr) { secret.find { |col| names?([ expr ], col) } }
956
+ conds.each { |expr| (col = named.call(expr)) and return col }
957
+ outputs.each do |expr|
958
+ next if expr.match?(PASSTHROUGH_OUTPUT)
959
+
960
+ (col = named.call(expr)) and return col
961
+ end
962
+ nil
963
+ end
964
+
965
+ # A secret base column some node passes through and the top node leaves unnamed, so redaction by position cannot
966
+ # pin it. A column handed to a node that reads by name and names it nowhere (`count(*)`) is read by nothing.
967
+ def laundered_secret_column(plan)
968
+ top = secret_base_columns(plan, plan.output.flatten)
969
+ plan.nodes.each do |node|
970
+ reader = plan.reader(node)
971
+ secret_base_columns(plan, Array(node["Output"])).each do |var|
972
+ next if top.include?(var) || (reader && !names?(plan.expressions(reader), var.last))
973
+
974
+ return var.last
975
+ end
976
+ end
977
+ nil
978
+ end
979
+
980
+ def names?(exprs, column)
981
+ pattern = SecretColumns.identifier(column)
982
+ exprs.any? { |expr| SecretColumns.fold(expr.gsub(PLAN_LITERAL, "''")).match?(pattern) }
983
+ end
984
+
985
+ def secret_base_columns(plan, output)
986
+ output.flat_map { |expr| plan.base_columns(expr) }.select { |rel, col| @secrets.of_relation(rel).include?(col) }.to_set
987
+ end
988
+
989
+ def function_names(exprs)
990
+ exprs.each_with_object(Set.new) { |expr, names| expr.scan(/([A-Za-z_]\w*)\s*\(/) { |(fn)| names << fn.downcase } }
991
+ end
992
+
993
+ def volatile_function(called)
994
+ names = called - PG_VOLATILE_ALLOWLIST
995
+ return nil if names.empty?
996
+
997
+ quoted = names.map { |n| @connection.quote(n) }.join(", ")
998
+ @budget.spend do |left|
999
+ @family.bounded(@connection, timeout: left) do
1000
+ @connection.select_values(
1001
+ "SELECT DISTINCT lower(proname) FROM pg_proc WHERE lower(proname) IN (#{quoted}) AND provolatile = 'v'"
1002
+ ).first
1003
+ end
1004
+ end
1005
+ end
1006
+ end
1007
+ end