rails-ai-context 5.35.0 → 5.35.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1024 -0
- data/CONTEXT.md +37 -7
- data/CONTRIBUTING.md +30 -1
- data/README.md +9 -9
- data/SECURITY.md +2 -2
- data/docs/ARCHITECTURE.md +12 -4
- data/docs/CLI.md +44 -15
- data/docs/COMPATIBILITY.md +34 -8
- data/docs/CONFIGURATION.md +28 -5
- data/docs/CUSTOM_TOOLS.md +8 -2
- data/docs/FAQ.md +5 -5
- data/docs/GUIDE.md +33 -65
- data/docs/INTROSPECTORS.md +4 -4
- data/docs/QUICKSTART.md +1 -1
- data/docs/RECIPES.md +5 -5
- data/docs/RELEASING.md +3 -1
- data/docs/SECURITY.md +255 -74
- data/docs/SETUP.md +1 -1
- data/docs/STANDALONE.md +2 -2
- data/docs/TOOLS.md +151 -90
- data/docs/TROUBLESHOOTING.md +123 -8
- data/docs/_config.yml +1 -1
- data/docs/adr/0008-query-runs-only-where-it-can-guard.md +38 -0
- data/docs/adr/0009-dbhub-shapes-in-ruby.md +26 -0
- data/docs/adr/0010-one-config-tool-with-sections.md +32 -0
- data/docs/index.md +2 -2
- data/exe/rails-ai-context +24 -9
- data/lib/generators/rails_ai_context/install/install_generator.rb +2 -0
- data/lib/rails_ai_context/app_databases.rb +377 -0
- data/lib/rails_ai_context/app_kind.rb +2 -0
- data/lib/rails_ai_context/cli/app_root.rb +1 -0
- data/lib/rails_ai_context/cli/tool_runner.rb +110 -99
- data/lib/rails_ai_context/configuration.rb +15 -8
- data/lib/rails_ai_context/database_family/mysql_runtime.rb +170 -0
- data/lib/rails_ai_context/database_family/postgresql_plan.rb +124 -0
- data/lib/rails_ai_context/database_family/postgresql_runtime.rb +211 -0
- data/lib/rails_ai_context/database_family/sqlite_runtime.rb +391 -0
- data/lib/rails_ai_context/database_family.rb +282 -0
- data/lib/rails_ai_context/database_file.rb +61 -7
- data/lib/rails_ai_context/database_yml.rb +7 -29
- data/lib/rails_ai_context/doctor.rb +15 -16
- data/lib/rails_ai_context/instrumentation.rb +1 -1
- data/lib/rails_ai_context/introspectors/connection_pool_introspector.rb +8 -23
- data/lib/rails_ai_context/introspectors/database_stats_introspector.rb +6 -70
- data/lib/rails_ai_context/introspectors/declared_constant.rb +19 -7
- data/lib/rails_ai_context/introspectors/env_introspector.rb +1 -1
- data/lib/rails_ai_context/introspectors/env_references.rb +1 -1
- data/lib/rails_ai_context/introspectors/job_introspector.rb +7 -7
- data/lib/rails_ai_context/introspectors/migration_introspector.rb +8 -11
- data/lib/rails_ai_context/introspectors/migration_replay/statements.rb +9 -9
- data/lib/rails_ai_context/introspectors/model_introspector.rb +33 -39
- data/lib/rails_ai_context/introspectors/multi_database_introspector.rb +33 -30
- data/lib/rails_ai_context/introspectors/node_source.rb +2 -0
- data/lib/rails_ai_context/introspectors/performance_introspector.rb +80 -24
- data/lib/rails_ai_context/introspectors/pg_partitions.rb +1 -4
- data/lib/rails_ai_context/introspectors/schema_conventions.rb +6 -8
- data/lib/rails_ai_context/introspectors/schema_dump_path.rb +12 -14
- data/lib/rails_ai_context/introspectors/schema_introspector.rb +47 -27
- data/lib/rails_ai_context/introspectors/schema_reader.rb +1 -1
- data/lib/rails_ai_context/introspectors/source_scan.rb +39 -4
- data/lib/rails_ai_context/migration_status.rb +18 -46
- data/lib/rails_ai_context/payload.rb +48 -1
- data/lib/rails_ai_context/pending_migrations.rb +3 -4
- data/lib/rails_ai_context/run_cache.rb +5 -0
- data/lib/rails_ai_context/schema_adapter.rb +13 -18
- data/lib/rails_ai_context/secret_columns.rb +506 -0
- data/lib/rails_ai_context/serializers/claude_rules_serializer.rb +2 -2
- data/lib/rails_ai_context/serializers/markdown_serializer.rb +14 -10
- data/lib/rails_ai_context/serializers/section_facts.rb +12 -2
- data/lib/rails_ai_context/serializers/stack_overview_helper.rb +5 -2
- data/lib/rails_ai_context/serializers/tool_guide_helper.rb +35 -7
- data/lib/rails_ai_context/server.rb +143 -29
- data/lib/rails_ai_context/sql_gate.rb +1007 -0
- data/lib/rails_ai_context/tasks/rails_ai_context.rake +1 -2
- data/lib/rails_ai_context/test_helper.rb +3 -1
- data/lib/rails_ai_context/tools/analyze_feature.rb +11 -7
- data/lib/rails_ai_context/tools/base_tool.rb +71 -48
- data/lib/rails_ai_context/tools/dependency_graph.rb +1 -1
- data/lib/rails_ai_context/tools/diagnose.rb +105 -42
- data/lib/rails_ai_context/tools/generate_test.rb +4 -6
- data/lib/rails_ai_context/tools/get_active_support.rb +1 -6
- data/lib/rails_ai_context/tools/get_api.rb +1 -6
- data/lib/rails_ai_context/tools/get_autoload.rb +1 -6
- data/lib/rails_ai_context/tools/get_callbacks.rb +5 -5
- data/lib/rails_ai_context/tools/get_component_catalog.rb +6 -7
- data/lib/rails_ai_context/tools/get_concern.rb +6 -6
- data/lib/rails_ai_context/tools/get_config.rb +189 -12
- data/lib/rails_ai_context/tools/get_context.rb +111 -105
- data/lib/rails_ai_context/tools/get_controllers.rb +8 -5
- data/lib/rails_ai_context/tools/get_conventions.rb +1 -6
- data/lib/rails_ai_context/tools/get_engines.rb +1 -6
- data/lib/rails_ai_context/tools/get_env.rb +1 -6
- data/lib/rails_ai_context/tools/get_env_config.rb +2 -8
- data/lib/rails_ai_context/tools/get_frontend_stack.rb +1 -6
- data/lib/rails_ai_context/tools/get_gems.rb +1 -6
- data/lib/rails_ai_context/tools/get_helper_methods.rb +4 -5
- data/lib/rails_ai_context/tools/get_i18n.rb +1 -7
- data/lib/rails_ai_context/tools/get_job_pattern.rb +1 -1
- data/lib/rails_ai_context/tools/get_mailers.rb +1 -1
- data/lib/rails_ai_context/tools/get_model_details.rb +5 -8
- data/lib/rails_ai_context/tools/get_partial_interface.rb +3 -4
- data/lib/rails_ai_context/tools/get_schema.rb +10 -8
- data/lib/rails_ai_context/tools/get_service_pattern.rb +60 -24
- data/lib/rails_ai_context/tools/get_turbo_map.rb +1 -1
- data/lib/rails_ai_context/tools/migration_advisor.rb +47 -32
- data/lib/rails_ai_context/tools/onboard.rb +24 -24
- data/lib/rails_ai_context/tools/performance_check.rb +15 -2
- data/lib/rails_ai_context/tools/query.rb +142 -1356
- data/lib/rails_ai_context/tools/read_logs.rb +1 -1
- data/lib/rails_ai_context/tools/review_changes.rb +6 -7
- data/lib/rails_ai_context/tools/runtime_info.rb +125 -88
- data/lib/rails_ai_context/tools/safe_call.rb +70 -28
- data/lib/rails_ai_context/tools/search_code.rb +0 -4
- data/lib/rails_ai_context/tools/search_docs.rb +3 -1
- data/lib/rails_ai_context/tools/security_scan.rb +9 -5
- data/lib/rails_ai_context/tools/session_context.rb +18 -12
- data/lib/rails_ai_context/tools/validate.rb +1 -0
- data/lib/rails_ai_context/tools/validate_semantics.rb +5 -5
- data/lib/rails_ai_context/version.rb +1 -1
- data/lib/rails_ai_context.rb +4 -7
- data/server.json +2 -2
- metadata +14 -3
|
@@ -0,0 +1,1007 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "set"
|
|
4
|
+
require "strscan"
|
|
5
|
+
require "timeout"
|
|
6
|
+
|
|
7
|
+
module RailsAiContext
|
|
8
|
+
# Decides whether rails_query runs a statement: checks on its text, and on PostgreSQL on its plan.
|
|
9
|
+
# The read-only transaction behind it is a second guard (ADR-0008).
|
|
10
|
+
class SqlGate
|
|
11
|
+
# sql: the comment-free text to run, or nil with the refusal. plan: the
|
|
12
|
+
# PostgreSQL plan the checks read, for the result redaction to reuse.
|
|
13
|
+
Verdict = Data.define(:sql, :refusal, :plan) do
|
|
14
|
+
def refused?
|
|
15
|
+
!refusal.nil?
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# The seconds a group of reads has between them. Each read is charged the time it took itself.
|
|
20
|
+
class Budget
|
|
21
|
+
def initialize(seconds)
|
|
22
|
+
@left = seconds
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def spent?
|
|
26
|
+
!@left.positive?
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Runs the block with the seconds left.
|
|
30
|
+
# @raise [DatabaseFamily::TimedOut] when earlier reads used them all
|
|
31
|
+
def spend
|
|
32
|
+
raise DatabaseFamily::TimedOut, "the reads beside the statement used the whole timeout" if spent?
|
|
33
|
+
|
|
34
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
35
|
+
yield @left
|
|
36
|
+
ensure
|
|
37
|
+
@left -= Process.clock_gettime(Process::CLOCK_MONOTONIC) - started if started
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# The longest statement written by hand is far under it: 100,000 ids in an IN list are 700 KB. The checks on a
|
|
42
|
+
# statement's text have text_timeout seconds to read one.
|
|
43
|
+
MAX_BYTES = 1_000_000
|
|
44
|
+
|
|
45
|
+
# The text checks ran past the time they have.
|
|
46
|
+
class TextChecksTimedOut < RailsAiContext::Error; end
|
|
47
|
+
|
|
48
|
+
# @return [String, nil] the refusal of a call that holds no statement
|
|
49
|
+
def self.blank_refusal(sql)
|
|
50
|
+
"SQL query is required." if sql.to_s.strip.empty?
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# @return [String, nil] the refusal of a statement longer than the checks read
|
|
54
|
+
def self.length_refusal(sql)
|
|
55
|
+
return nil unless sql.to_s.bytesize > MAX_BYTES
|
|
56
|
+
|
|
57
|
+
"Blocked: the statement is #{sql.bytesize} bytes long, and rails_query checks a statement of #{MAX_BYTES} bytes at most."
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# REPLACE is the MySQL statement; `replace(` is the string function every
|
|
61
|
+
# database has, so a REPLACE followed by an opening parenthesis is let be.
|
|
62
|
+
BLOCKED_KEYWORDS = /\b(INSERT|UPDATE|DELETE|DROP|ALTER|TRUNCATE|CREATE|GRANT|REVOKE|SET|COPY|MERGE|REPLACE(?!\s*\())\b/i
|
|
63
|
+
# SHOW CREATE TABLE / VIEW print a definition; every other SHOW CREATE is
|
|
64
|
+
# refused by the CREATE keyword - SHOW CREATE USER prints credential hashes.
|
|
65
|
+
SHOW_CREATE_DEFINITION = /\A\s*SHOW\s+CREATE\s+(?:TABLE|VIEW)\b/i
|
|
66
|
+
BLOCKED_CLAUSES = /\bFOR\s+(?:UPDATE|SHARE|NO\s+KEY\s+UPDATE|KEY\s+SHARE)\b|\bLOCK\s+IN\s+SHARE\s+MODE\b/i
|
|
67
|
+
# A list keyed by family holds each name under the family that has it, so a name is refused where it can be read.
|
|
68
|
+
|
|
69
|
+
# Other sessions' statements, replication hosts and users, and the row data a log holds.
|
|
70
|
+
BLOCKED_SHOWS = { mysql: /\bSHOW\s+(?:(?:FULL|ALL)\s+)?(?:GRANTS|PROCESSLIST|BINLOG|BINARY|RELAYLOG|SLAVES?|MASTER|REPLICAS?|ENGINE|
|
|
71
|
+
PROFILES?|OPEN\s+TABLES|EXPLAIN|ANALYZE|USER_STATISTICS|CLIENT_STATISTICS|WSREP_\w+)\b/ix }.freeze
|
|
72
|
+
# The plan of the statement another session is running.
|
|
73
|
+
CONNECTION_PLAN = { mysql: /\A\s*(?:EXPLAIN|DESCRIBE|DESC)\b.*\bFOR\s+CONNECTION\b/mi }.freeze
|
|
74
|
+
# A SHOW filter runs an expression for each row, and MySQL's statement timeout holds SELECT only.
|
|
75
|
+
SHOW_WHERE = /\A\s*SHOW\b.*\bWHERE\b/mi
|
|
76
|
+
MULTI_STATEMENT = /;\s*\S/
|
|
77
|
+
ALLOWED_PREFIX = /\A\s*(SELECT|WITH|SHOW|EXPLAIN|DESCRIBE|DESC)\b/i
|
|
78
|
+
|
|
79
|
+
# SELECT-callable functions that reach files or the network through the database process.
|
|
80
|
+
# To the database they are reads, so a read-only transaction lets them through.
|
|
81
|
+
BLOCKED_FUNCTIONS = {
|
|
82
|
+
postgresql: /\b(
|
|
83
|
+
pg_read_binary_file | pg_read_file |
|
|
84
|
+
pg_ls_dir | pg_ls_logdir | pg_ls_tmpdir | pg_ls_waldir | pg_ls_archive_statusdir |
|
|
85
|
+
pg_stat_file | pg_current_logfile |
|
|
86
|
+
lo_import | lo_export |
|
|
87
|
+
dblink[a-z_]*
|
|
88
|
+
)\b/ix,
|
|
89
|
+
mysql: /\b(LOAD\s+DATA | load_file)\b/ix,
|
|
90
|
+
sqlite: /\b(load_extension)\b/i
|
|
91
|
+
}.freeze
|
|
92
|
+
|
|
93
|
+
# PostgreSQL functions that return a whole table as one value, its secret columns included.
|
|
94
|
+
BLOCKED_DUMPS = { postgresql: /\b((?:query|table|schema|database|cursor)_to_xml(?:schema|_and_xmlschema)?)\s*\(/i }.freeze
|
|
95
|
+
|
|
96
|
+
# Relations that hold values sampled from other tables' columns, other sessions' statements or stored
|
|
97
|
+
# credentials, where no check on a column's name applies.
|
|
98
|
+
BLOCKED_RELATIONS = {
|
|
99
|
+
postgresql: /\b(
|
|
100
|
+
pg_stats(?:_ext(?:_exprs)?)? | pg_statistic(?:_ext(?:_data)?)? | pg_largeobject |
|
|
101
|
+
pg_stat_activity | pg_stat_get_activity | pg_stat_get_backend_activity |
|
|
102
|
+
pg_stat_statements | pg_prepared_statements | pg_cursors |
|
|
103
|
+
pg_user_mappings? | user_mapping_options | pg_subscription |
|
|
104
|
+
pg_foreign_server | pg_foreign_data_wrapper | pg_foreign_table |
|
|
105
|
+
pg_settings | pg_file_settings | pg_show_all_settings | pg_show_all_file_settings | pg_hba_file_rules | pg_ident_file_mappings
|
|
106
|
+
)\b/ix,
|
|
107
|
+
mysql: /\b(?:(column_statistics | column_stats | global_priv | processlist | performance_schema | general_log | slow_log)\b|
|
|
108
|
+
information_schema[`"]?\s*\.\s*[`"]?(innodb_trx | innodb_lock\w* | innodb_ft_\w+ |
|
|
109
|
+
user_statistics | client_statistics | profiling | optimizer_trace)\b)/ix,
|
|
110
|
+
sqlite: /\b(sqlite_stat[134] | sqlite_stmt)\b/ix
|
|
111
|
+
}.freeze
|
|
112
|
+
|
|
113
|
+
# MySQL's own schemas, read with string literals masked. An alias cannot be told from a schema without a
|
|
114
|
+
# parser, so an alias of either name is refused with them.
|
|
115
|
+
SYSTEM_SCHEMAS = { mysql: /\b(sys|mysql)[`"]?\s*\./i }.freeze
|
|
116
|
+
|
|
117
|
+
# The SQLite pragma functions that describe the schema or the database file. Every other one, and the
|
|
118
|
+
# counters under it, answer for the connection that runs the statement, which is rails_query's own.
|
|
119
|
+
SQLITE_FILE_PRAGMAS = %w[
|
|
120
|
+
table_info table_xinfo table_list index_list index_info index_xinfo foreign_key_list foreign_key_check
|
|
121
|
+
integrity_check quick_check user_version application_id schema_version page_size page_count freelist_count
|
|
122
|
+
encoding auto_vacuum journal_mode compile_options pragma_list
|
|
123
|
+
].to_set.freeze
|
|
124
|
+
SQLITE_CONNECTION_STATE = /\bpragma_(\w+)|\b(changes|total_changes|last_insert_rowid)\s*\(/i
|
|
125
|
+
|
|
126
|
+
# Catalogs refused when a statement names them. A view over one runs, and the plan checks hold the
|
|
127
|
+
# column that carries the credential (SecretColumns::CATALOG_CREDENTIALS) as a secret column.
|
|
128
|
+
BLOCKED_CATALOGS = { postgresql: /\b(#{(SecretColumns::CATALOG_CREDENTIALS.keys + %w[pg_shadow]).join("|")})\b/i }.freeze
|
|
129
|
+
|
|
130
|
+
# PostgreSQL settings whose value can be a connection string with its password, or a command line.
|
|
131
|
+
SECRET_SETTINGS = /\b(primary_conninfo|ssl_passphrase_command|archive_command|restore_command|archive_cleanup_command|recovery_end_command)\b/i
|
|
132
|
+
# A call is matched with the name it reads when that name is one plain string, and with none otherwise.
|
|
133
|
+
SETTING_CALL = /\bcurrent_setting\s*\((?:\s*'([^'\\]*)'\s*(?=[,)]))?/i
|
|
134
|
+
PLAN_SETTING_CALL = /\bcurrent_setting\((?:'((?:''|[^'])*)'::text(?=[,)]))?/
|
|
135
|
+
SHOWN_SETTING = /\A\s*SHOW\b(.*?);?\s*\z/mi
|
|
136
|
+
|
|
137
|
+
# Checked before the SELECT INTO check, which also matches, so the refusal names the disk write.
|
|
138
|
+
BLOCKED_OUTPUT = /\bINTO\s+(OUTFILE|DUMPFILE)\b/i
|
|
139
|
+
|
|
140
|
+
# PostgreSQL functions whose session or server effect outlives a read-only
|
|
141
|
+
# transaction. The plan's VOLATILE check refuses the ones not named here.
|
|
142
|
+
BLOCKED_ADMIN = { postgresql: /\b(
|
|
143
|
+
pg_terminate_backend | pg_cancel_backend |
|
|
144
|
+
pg_reload_conf | pg_rotate_logfile[a-z_]* |
|
|
145
|
+
pg_stat_reset[a-z_]* |
|
|
146
|
+
pg_switch_wal | pg_switch_xlog | pg_create_restore_point |
|
|
147
|
+
pg_create_(?:physical|logical)_replication_slot | pg_drop_replication_slot |
|
|
148
|
+
pg_replication_origin_[a-z_]* | pg_logical_emit_message |
|
|
149
|
+
pg_advisory_lock[a-z_]* | pg_advisory_unlock[a-z_]* |
|
|
150
|
+
pg_advisory_xact_lock[a-z_]* | pg_try_advisory_lock[a-z_]* |
|
|
151
|
+
pg_promote | pg_wal_replay_pause | pg_wal_replay_resume |
|
|
152
|
+
pg_xlog_replay_pause | pg_xlog_replay_resume |
|
|
153
|
+
pg_notify | set_config | txid_current[a-z_]* | pg_current_xact_id[a-z_]* |
|
|
154
|
+
pg_import_system_collations
|
|
155
|
+
)\s*\(/ix }.freeze
|
|
156
|
+
|
|
157
|
+
# GET_LOCK leaves a named lock on the pooled connection for a later caller to inherit, and the
|
|
158
|
+
# pos_wait functions block on replication. MySQL has no volatility catalogue, so these are named.
|
|
159
|
+
BLOCKED_SESSION = { mysql: /\b(
|
|
160
|
+
get_lock | release_lock | release_all_locks | is_free_lock | is_used_lock |
|
|
161
|
+
master_pos_wait | source_pos_wait
|
|
162
|
+
)\s*\(/ix }.freeze
|
|
163
|
+
|
|
164
|
+
# A value either one sets stays on the pooled connection, in or out of a read-only transaction.
|
|
165
|
+
VARIABLE_ASSIGNMENT = { mysql: /:=/ }.freeze
|
|
166
|
+
SETS_LAST_INSERT_ID = { mysql: /\blast_insert_id\s*\(\s*[^\s)]/i }.freeze
|
|
167
|
+
|
|
168
|
+
# VOLATILE built-ins that only read: pg_proc marks a reader of live server
|
|
169
|
+
# state VOLATILE too. Any other VOLATILE function in the plan is refused.
|
|
170
|
+
PG_VOLATILE_ALLOWLIST = %w[
|
|
171
|
+
random clock_timestamp gen_random_uuid timeofday uuid_generate_v4
|
|
172
|
+
pg_sleep pg_sleep_for pg_sleep_until
|
|
173
|
+
pg_total_relation_size pg_relation_size pg_table_size pg_indexes_size
|
|
174
|
+
pg_database_size pg_tablespace_size
|
|
175
|
+
pg_lock_status pg_blocking_pids pg_safe_snapshot_blocking_pids
|
|
176
|
+
pg_is_in_recovery pg_is_wal_replay_paused pg_get_wal_replay_pause_state
|
|
177
|
+
pg_current_wal_lsn pg_current_wal_insert_lsn pg_current_wal_flush_lsn
|
|
178
|
+
pg_last_wal_receive_lsn pg_last_wal_replay_lsn pg_last_xact_replay_timestamp
|
|
179
|
+
].freeze
|
|
180
|
+
|
|
181
|
+
# Any character past ASCII is an identifier character to SQLite and MySQL, a no-break space included.
|
|
182
|
+
BARE_WORD = /(?:[\p{Word}$]|[^\x00-\x7F])+/
|
|
183
|
+
NUMBER = /\A(?:\d+(?:e\d+)?|0x\h+|0b[01]+)\z/i
|
|
184
|
+
# Words a parenthesis can follow that name no table and no alias. MySQL, whose derived table
|
|
185
|
+
# takes a bare alias and a column list, reserves each one, so none can stand for an alias there.
|
|
186
|
+
NO_ALIAS = %w[
|
|
187
|
+
AND AS BETWEEN CROSS DIV ELSE EXCEPT FOR FROM GROUP HAVING IN INNER INTERSECT INTO IS JOIN LATERAL LEFT LIKE LIMIT
|
|
188
|
+
MOD NATURAL NOT ON OR ORDER PARTITION REGEXP RIGHT RLIKE STRAIGHT_JOIN THEN UNION USING WHEN WHERE XOR
|
|
189
|
+
].to_set.freeze
|
|
190
|
+
SUBQUERY_START = %w[SELECT WITH VALUES TABLE].freeze
|
|
191
|
+
# What a VALUES body may not hold if its column-alias list is to rename constants only.
|
|
192
|
+
ROW_SOURCES = %w[SELECT TABLE WITH UNION EXCEPT INTERSECT].freeze
|
|
193
|
+
|
|
194
|
+
IN_SHORTHAND = "`x IN t` is SQLite's shorthand for `x IN (SELECT * FROM t)`, and a value read through such a wildcard can be " \
|
|
195
|
+
"compared past the redaction, so write the subquery with its columns named."
|
|
196
|
+
|
|
197
|
+
# A UNION names its columns after its first SELECT, so another table's rows leave under those names.
|
|
198
|
+
# INTERSECT and EXCEPT return only rows of their first SELECT, so they stay.
|
|
199
|
+
SET_UNION = /\bUNION\b/i
|
|
200
|
+
|
|
201
|
+
# A quoted span by the quote that opens it. A doubled quote stays inside the span.
|
|
202
|
+
PLAIN_SPAN = { "'" => /'(?:''|[^'])*'?/m, '"' => /"(?:""|[^"])*"?/m, "`" => /`(?:``|[^`])*`?/m }.freeze
|
|
203
|
+
# A backslash escapes the next character: every MySQL string, and a PostgreSQL E'...' one.
|
|
204
|
+
ESCAPED_SPAN = { "'" => /'(?:\\.|''|[^'\\])*'?/m, '"' => /"(?:\\.|""|[^"\\])*"?/m }.freeze
|
|
205
|
+
# The quote of an E'...' string: the E stands right before it, as a word of its own.
|
|
206
|
+
E_QUOTE = /(?<=(?<![\p{Word}$]|[^\x00-\x7F])[eE])'/
|
|
207
|
+
# SQLite reads [name] as an identifier.
|
|
208
|
+
BRACKETED = /\[[^\]\n]*\]/
|
|
209
|
+
DOLLAR_QUOTED = /(?<![\p{Word}$])\$(\p{Word}*)\$.*?\$\1\$/m
|
|
210
|
+
DOLLAR_TAG = /(?<![\p{Word}$])\$\p{Word}*\$/
|
|
211
|
+
UNQUOTED = /[^'"`$\/\-#*\[]+/
|
|
212
|
+
|
|
213
|
+
# A constant in a plan expression. PostgreSQL's deparser always doubles
|
|
214
|
+
# quotes and never emits an E'' string, so this matches one exactly.
|
|
215
|
+
PLAN_LITERAL = /'(?:''|[^'])*'/
|
|
216
|
+
|
|
217
|
+
# Conditions and keys, which are never a plain pass-through of a column.
|
|
218
|
+
PLAN_COND_KEYS = [
|
|
219
|
+
"Filter", "Index Cond", "Recheck Cond", "Hash Cond", "Merge Cond",
|
|
220
|
+
"Join Filter", "One-Time Filter", "TID Cond", "Function Call",
|
|
221
|
+
"Sort Key", "Group Key", "Presorted Key", "Order By", "Cache Key"
|
|
222
|
+
].freeze
|
|
223
|
+
|
|
224
|
+
PLAN_IDENT = /(?:[A-Za-z_]\w*|"(?:""|[^"])*")/
|
|
225
|
+
PASSTHROUGH_OUTPUT = /\A\(?\s*(?:#{PLAN_IDENT}\.)?#{PLAN_IDENT}\s*\)?\z/
|
|
226
|
+
|
|
227
|
+
# SHOW reads a setting and PostgreSQL has no DESCRIBE, so these have no plan.
|
|
228
|
+
# Every other statement is planned, as is whatever a leading EXPLAIN would run.
|
|
229
|
+
NO_PLAN = /\A\s*(?:SHOW|DESCRIBE|DESC)\b/i
|
|
230
|
+
EXPLAIN_PREFIX = /\A\s*EXPLAIN\s+(?:\([^)]*\)\s*)?(?:ANALYZE\s+)?(?:VERBOSE\s+)?/i
|
|
231
|
+
|
|
232
|
+
# A statement's text under a pointer. A lookbehind sees the text before the pointer, and where each dollar
|
|
233
|
+
# tag last stands is read in one pass: a search for its closer from every opener is quadratic.
|
|
234
|
+
class Scanner < StringScanner
|
|
235
|
+
def initialize(text)
|
|
236
|
+
super(text, fixed_anchor: true)
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
# The [name] at the pointer. A line with no closing bracket is read once, however many brackets open on it.
|
|
240
|
+
def bracketed
|
|
241
|
+
return nil if @unclosed_to && pos < @unclosed_to
|
|
242
|
+
|
|
243
|
+
span = scan(BRACKETED)
|
|
244
|
+
@unclosed_to = pos + check(/[^\]\n]*/).bytesize unless span
|
|
245
|
+
span
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
# Whether the dollar tag at the pointer stands again after itself, to close the quote it opens.
|
|
249
|
+
def closed?(tag)
|
|
250
|
+
@last ||= {}.tap do |last|
|
|
251
|
+
tags = StringScanner.new(string)
|
|
252
|
+
last["$#{tags[1]}"] = tags.pos - 1 while tags.skip_until(/\$(?=(\p{Word}*\$))/)
|
|
253
|
+
end
|
|
254
|
+
@last.fetch(tag, -1) >= pos + tag.bytesize
|
|
255
|
+
end
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# budget: the seconds the reads the gate sends (the plan, the catalog lookups) share with the caller's other reads.
|
|
259
|
+
# text_timeout: the seconds the checks on the statement's text have to themselves.
|
|
260
|
+
def initialize(family:, secrets:, connection:, budget:, text_timeout: RailsAiContext.configuration.query_timeout)
|
|
261
|
+
@text_timeout = text_timeout
|
|
262
|
+
@family = family
|
|
263
|
+
@secrets = secrets
|
|
264
|
+
@connection = connection
|
|
265
|
+
@budget = budget
|
|
266
|
+
@backslash = family.backslash_escapes?
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
def check(sql)
|
|
270
|
+
if (refusal = self.class.blank_refusal(sql) || self.class.length_refusal(sql))
|
|
271
|
+
return refuse(refusal)
|
|
272
|
+
end
|
|
273
|
+
|
|
274
|
+
text, refusal = text_verdict(sql)
|
|
275
|
+
return refuse(refusal) if refusal
|
|
276
|
+
planned = @family.plan_checks? && !text.match?(NO_PLAN)
|
|
277
|
+
plan = @budget.spend { |left| @family.plan(@connection, text.sub(EXPLAIN_PREFIX, ""), timeout: left) } if planned
|
|
278
|
+
if planned && plan.nil?
|
|
279
|
+
return refuse("Blocked: the query is not run because its plan could not be read. On PostgreSQL " \
|
|
280
|
+
"the plan shows which columns a query returns, so no query runs without one.")
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
refusal = plan_refusal(plan, text) || unvouched_literal_refusal(text, plan)
|
|
284
|
+
refusal ? refuse(refusal) : Verdict.new(sql: text, refusal: nil, plan: plan)
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
# The text with its comments out and every quoted span copied whole, as the family reads them.
|
|
288
|
+
def self.comment_free(sql, family, backslash: family.backslash_escapes?)
|
|
289
|
+
scanner = Scanner.new(sql)
|
|
290
|
+
out = +""
|
|
291
|
+
executable = false
|
|
292
|
+
|
|
293
|
+
until scanner.eos?
|
|
294
|
+
if (span = quoted(scanner, family, backslash: backslash)&.last || scanner.scan(UNQUOTED))
|
|
295
|
+
out << span
|
|
296
|
+
elsif family.executable_comments? && scanner.scan(/\/\*!\d*/)
|
|
297
|
+
# Its content stays for the checks to read: `SELECT /*!50000 LOAD_FILE(...) */`.
|
|
298
|
+
executable = true
|
|
299
|
+
out << " "
|
|
300
|
+
elsif executable && scanner.scan(/\*\//)
|
|
301
|
+
executable = false
|
|
302
|
+
out << " "
|
|
303
|
+
elsif comment(scanner, family)
|
|
304
|
+
out << " "
|
|
305
|
+
else
|
|
306
|
+
out << scanner.getch
|
|
307
|
+
end
|
|
308
|
+
end
|
|
309
|
+
|
|
310
|
+
out.strip
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
# The quoted span that opens where the scanner stands, as the family reads it: [:string or :quoted, the span], or nil.
|
|
314
|
+
# brackets: whether [name] quotes a name in this text. backslash: whether one escapes the next character of a string.
|
|
315
|
+
def self.quoted(scanner, family, brackets: family.sqlite?, backslash: family.backslash_escapes?)
|
|
316
|
+
case scanner.peek(1)
|
|
317
|
+
when "'" then [ :string, scanner.scan(backslash || scanner.check(E_QUOTE) ? ESCAPED_SPAN["'"] : PLAIN_SPAN["'"]) ]
|
|
318
|
+
when '"' then [ :quoted, scanner.scan(backslash ? ESCAPED_SPAN['"'] : PLAIN_SPAN['"']) ]
|
|
319
|
+
when "`" then [ :quoted, scanner.scan(PLAIN_SPAN["`"]) ]
|
|
320
|
+
when "[" then brackets && (span = scanner.bracketed) ? [ :quoted, span ] : nil
|
|
321
|
+
when "$"
|
|
322
|
+
tag = scanner.check(DOLLAR_TAG) if family.dollar_quoting?
|
|
323
|
+
[ :string, scanner.scan(DOLLAR_QUOTED) ] if tag && scanner.closed?(tag)
|
|
324
|
+
end
|
|
325
|
+
end
|
|
326
|
+
|
|
327
|
+
# Skips the comment that opens where the scanner stands, as the family reads it. One that never closes runs to the end.
|
|
328
|
+
# @return [Boolean] whether one opened there
|
|
329
|
+
def self.comment(scanner, family)
|
|
330
|
+
return !scanner.skip(family.line_comment).nil? unless scanner.skip(/\/\*/)
|
|
331
|
+
|
|
332
|
+
depth = 1
|
|
333
|
+
marks = family.nested_comments? ? /\/\*|\*\// : /\*\//
|
|
334
|
+
depth += scanner.matched == "/*" ? 1 : -1 while depth.positive? && scanner.skip_until(marks)
|
|
335
|
+
scanner.terminate if depth.positive?
|
|
336
|
+
true
|
|
337
|
+
end
|
|
338
|
+
|
|
339
|
+
private
|
|
340
|
+
|
|
341
|
+
def refuse(reason)
|
|
342
|
+
Verdict.new(sql: nil, refusal: reason, plan: nil)
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
# [the comment-free text, the refusal the text checks give or nil]. The checks send nothing and hold nothing,
|
|
346
|
+
# so ending them from outside leaves nothing half done.
|
|
347
|
+
def text_verdict(sql)
|
|
348
|
+
Timeout.timeout(@text_timeout, TextChecksTimedOut) do
|
|
349
|
+
refusal = text_refusal(sql)
|
|
350
|
+
next [ nil, refusal ] if refusal
|
|
351
|
+
|
|
352
|
+
text = comment_free(sql)
|
|
353
|
+
# With no backslash in it the text reads the same both ways, and the pass above decided.
|
|
354
|
+
if @family.backslash_escapes? && text.include?("\\") && (refusal = without_backslash_escapes { text_refusal(text) })
|
|
355
|
+
refusal = "#{refusal.delete_suffix(".")}. That is how MySQL reads the statement when the session's sql_mode holds NO_BACKSLASH_ESCAPES, " \
|
|
356
|
+
"and rails_query does not read the session's mode. Write a quote inside a string doubled, so that both readings agree."
|
|
357
|
+
end
|
|
358
|
+
[ text, refusal ]
|
|
359
|
+
end
|
|
360
|
+
rescue TextChecksTimedOut
|
|
361
|
+
[ nil, "Blocked: the checks on the statement's text ran longer than query_timeout (#{@text_timeout} seconds), so the statement " \
|
|
362
|
+
"was not run. Send a shorter statement." ]
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
# MySQL reads a backslash in a string as an ordinary character under NO_BACKSLASH_ESCAPES. The block's
|
|
366
|
+
# checks read the text that way.
|
|
367
|
+
def without_backslash_escapes
|
|
368
|
+
@backslash = false
|
|
369
|
+
yield
|
|
370
|
+
ensure
|
|
371
|
+
@backslash = @family.backslash_escapes?
|
|
372
|
+
end
|
|
373
|
+
|
|
374
|
+
def text_refusal(sql)
|
|
375
|
+
# Read on the raw text too, in case a novel comment form defeats the stripping.
|
|
376
|
+
if (m = listed(BLOCKED_FUNCTIONS, sql))
|
|
377
|
+
return "Blocked: dangerous function #{m[0]} (filesystem/network primitive)"
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
cleaned = comment_free(sql).squeeze(" ")
|
|
381
|
+
# Statement checks read quote-masked text, so `LIKE '%update%'` is data. A disk write and the
|
|
382
|
+
# file functions stay on the raw text: no read-only transaction stops them.
|
|
383
|
+
masked = mask_quoted(cleaned)
|
|
384
|
+
return "Blocked: multiple statements (no semicolons)" if masked.match?(MULTI_STATEMENT)
|
|
385
|
+
# MariaDB in its MSSQL mode reads [name] as a quoted name, and no statement of this family holds a bracket otherwise.
|
|
386
|
+
if @family.mysql? && masked.include?("[")
|
|
387
|
+
return "Blocked: a `[` outside a string or a quoted name is not run. MariaDB reads it as the start of a quoted name when the " \
|
|
388
|
+
"session's sql_mode holds MSSQL, and rails_query does not read the session's mode. Quote a name in backticks."
|
|
389
|
+
end
|
|
390
|
+
# PostgreSQL's read-only transaction refuses a locking read, so a literal there is data.
|
|
391
|
+
# MySQL's runs FOR SHARE and LOCK IN SHARE MODE, so there the raw text decides.
|
|
392
|
+
locking = @family.refuses_locking_reads? ? masked : cleaned
|
|
393
|
+
return "Blocked: locking read (FOR UPDATE/SHARE or LOCK IN SHARE MODE clause)" if locking.match?(BLOCKED_CLAUSES)
|
|
394
|
+
return "Blocked: sensitive SHOW command" if listed(BLOCKED_SHOWS, cleaned)
|
|
395
|
+
if listed(CONNECTION_PLAN, masked)
|
|
396
|
+
return "Blocked: EXPLAIN FOR CONNECTION is not run. It shows the plan of the statement another session is running."
|
|
397
|
+
end
|
|
398
|
+
if masked.match?(SHOW_WHERE)
|
|
399
|
+
return "Blocked: SHOW with a WHERE clause is not run, because MySQL's statement timeout does not hold it. " \
|
|
400
|
+
"SHOW ... LIKE runs. For any other filter, SELECT from information_schema, " \
|
|
401
|
+
"for example `SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE()`."
|
|
402
|
+
end
|
|
403
|
+
return "Blocked: SELECT INTO OUTFILE / DUMPFILE writes to disk" if cleaned.match?(BLOCKED_OUTPUT)
|
|
404
|
+
# No second statement is left by here, so an INTO after a SELECT belongs to it.
|
|
405
|
+
select = masked.index(/\bSELECT\b/i)
|
|
406
|
+
if select && masked.rindex(/\bINTO\b/i).to_i > select
|
|
407
|
+
# After a WITH clause the INTO can be a write's own.
|
|
408
|
+
write = masked[/\b(?:INSERT|MERGE|REPLACE(?!\s*\())\b(?=[^;]*\bINTO\b)/i]
|
|
409
|
+
return write ? "Blocked: contains #{write}" : "Blocked: SELECT INTO creates a table"
|
|
410
|
+
end
|
|
411
|
+
|
|
412
|
+
if (m = listed(BLOCKED_FUNCTIONS, cleaned))
|
|
413
|
+
return "Blocked: dangerous function #{m[0]} (filesystem/network primitive)"
|
|
414
|
+
end
|
|
415
|
+
if (m = listed(BLOCKED_DUMPS, cleaned))
|
|
416
|
+
return dump_refusal(m[1])
|
|
417
|
+
end
|
|
418
|
+
# Matched again as MySQL folds a name, so a letter it reads as a plain one names the same relation.
|
|
419
|
+
if (m = [ cleaned, SecretColumns.fold(cleaned) ].filter_map { |read| listed(BLOCKED_RELATIONS, read) || listed(BLOCKED_CATALOGS, read) }.first)
|
|
420
|
+
return never_read_refusal(m.captures.compact.first)
|
|
421
|
+
end
|
|
422
|
+
if (m = listed(SYSTEM_SCHEMAS, mask_quoted(cleaned, :string)))
|
|
423
|
+
text_only = ->(kind, span) { kind == :string || (span.start_with?('"') && !span.match?(/\A"\w+"\z/)) }
|
|
424
|
+
return "#{never_read_refusal(m[1])} A name before a dot is read as that schema, so an alias named `#{m[1]}` needs another name." if
|
|
425
|
+
listed(SYSTEM_SCHEMAS, mask_quoted(cleaned, &text_only))
|
|
426
|
+
|
|
427
|
+
return "#{never_read_refusal(m[1])} The name stands before a dot inside double-quoted text, which MySQL reads as a name when the " \
|
|
428
|
+
"session's sql_mode holds ANSI_QUOTES, and rails_query does not read the session's mode. Write the text in single quotes."
|
|
429
|
+
end
|
|
430
|
+
table, why = @secrets.unread_table_in(cleaned)
|
|
431
|
+
return "Blocked: `#{table}` is not read: #{why}." if table
|
|
432
|
+
if (state = connection_state(cleaned))
|
|
433
|
+
return "Blocked: `#{state}` tells of the connection that runs it. rails_query runs a statement on a connection of its own and " \
|
|
434
|
+
"sets some of that connection's values itself, so the answer would not be the app's. A pragma function that describes the " \
|
|
435
|
+
"schema or the database file runs, for example `pragma_table_info`."
|
|
436
|
+
end
|
|
437
|
+
|
|
438
|
+
if (refusal = setting_refusal(cleaned))
|
|
439
|
+
return refusal
|
|
440
|
+
end
|
|
441
|
+
if (m = listed(BLOCKED_ADMIN, cleaned))
|
|
442
|
+
return "Blocked: administrative function #{m[0].sub(/\s*\(\z/, "")} has a session or server-wide " \
|
|
443
|
+
"effect that a read-only transaction does not prevent. rails_query runs " \
|
|
444
|
+
"read-only queries for inspection; it will not run server-control functions."
|
|
445
|
+
end
|
|
446
|
+
if (m = listed(BLOCKED_SESSION, cleaned))
|
|
447
|
+
return "Blocked: session function #{m[0].sub(/\s*\(\z/, "")} leaves state (a named lock) on the pooled " \
|
|
448
|
+
"connection after the query returns. rails_query will not run it."
|
|
449
|
+
end
|
|
450
|
+
if listed(VARIABLE_ASSIGNMENT, masked)
|
|
451
|
+
return "Blocked: a user variable assignment (`:=`) is not run. It leaves a value on the pooled connection after the query " \
|
|
452
|
+
"returns, which a read-only transaction does not stop."
|
|
453
|
+
end
|
|
454
|
+
if listed(SETS_LAST_INSERT_ID, cleaned)
|
|
455
|
+
return "Blocked: LAST_INSERT_ID with an argument is not run. It sets a value that stays on the pooled connection after the " \
|
|
456
|
+
"query returns, which a read-only transaction does not stop. LAST_INSERT_ID() with no argument runs."
|
|
457
|
+
end
|
|
458
|
+
|
|
459
|
+
tokens = tokens(cleaned)
|
|
460
|
+
if renaming_alias_list?(tokens)
|
|
461
|
+
return "Blocked: a column-alias list (e.g. `t(a, b, c)`) renames the columns a " \
|
|
462
|
+
"wildcard returns, so secret columns would leave under harmless names. " \
|
|
463
|
+
"Name the columns you want in the SELECT list instead."
|
|
464
|
+
end
|
|
465
|
+
if quoted_call?(tokens)
|
|
466
|
+
return "Blocked: a quoted name is followed by a parenthesis, the way a function is called. rails_query reads " \
|
|
467
|
+
"a function's name to decide whether it runs, so write the name without quotes."
|
|
468
|
+
end
|
|
469
|
+
|
|
470
|
+
# On PostgreSQL a literal is data, since the plan checks read the query itself. MySQL's quote
|
|
471
|
+
# modes and SQLite's t.'col' identifiers could fool a masker, so there the raw text decides.
|
|
472
|
+
literal_free = @family.masks_literals? ? mask_sql_literals(cleaned) : cleaned
|
|
473
|
+
|
|
474
|
+
if literal_free.match?(SET_UNION)
|
|
475
|
+
return "Blocked: UNION is not run. A UNION takes its column names from its first SELECT, " \
|
|
476
|
+
"so rows of another table would come out under those names, past the redaction that " \
|
|
477
|
+
"reads them. Query each table on its own; a recursive CTE needs UNION, so it is not run either."
|
|
478
|
+
end
|
|
479
|
+
|
|
480
|
+
if (m = masked.sub(SHOW_CREATE_DEFINITION, "SHOW").match(BLOCKED_KEYWORDS))
|
|
481
|
+
return "Blocked: contains #{m[0]}"
|
|
482
|
+
end
|
|
483
|
+
|
|
484
|
+
return "Only SELECT, WITH, SHOW, EXPLAIN, DESCRIBE allowed" unless cleaned.match?(ALLOWED_PREFIX)
|
|
485
|
+
|
|
486
|
+
if (offending = @secrets.named_in(literal_free))
|
|
487
|
+
return secret_column_refusal(offending)
|
|
488
|
+
end
|
|
489
|
+
|
|
490
|
+
# With no plan to read, a value a subquery's wildcard returns can be compared or renamed unseen. Which
|
|
491
|
+
# relation the wildcard reads is not told without a parser, so the rule is the two together.
|
|
492
|
+
wildcard = subquery_wildcard(tokens) unless @family.plan_checks?
|
|
493
|
+
if wildcard && (relation = @secrets.relation_named_in(names_text(tokens)))
|
|
494
|
+
holds = "is not run in a statement that names `#{relation}`, which holds the secret column `#{@secrets.of_relation(relation).first}`."
|
|
495
|
+
return "Blocked: `TABLE` inside a subquery #{holds} `TABLE t` reads every column of `t`, and on #{@family.name} a value read that " \
|
|
496
|
+
"way can be compared or renamed past the redaction, so SELECT the columns you need in the subquery." if wildcard == :table
|
|
497
|
+
|
|
498
|
+
return "Blocked: `IN` before a table's name #{holds} #{IN_SHORTHAND}" if wildcard == :in
|
|
499
|
+
|
|
500
|
+
return "Blocked: a `*` inside a subquery #{holds} On #{@family.name} a value read through such a wildcard can be compared or " \
|
|
501
|
+
"renamed past the redaction, so name the columns in the subquery."
|
|
502
|
+
end
|
|
503
|
+
if wildcard && wildcard != :table && (relation = string_naming_relation(tokens))
|
|
504
|
+
return "Blocked: #{wildcard == :in ? "`IN` before a table's name" : "a `*` inside a subquery"} is not run in a statement that holds " \
|
|
505
|
+
"the string '#{relation}'. SQLite reads a string as a table's name where a name can stand, and the table `#{relation}` holds the " \
|
|
506
|
+
"secret column `#{@secrets.of_relation(relation).first}`. " + (wildcard == :in ? IN_SHORTHAND :
|
|
507
|
+
"A value read through such a wildcard can be compared or renamed past the redaction, so name the columns in the subquery.")
|
|
508
|
+
end
|
|
509
|
+
|
|
510
|
+
# A definition is schema text, so SHOW CREATE VIEW still shows which tables a refused view reads.
|
|
511
|
+
# With a plan the view is told there, by its schema and name: the text does not say which schema a name is in.
|
|
512
|
+
return nil if cleaned.match?(SHOW_CREATE_DEFINITION) || @family.plan_checks?
|
|
513
|
+
|
|
514
|
+
view, why = @secrets.refused_view_in(literal_free)
|
|
515
|
+
view_refusal(view, why) if view
|
|
516
|
+
end
|
|
517
|
+
|
|
518
|
+
def view_refusal(view, why)
|
|
519
|
+
"Blocked: the view `#{view}` is not queried: #{why}. A view can rename a column past the redaction " \
|
|
520
|
+
"on #{@family.name}, so query the tables it reads."
|
|
521
|
+
end
|
|
522
|
+
|
|
523
|
+
# The plan names every relation a statement reads, through any view and in any schema. A view that stores
|
|
524
|
+
# its rows holds what its definition read, so the rule follows that definition as it follows a statement.
|
|
525
|
+
def unread_relation_refusal(plan)
|
|
526
|
+
relations = plan.relations.map(&:last)
|
|
527
|
+
unknown = plan.relations.reject { |_schema, relation| @secrets.knows?(relation) }.map(&:first)
|
|
528
|
+
beside = ->(&reads) { @budget.spend { |left| @family.bounded(@connection, timeout: left, &reads) } }
|
|
529
|
+
beside.call { @secrets.read_schemas(unknown) } if unknown.any?
|
|
530
|
+
stored = relations.select { |relation| @secrets.stored_view?(relation) }
|
|
531
|
+
beside.call { @secrets.reach(stored.join(" ")) } if stored.any?
|
|
532
|
+
|
|
533
|
+
if (relation = relations.find { |name| !@secrets.knows?(name) })
|
|
534
|
+
return "Blocked: `#{relation}` is not queried: its columns could not be read, so rails_query cannot tell which of them are secret."
|
|
535
|
+
end
|
|
536
|
+
|
|
537
|
+
relations.lazy.filter_map do |placed|
|
|
538
|
+
why = @secrets.placed_view_refusal(placed, plan.name(placed))
|
|
539
|
+
view_refusal(plan.name(placed), why) if why
|
|
540
|
+
end.first
|
|
541
|
+
end
|
|
542
|
+
|
|
543
|
+
# The family's own part of a list, matched against the text.
|
|
544
|
+
def listed(list, text)
|
|
545
|
+
list[@family.key]&.match(text)
|
|
546
|
+
end
|
|
547
|
+
|
|
548
|
+
def never_read_refusal(name)
|
|
549
|
+
"Blocked: `#{name}` is not read. It holds values sampled from other tables, other sessions' statements " \
|
|
550
|
+
"or stored credentials, and rails_query cannot tell which of them are secret." \
|
|
551
|
+
"#{" To read one setting, run SHOW with its name." if name.match?(/settings/i)}"
|
|
552
|
+
end
|
|
553
|
+
|
|
554
|
+
def connection_state(cleaned)
|
|
555
|
+
return nil unless @family.sqlite?
|
|
556
|
+
|
|
557
|
+
cleaned.to_enum(:scan, SQLITE_CONNECTION_STATE).filter_map do
|
|
558
|
+
pragma, counter = Regexp.last_match.captures
|
|
559
|
+
name = counter&.downcase || ("pragma_#{pragma.downcase}" unless SQLITE_FILE_PRAGMAS.include?(pragma.downcase))
|
|
560
|
+
name unless name.nil? || @secrets.knows?(name)
|
|
561
|
+
end.first
|
|
562
|
+
end
|
|
563
|
+
|
|
564
|
+
def dump_refusal(function)
|
|
565
|
+
"Blocked: function #{function} returns a whole table as one value, secret columns included. " \
|
|
566
|
+
"SELECT the columns you need from the table itself."
|
|
567
|
+
end
|
|
568
|
+
|
|
569
|
+
# A setting is read by SHOW and by current_setting. SHOW has no plan, so on PostgreSQL its name is read
|
|
570
|
+
# here and has to be plain: a quoted name can spell any setting.
|
|
571
|
+
def setting_refusal(cleaned)
|
|
572
|
+
return nil unless @family.postgresql?
|
|
573
|
+
|
|
574
|
+
if (m = cleaned.match(SECRET_SETTINGS))
|
|
575
|
+
return secret_setting_refusal(m[1])
|
|
576
|
+
end
|
|
577
|
+
|
|
578
|
+
called = cleaned.to_enum(:scan, SETTING_CALL).map { Regexp.last_match[1] }
|
|
579
|
+
return unread_setting_name_refusal if called.include?(nil)
|
|
580
|
+
|
|
581
|
+
shown = cleaned.match(SHOWN_SETTING)&.then { |m| m[1].strip }
|
|
582
|
+
if shown && !shown.match?(/\A[\w. ]+\z/)
|
|
583
|
+
return "Blocked: SHOW takes a setting's plain name on PostgreSQL, with no quotes around it, for example `SHOW work_mem`."
|
|
584
|
+
end
|
|
585
|
+
if shown&.casecmp?("ALL")
|
|
586
|
+
return "Blocked: SHOW ALL is not run. It lists every setting, the ones that can hold a password among them. " \
|
|
587
|
+
"SHOW one setting by its name."
|
|
588
|
+
end
|
|
589
|
+
|
|
590
|
+
secret = (called + [ shown ]).compact.find { |name| secret_setting?(name) }
|
|
591
|
+
secret_setting_refusal(secret) if secret
|
|
592
|
+
end
|
|
593
|
+
|
|
594
|
+
# An app's own setting (`app.jwt_secret`) is told by its name, as a column is.
|
|
595
|
+
def secret_setting?(name)
|
|
596
|
+
name.match?(SECRET_SETTINGS) || name.split(/[.\s]+/).any? { |part| @secrets.secret?(part) }
|
|
597
|
+
end
|
|
598
|
+
|
|
599
|
+
def secret_setting_refusal(name)
|
|
600
|
+
"Blocked: the setting `#{name}` is not read. It can hold a password, a key or a command line that carries one."
|
|
601
|
+
end
|
|
602
|
+
|
|
603
|
+
def unread_setting_name_refusal
|
|
604
|
+
"Blocked: current_setting runs only with the setting's name written as one plain string, " \
|
|
605
|
+
"for example `current_setting('work_mem')`, so that rails_query can read which setting it returns."
|
|
606
|
+
end
|
|
607
|
+
|
|
608
|
+
# An alias or an expression renames a column past the redaction that
|
|
609
|
+
# reads output names, so a statement naming a secret column is refused whole.
|
|
610
|
+
def secret_column_refusal(offending)
|
|
611
|
+
source = @secrets.generated_from(offending)
|
|
612
|
+
reads = "`#{offending}` is a generated column that reads `#{source}`" if source
|
|
613
|
+
way_out = if @secrets.encrypted?(offending)
|
|
614
|
+
if source
|
|
615
|
+
"#{reads}, an `encrypts` value, so config.query_allowed_columns does not release it."
|
|
616
|
+
else
|
|
617
|
+
"A model declares `#{offending}` with `encrypts`, so config.query_allowed_columns does not release it."
|
|
618
|
+
end
|
|
619
|
+
else
|
|
620
|
+
"#{"#{reads}. " if source}If `#{offending}` is not secret in your app, add \"#{offending}\" to query_allowed_columns in " \
|
|
621
|
+
".rails-ai-context.yml (or in config/initializers/rails_ai_context.rb where the app bundles the gem)."
|
|
622
|
+
end
|
|
623
|
+
"Blocked: query references secret column `#{offending}`. " \
|
|
624
|
+
"Name the columns you need; `#{offending}` is secret and never " \
|
|
625
|
+
"returned, and an alias or expression over it cannot be redacted after " \
|
|
626
|
+
"the query runs, so the whole query is refused. #{way_out}"
|
|
627
|
+
end
|
|
628
|
+
|
|
629
|
+
def comment_free(sql)
|
|
630
|
+
self.class.comment_free(sql, @family, backslash: @backslash)
|
|
631
|
+
end
|
|
632
|
+
|
|
633
|
+
# Every quoted and dollar-quoted span blanked to spaces of the same length, so a check reads structure
|
|
634
|
+
# only and positions still line up. only: the one kind to blank, :string or :quoted. With a block, the spans it picks.
|
|
635
|
+
def mask_quoted(text, only = nil, &picks)
|
|
636
|
+
picks ||= ->(kind, _span) { only.nil? || kind == only }
|
|
637
|
+
scanner = Scanner.new(text)
|
|
638
|
+
out = +""
|
|
639
|
+
|
|
640
|
+
until scanner.eos?
|
|
641
|
+
kind, span = self.class.quoted(scanner, @family, backslash: @backslash)
|
|
642
|
+
out << (span.nil? ? scanner.scan(UNQUOTED) || scanner.getch : picks.call(kind, span) ? " " * span.length : span)
|
|
643
|
+
end
|
|
644
|
+
|
|
645
|
+
out
|
|
646
|
+
end
|
|
647
|
+
|
|
648
|
+
# [kind, text] for each token of comment-free text: :word (bare, a keyword or a name), :quoted (a name in
|
|
649
|
+
# double quotes, backticks or brackets), :string, :number, or :punct, one character.
|
|
650
|
+
def tokens(text)
|
|
651
|
+
scanner = Scanner.new(text)
|
|
652
|
+
found = []
|
|
653
|
+
until scanner.eos?
|
|
654
|
+
next if scanner.skip(/\s+/) || scanner.skip(/[uU]&(?=["'])/)
|
|
655
|
+
|
|
656
|
+
# A bracketed name is read on every family: an alias list is told by its shape, whatever quotes its names.
|
|
657
|
+
found << (self.class.quoted(scanner, @family, brackets: true, backslash: @backslash) ||
|
|
658
|
+
scanner.scan(BARE_WORD)&.then { |span| [ span.match?(NUMBER) ? :number : :word, span ] } || [ :punct, scanner.getch ])
|
|
659
|
+
end
|
|
660
|
+
found
|
|
661
|
+
end
|
|
662
|
+
|
|
663
|
+
# PostgreSQL's string literals emptied, with the E, B, X or U& that stands before one. A quoted name stays.
|
|
664
|
+
def mask_sql_literals(text)
|
|
665
|
+
scanner = Scanner.new(text)
|
|
666
|
+
out = +""
|
|
667
|
+
|
|
668
|
+
until scanner.eos?
|
|
669
|
+
kind, span = self.class.quoted(scanner, @family)
|
|
670
|
+
if kind == :string
|
|
671
|
+
out.chop! if out.end_with?("&")
|
|
672
|
+
# Read as a byte: a character past ASCII ends in one from 0x80 up, and it is part of a name.
|
|
673
|
+
before = out.getbyte(-2)
|
|
674
|
+
out.chop! if out.end_with?(*%w[e E b B x X u U]) && !(before && (before >= 0x80 || before.chr.match?(/[\w$]/)))
|
|
675
|
+
out << "''"
|
|
676
|
+
else
|
|
677
|
+
out << (span || scanner.scan(UNQUOTED) || scanner.getch)
|
|
678
|
+
end
|
|
679
|
+
end
|
|
680
|
+
|
|
681
|
+
out
|
|
682
|
+
end
|
|
683
|
+
|
|
684
|
+
# A column-alias list renames the columns a wildcard produced, past the redaction that reads names.
|
|
685
|
+
# It is told by its shape: a parenthesised list of names after a CTE's name, or after a table's alias.
|
|
686
|
+
def renaming_alias_list?(tokens)
|
|
687
|
+
pairs = paren_pairs(tokens)
|
|
688
|
+
tokens.each_index.any? do |open|
|
|
689
|
+
close = pairs[open] if tokens[open] == [ :punct, "(" ]
|
|
690
|
+
kinds = name_list_kinds(tokens, open, close) if close
|
|
691
|
+
kinds && (cte_list?(tokens, pairs, open, close) || alias_list?(tokens, pairs, open, close, kinds))
|
|
692
|
+
end
|
|
693
|
+
end
|
|
694
|
+
|
|
695
|
+
# "(" index => its ")" index, and back.
|
|
696
|
+
def paren_pairs(tokens)
|
|
697
|
+
pairs = {}
|
|
698
|
+
stack = []
|
|
699
|
+
tokens.each_with_index do |token, i|
|
|
700
|
+
if token == [ :punct, "(" ] then stack << i
|
|
701
|
+
elsif token == [ :punct, ")" ] && (open = stack.pop)
|
|
702
|
+
pairs[open] = i
|
|
703
|
+
pairs[i] = open
|
|
704
|
+
end
|
|
705
|
+
end
|
|
706
|
+
pairs
|
|
707
|
+
end
|
|
708
|
+
|
|
709
|
+
# The kinds of the names between the parentheses, or nil when they hold anything but names and commas.
|
|
710
|
+
# A type modifier holds numbers and a record function's list holds types, so neither is one.
|
|
711
|
+
def name_list_kinds(tokens, open, close)
|
|
712
|
+
return nil unless (close - open).even?
|
|
713
|
+
|
|
714
|
+
# Read in place and left at the first token that is no name: a copy for each parenthesis of a deep nest is quadratic.
|
|
715
|
+
kinds = []
|
|
716
|
+
(open + 1).step(close - 1, 2) do |i|
|
|
717
|
+
return nil unless %i[word quoted string].include?(tokens[i][0]) && (i + 1 == close || tokens[i + 1] == [ :punct, "," ])
|
|
718
|
+
|
|
719
|
+
kinds << tokens[i][0]
|
|
720
|
+
end
|
|
721
|
+
kinds.uniq
|
|
722
|
+
end
|
|
723
|
+
|
|
724
|
+
# name (a, b) AS [NOT] [MATERIALIZED] (body). SQLite takes a string for a name here.
|
|
725
|
+
def cte_list?(tokens, pairs, open, close)
|
|
726
|
+
return false unless open.positive? && %i[word quoted string].include?(tokens[open - 1][0]) && word?(tokens[close + 1], "AS")
|
|
727
|
+
|
|
728
|
+
body = close + 2
|
|
729
|
+
body += 1 if word?(tokens[body], "NOT")
|
|
730
|
+
body += 1 if word?(tokens[body], "MATERIALIZED")
|
|
731
|
+
tokens[body] == [ :punct, "(" ] && !values_only?(tokens, body, pairs[body])
|
|
732
|
+
end
|
|
733
|
+
|
|
734
|
+
# AS t (a, b) after anything, and with no AS: (subquery) t (a, b) and FROM table t (a, b).
|
|
735
|
+
def alias_list?(tokens, pairs, open, close, kinds)
|
|
736
|
+
return false unless open >= 2 && %i[word quoted string].include?(tokens[open - 1][0])
|
|
737
|
+
|
|
738
|
+
before = open - 2
|
|
739
|
+
if word?(tokens[before], "AS")
|
|
740
|
+
source = pairs[before - 1] if before.positive? && tokens[before - 1] == [ :punct, ")" ]
|
|
741
|
+
return true unless source
|
|
742
|
+
return false if values_only?(tokens, source, before - 1)
|
|
743
|
+
|
|
744
|
+
# One name after a function call names the one column the function returns.
|
|
745
|
+
close - open > 2 || subquery_at?(tokens, source)
|
|
746
|
+
elsif kinds.include?(:string) || !name?(tokens[open - 1])
|
|
747
|
+
false
|
|
748
|
+
elsif tokens[before] == [ :punct, ")" ]
|
|
749
|
+
source = pairs[before]
|
|
750
|
+
!source.nil? && subquery_at?(tokens, source) && !values_only?(tokens, source, before)
|
|
751
|
+
else
|
|
752
|
+
word?(tokens[before], "ORDINALITY") || table_at?(tokens, before)
|
|
753
|
+
end
|
|
754
|
+
end
|
|
755
|
+
|
|
756
|
+
# Whether the token at `last` ends a table name in a FROM clause: [FROM|JOIN] [ONLY] schema.table [*].
|
|
757
|
+
def table_at?(tokens, last)
|
|
758
|
+
last -= 1 if tokens[last] == [ :punct, "*" ]
|
|
759
|
+
return false unless last.positive? && name?(tokens[last])
|
|
760
|
+
|
|
761
|
+
last -= 2 while last >= 2 && tokens[last - 1] == [ :punct, "." ] && name?(tokens[last - 2])
|
|
762
|
+
last -= 1 if last.positive? && word?(tokens[last - 1], "ONLY")
|
|
763
|
+
last.positive? && %w[FROM JOIN STRAIGHT_JOIN].any? { |anchor| word?(tokens[last - 1], anchor) }
|
|
764
|
+
end
|
|
765
|
+
|
|
766
|
+
def word?(token, keyword)
|
|
767
|
+
!token.nil? && token[0] == :word && token[1].casecmp?(keyword)
|
|
768
|
+
end
|
|
769
|
+
|
|
770
|
+
# A quoted name, or a bare word that can name a table or an alias.
|
|
771
|
+
def name?(token)
|
|
772
|
+
token[0] == :quoted || (token[0] == :word && !NO_ALIAS.include?(token[1].upcase))
|
|
773
|
+
end
|
|
774
|
+
|
|
775
|
+
# A quoted name can spell any function, past every check that reads a function's name.
|
|
776
|
+
def quoted_call?(tokens)
|
|
777
|
+
tokens.each_index.any? do |i|
|
|
778
|
+
following = i + (word?(tokens[i + 1], "UESCAPE") ? 3 : 1)
|
|
779
|
+
tokens[i][0] == :quoted && tokens[following] == [ :punct, "(" ]
|
|
780
|
+
end
|
|
781
|
+
end
|
|
782
|
+
|
|
783
|
+
def subquery_at?(tokens, open)
|
|
784
|
+
following = tokens[open + 1]
|
|
785
|
+
following == [ :punct, "(" ] || SUBQUERY_START.any? { |start| word?(following, start) }
|
|
786
|
+
end
|
|
787
|
+
|
|
788
|
+
# A list over a body of VALUES rows renames constants, so it is left alone.
|
|
789
|
+
def values_only?(tokens, open, close)
|
|
790
|
+
return false unless close && word?(tokens[open + 1], "VALUES")
|
|
791
|
+
|
|
792
|
+
before = row_sources_before(tokens)
|
|
793
|
+
before[close] == before[open + 2]
|
|
794
|
+
end
|
|
795
|
+
|
|
796
|
+
# How many row-source words stand before each token, so that no body is scanned for them on its own.
|
|
797
|
+
def row_sources_before(tokens)
|
|
798
|
+
(@row_sources ||= {}.compare_by_identity)[tokens] ||= tokens.each_with_object([ 0 ]) do |token, counts|
|
|
799
|
+
counts << counts.last + (ROW_SOURCES.any? { |source| word?(token, source) } ? 1 : 0)
|
|
800
|
+
end
|
|
801
|
+
end
|
|
802
|
+
|
|
803
|
+
# :star for a select-list wildcard inside parentheses, :table for `TABLE t` there off SQLite, :in for SQLite's
|
|
804
|
+
# `x IN t` anywhere, else nil. `count(*)` reads no column and EXISTS returns none, so both are left alone.
|
|
805
|
+
def subquery_wildcard(tokens)
|
|
806
|
+
enclosing = []
|
|
807
|
+
tokens.each_with_index do |token, i|
|
|
808
|
+
following = tokens[i + 1]
|
|
809
|
+
return :in if @family.sqlite? && word?(token, "IN") && following && following != [ :punct, "(" ]
|
|
810
|
+
|
|
811
|
+
enclosing << i if token == [ :punct, "(" ]
|
|
812
|
+
enclosing.pop if token == [ :punct, ")" ]
|
|
813
|
+
next if enclosing.empty? || (enclosing.last.positive? && word?(tokens[enclosing.last - 1], "EXISTS"))
|
|
814
|
+
|
|
815
|
+
star = token == [ :punct, "*" ] && (following.nil? || [ ",", ")" ].include?(following[1]) || word?(following, "FROM")) &&
|
|
816
|
+
!(tokens[i - 1] == [ :punct, "(" ] && following == [ :punct, ")" ])
|
|
817
|
+
return :star if star
|
|
818
|
+
return :table if !@family.sqlite? && word?(token, "TABLE") && tokens[i - 1] != [ :punct, "." ]
|
|
819
|
+
end
|
|
820
|
+
nil
|
|
821
|
+
end
|
|
822
|
+
|
|
823
|
+
# The statement with each string emptied: where a string names a relation, string_naming_relation reads it.
|
|
824
|
+
def names_text(tokens)
|
|
825
|
+
tokens.map { |kind, text| kind == :string ? "''" : text }.join(" ")
|
|
826
|
+
end
|
|
827
|
+
|
|
828
|
+
# SQLite reads a string as a name wherever a name can stand, so a string equal to such a relation's name is one.
|
|
829
|
+
def string_naming_relation(tokens)
|
|
830
|
+
return nil unless @family.sqlite?
|
|
831
|
+
|
|
832
|
+
tokens.each do |kind, text|
|
|
833
|
+
name = text[1..].delete_suffix("'").gsub("''", "'") if kind == :string
|
|
834
|
+
return name if name && @secrets.relation_secret?(name)
|
|
835
|
+
end
|
|
836
|
+
nil
|
|
837
|
+
end
|
|
838
|
+
|
|
839
|
+
def plan_refusal(top, text)
|
|
840
|
+
return nil unless top
|
|
841
|
+
|
|
842
|
+
plan = DatabaseFamily::PostgresqlPlan.new(top)
|
|
843
|
+
refusal = unread_relation_refusal(plan)
|
|
844
|
+
return refusal if refusal
|
|
845
|
+
|
|
846
|
+
raw = plan.nodes.flat_map { |node| plan.expressions(node) }
|
|
847
|
+
outputs = []
|
|
848
|
+
conds = []
|
|
849
|
+
plan.nodes.each do |node|
|
|
850
|
+
outputs.concat(Array(node["Output"]).map { |e| e.to_s.gsub(PLAN_LITERAL, "''") })
|
|
851
|
+
PLAN_COND_KEYS.each { |k| conds.concat(Array(node[k]).map { |e| e.to_s.gsub(PLAN_LITERAL, "''") }) }
|
|
852
|
+
end
|
|
853
|
+
foreign = plan.nodes.select { |node| node["Remote SQL"] }
|
|
854
|
+
conds.concat(foreign.map { |node| remote_uses(node) }, local_names(foreign))
|
|
855
|
+
# EXPLAIN prints no row of a VALUES list and no LIMIT, so the functions the text names are read too.
|
|
856
|
+
called = function_names(raw.map { |expr| expr.gsub(PLAN_LITERAL, "''") } + [ mask_sql_literals(text) ])
|
|
857
|
+
|
|
858
|
+
# A view reaches a relation or a function the text check did not see.
|
|
859
|
+
sources = plan.nodes.flat_map { |node| node.values_at("Relation Name", "Function Name") }.compact + called.to_a
|
|
860
|
+
if (name = sources.find { |source| source.match?(/\A#{BLOCKED_RELATIONS.fetch(:postgresql)}\z/) })
|
|
861
|
+
return never_read_refusal(name)
|
|
862
|
+
end
|
|
863
|
+
if (name = called.find { |function| "#{function}(".match?(/\A#{BLOCKED_DUMPS.fetch(:postgresql)}\z/) })
|
|
864
|
+
return dump_refusal(name)
|
|
865
|
+
end
|
|
866
|
+
settings = raw.flat_map { |expr| expr.to_enum(:scan, PLAN_SETTING_CALL).map { Regexp.last_match[1] } }
|
|
867
|
+
return unread_setting_name_refusal if settings.include?(nil)
|
|
868
|
+
if (name = settings.find { |setting| secret_setting?(setting) })
|
|
869
|
+
return secret_setting_refusal(name)
|
|
870
|
+
end
|
|
871
|
+
if (rel = whole_row_leak(outputs + conds, plan.aliases))
|
|
872
|
+
return "Blocked: the query serialises a whole row of `#{plan.name(rel)}`, which carries every " \
|
|
873
|
+
"column including the secret ones. Name the columns you need in the SELECT list."
|
|
874
|
+
end
|
|
875
|
+
if (col = secret_column_in_plan(outputs, conds))
|
|
876
|
+
return "Blocked: the query's plan uses the secret column `#{col}` (expanded from a " \
|
|
877
|
+
"wildcard or a row reference). Name the other columns you need; `#{col}` is " \
|
|
878
|
+
"never returned."
|
|
879
|
+
end
|
|
880
|
+
if (col = laundered_secret_column(plan))
|
|
881
|
+
return "Blocked: the query passes the secret column `#{col}` through a CTE, subquery " \
|
|
882
|
+
"or set operation whose output the tool cannot map to a column, so it could leave " \
|
|
883
|
+
"unredacted. Name the other columns you need; `#{col}` is never returned."
|
|
884
|
+
end
|
|
885
|
+
if (fn = volatile_function(called))
|
|
886
|
+
"Blocked: function #{fn} is VOLATILE (pg_proc.provolatile = 'v') and not on " \
|
|
887
|
+
"rails_query's list of read-only VOLATILE functions, so it is not run. Some VOLATILE " \
|
|
888
|
+
"functions change server or session state in ways a read-only transaction does not " \
|
|
889
|
+
"stop; only those known to just read state (sizes, lock status, recovery and WAL " \
|
|
890
|
+
"position, random values) are allowed."
|
|
891
|
+
end
|
|
892
|
+
end
|
|
893
|
+
|
|
894
|
+
# What the statement a foreign scan sends does with a column past fetching it: its joins, conditions and sorts.
|
|
895
|
+
# GROUP BY names a key by its place in the select list, so a grouped statement is read whole.
|
|
896
|
+
def remote_uses(node)
|
|
897
|
+
remote = node["Remote SQL"].to_s.gsub(PLAN_LITERAL, "''")
|
|
898
|
+
past_select = remote.sub(/\ASELECT\b.*?\bFROM\b/m, "")
|
|
899
|
+
past_select.match?(/\bGROUP\s+BY\b/i) ? remote : past_select
|
|
900
|
+
end
|
|
901
|
+
|
|
902
|
+
# The foreign columns a remote statement uses under the name their column_name option gives them, by their own
|
|
903
|
+
# names. A join or an aggregate the remote server runs names its tables under "Relations".
|
|
904
|
+
def local_names(foreign)
|
|
905
|
+
return [] if foreign.empty?
|
|
906
|
+
|
|
907
|
+
renamed = @budget.spend do |left|
|
|
908
|
+
@family.bounded(@connection, timeout: left) do
|
|
909
|
+
@connection.select_rows(
|
|
910
|
+
"SELECT c.relname, a.attname, o.option_value FROM pg_attribute a JOIN pg_class c ON c.oid = a.attrelid " \
|
|
911
|
+
"CROSS JOIN LATERAL pg_options_to_table(a.attfdwoptions) o WHERE o.option_name = 'column_name' AND NOT a.attisdropped"
|
|
912
|
+
)
|
|
913
|
+
end
|
|
914
|
+
end
|
|
915
|
+
renamed.filter_map do |table, local, far|
|
|
916
|
+
local if foreign.any? { |node| names?(node.values_at("Relation Name", "Relations").map(&:to_s), table) && names?([ remote_uses(node) ], far) }
|
|
917
|
+
end
|
|
918
|
+
end
|
|
919
|
+
|
|
920
|
+
# A secret name the text check passed as a literal runs only when a plan cleared the checks,
|
|
921
|
+
# so the plan backs a masker that a quote trick fooled.
|
|
922
|
+
def unvouched_literal_refusal(text, plan)
|
|
923
|
+
return nil unless @family.plan_checks?
|
|
924
|
+
|
|
925
|
+
offending = @secrets.named_in(text) or return nil
|
|
926
|
+
return nil if plan && @secrets.named_in(mask_sql_literals(text)).nil?
|
|
927
|
+
|
|
928
|
+
"Blocked: query references secret column `#{offending}`. It is read as data " \
|
|
929
|
+
"only when the query plan confirms it, and the plan could not be obtained, so the " \
|
|
930
|
+
"query is refused. Name the columns you need; `#{offending}` is never returned."
|
|
931
|
+
end
|
|
932
|
+
|
|
933
|
+
# A whole-row `alias.*` where any relation the alias stands for holds a secret column. An alias the
|
|
934
|
+
# plan does not resolve is refused when any relation in it is secret.
|
|
935
|
+
def whole_row_leak(exprs, aliases)
|
|
936
|
+
any_secret = aliases.values.flatten.any? { |rel| @secrets.relation_secret?(rel) }
|
|
937
|
+
exprs.each do |expr|
|
|
938
|
+
expr.scan(/(?:([A-Za-z_]\w*)|"((?:""|[^"])+)")\.\*/) do |bare, quoted|
|
|
939
|
+
al = bare || quoted.gsub('""', '"')
|
|
940
|
+
rels = aliases[al]
|
|
941
|
+
secret = rels&.find { |rel| @secrets.relation_secret?(rel) }
|
|
942
|
+
return secret if secret
|
|
943
|
+
return al if rels.nil? && any_secret
|
|
944
|
+
end
|
|
945
|
+
end
|
|
946
|
+
nil
|
|
947
|
+
end
|
|
948
|
+
|
|
949
|
+
# A secret column anywhere but a plain pass-through output. The text check refused any statement
|
|
950
|
+
# naming one, so a name here came from a wildcard, an expanded row reference or a view.
|
|
951
|
+
def secret_column_in_plan(outputs, conds)
|
|
952
|
+
secret = @secrets.in_schema
|
|
953
|
+
return nil if secret.empty?
|
|
954
|
+
|
|
955
|
+
named = ->(expr) { secret.find { |col| names?([ expr ], col) } }
|
|
956
|
+
conds.each { |expr| (col = named.call(expr)) and return col }
|
|
957
|
+
outputs.each do |expr|
|
|
958
|
+
next if expr.match?(PASSTHROUGH_OUTPUT)
|
|
959
|
+
|
|
960
|
+
(col = named.call(expr)) and return col
|
|
961
|
+
end
|
|
962
|
+
nil
|
|
963
|
+
end
|
|
964
|
+
|
|
965
|
+
# A secret base column some node passes through and the top node leaves unnamed, so redaction by position cannot
|
|
966
|
+
# pin it. A column handed to a node that reads by name and names it nowhere (`count(*)`) is read by nothing.
|
|
967
|
+
def laundered_secret_column(plan)
|
|
968
|
+
top = secret_base_columns(plan, plan.output.flatten)
|
|
969
|
+
plan.nodes.each do |node|
|
|
970
|
+
reader = plan.reader(node)
|
|
971
|
+
secret_base_columns(plan, Array(node["Output"])).each do |var|
|
|
972
|
+
next if top.include?(var) || (reader && !names?(plan.expressions(reader), var.last))
|
|
973
|
+
|
|
974
|
+
return var.last
|
|
975
|
+
end
|
|
976
|
+
end
|
|
977
|
+
nil
|
|
978
|
+
end
|
|
979
|
+
|
|
980
|
+
def names?(exprs, column)
|
|
981
|
+
pattern = SecretColumns.identifier(column)
|
|
982
|
+
exprs.any? { |expr| SecretColumns.fold(expr.gsub(PLAN_LITERAL, "''")).match?(pattern) }
|
|
983
|
+
end
|
|
984
|
+
|
|
985
|
+
def secret_base_columns(plan, output)
|
|
986
|
+
output.flat_map { |expr| plan.base_columns(expr) }.select { |rel, col| @secrets.of_relation(rel).include?(col) }.to_set
|
|
987
|
+
end
|
|
988
|
+
|
|
989
|
+
def function_names(exprs)
|
|
990
|
+
exprs.each_with_object(Set.new) { |expr, names| expr.scan(/([A-Za-z_]\w*)\s*\(/) { |(fn)| names << fn.downcase } }
|
|
991
|
+
end
|
|
992
|
+
|
|
993
|
+
def volatile_function(called)
|
|
994
|
+
names = called - PG_VOLATILE_ALLOWLIST
|
|
995
|
+
return nil if names.empty?
|
|
996
|
+
|
|
997
|
+
quoted = names.map { |n| @connection.quote(n) }.join(", ")
|
|
998
|
+
@budget.spend do |left|
|
|
999
|
+
@family.bounded(@connection, timeout: left) do
|
|
1000
|
+
@connection.select_values(
|
|
1001
|
+
"SELECT DISTINCT lower(proname) FROM pg_proc WHERE lower(proname) IN (#{quoted}) AND provolatile = 'v'"
|
|
1002
|
+
).first
|
|
1003
|
+
end
|
|
1004
|
+
end
|
|
1005
|
+
end
|
|
1006
|
+
end
|
|
1007
|
+
end
|