woods 2.0.0.beta4 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +495 -471
- data/CONTRIBUTING.md +12 -2
- data/README.md +11 -26
- data/docs/AGENT_GUIDE.md +31 -12
- data/docs/AGENT_SETUP.md +17 -10
- data/docs/AUTOMATIC_MAINTENANCE.md +222 -0
- data/docs/BACKEND_MATRIX.md +13 -7
- data/docs/CLIENT_HOOKS.md +1 -1
- data/docs/CONFIGURATION_REFERENCE.md +44 -27
- data/docs/CONSOLE_MCP_SETUP.md +95 -16
- data/docs/DOCKER_SETUP.md +15 -0
- data/docs/EVALUATION.md +10 -4
- data/docs/EXTRACTOR_REFERENCE.md +14 -2
- data/docs/FAQ.md +14 -3
- data/docs/GETTING_STARTED.md +18 -17
- data/docs/INCREMENTAL_EXTRACTION.md +8 -3
- data/docs/INDEX_LAYOUT.md +2 -2
- data/docs/MCP_HTTP_TRANSPORT.md +54 -2
- data/docs/MCP_SERVERS.md +28 -11
- data/docs/MCP_TOOL_COOKBOOK.md +1 -1
- data/docs/MCP_WORKTREE_SETUP.md +13 -1
- data/docs/PUBLISHED_INDEX.md +1 -1
- data/docs/README.md +2 -1
- data/docs/RETRIEVAL_GUIDE.md +57 -8
- data/docs/SOURCE_FRESHNESS.md +1 -1
- data/docs/TOKEN_BENCHMARK.md +16 -10
- data/docs/TROUBLESHOOTING.md +133 -37
- data/docs/UPGRADING_TO_2.md +69 -7
- data/docs/WATCH_DAEMON.md +172 -17
- data/docs/WHY_WOODS.md +9 -5
- data/exe/woods-console +13 -11
- data/exe/woods-mcp-http +16 -9
- data/exe/woods-watch +5 -0
- data/lib/generators/woods/watch_generator.rb +53 -0
- data/lib/puma/plugin/woods.rb +10 -0
- data/lib/tasks/woods.rake +14 -0
- data/lib/woods/cache/cache_middleware.rb +18 -11
- data/lib/woods/console/adapter_family.rb +39 -0
- data/lib/woods/console/credential_index.rb +33 -3
- data/lib/woods/console/embedded_executor.rb +401 -43
- data/lib/woods/console/model_validator.rb +8 -0
- data/lib/woods/console/rack_middleware.rb +39 -10
- data/lib/woods/console/redactor.rb +24 -10
- data/lib/woods/console/safe_context.rb +44 -7
- data/lib/woods/console/sql_noise_stripper.rb +41 -12
- data/lib/woods/console/sql_table_scanner.rb +45 -34
- data/lib/woods/console/sql_validator.rb +37 -2
- data/lib/woods/console/stdio_transport.rb +27 -0
- data/lib/woods/extractor.rb +25 -7
- data/lib/woods/git_command.rb +6 -7
- data/lib/woods/git_provenance.rb +4 -6
- data/lib/woods/mcp/bearer_auth.rb +1 -1
- data/lib/woods/mcp/bootstrapper.rb +3 -1
- data/lib/woods/mcp/initialization_guidance.rb +1 -1
- data/lib/woods/mcp/origin_guard.rb +24 -77
- data/lib/woods/mcp/origin_policy.rb +124 -0
- data/lib/woods/mcp/server.rb +41 -9
- data/lib/woods/railtie_support.rb +8 -0
- data/lib/woods/retrieval/corpus_status.rb +46 -0
- data/lib/woods/retriever.rb +19 -7
- data/lib/woods/storage/local_corpus_stats.rb +32 -0
- data/lib/woods/storage/metadata_store.rb +20 -0
- data/lib/woods/storage/vector_store.rb +10 -0
- data/lib/woods/version.rb +1 -1
- data/lib/woods/watch/child_environment.rb +30 -0
- data/lib/woods/watch/cli.rb +91 -0
- data/lib/woods/watch/daemon.rb +55 -7
- data/lib/woods/watch/event_stream.rb +70 -0
- data/lib/woods/watch/guardian.rb +142 -0
- data/lib/woods/watch/installation/layout.rb +70 -0
- data/lib/woods/watch/installation/options.rb +128 -0
- data/lib/woods/watch/installation/planner.rb +128 -0
- data/lib/woods/watch/installation/probe.rb +101 -0
- data/lib/woods/watch/installation/receipt.rb +77 -0
- data/lib/woods/watch/installation/recovery.rb +64 -0
- data/lib/woods/watch/installation/templates.rb +58 -0
- data/lib/woods/watch/installation.rb +56 -0
- data/lib/woods/watch/lifecycle.rb +182 -0
- data/lib/woods/watch/managed_child.rb +113 -0
- data/lib/woods/watch/managed_cleanup.rb +48 -0
- data/lib/woods/watch/managed_process.rb +144 -0
- data/lib/woods/watch/puma_adapter.rb +87 -0
- data/lib/woods/watch/puma_child.rb +66 -0
- data/lib/woods/watch/supervision_records.rb +95 -0
- data/lib/woods/watch/supervision_status.rb +104 -0
- data/lib/woods/watch/supervisor.rb +161 -0
- data/lib/woods/watch/supervisor_reporting.rb +46 -0
- data/plugin/.claude-plugin/plugin.json +1 -1
- data/plugin/skills/woods-agent-enable/SKILL.md +1 -1
- data/plugin/skills/woods-diagnose/SKILL.md +88 -8
- data/plugin/skills/woods-investigate/SKILL.md +6 -6
- data/plugin/skills/woods-mcp-config/SKILL.md +43 -1
- data/plugin/skills/woods-setup/SKILL.md +66 -4
- metadata +37 -5
|
@@ -10,6 +10,7 @@ require_relative 'eval_guard'
|
|
|
10
10
|
require_relative 'input_contract'
|
|
11
11
|
require_relative 'model_validator'
|
|
12
12
|
require_relative 'safe_context'
|
|
13
|
+
require_relative 'redactor'
|
|
13
14
|
require_relative 'scope_predicate_parser'
|
|
14
15
|
require_relative 'sql_validator'
|
|
15
16
|
require_relative 'sql_noise_stripper'
|
|
@@ -92,7 +93,7 @@ module Woods
|
|
|
92
93
|
|
|
93
94
|
normalize_params!(tool, params)
|
|
94
95
|
start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
95
|
-
result = @safe_context.execute { with_mysql_quote_modes {
|
|
96
|
+
result = @safe_context.execute { with_mysql_quote_modes { dispatch_with_key_redaction(tool, params) } }
|
|
96
97
|
elapsed = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round(1)
|
|
97
98
|
|
|
98
99
|
{ 'ok' => true, 'result' => result, 'timing_ms' => elapsed }
|
|
@@ -112,6 +113,58 @@ module Woods
|
|
|
112
113
|
|
|
113
114
|
private
|
|
114
115
|
|
|
116
|
+
def dispatch_with_key_redaction(tool, params)
|
|
117
|
+
output = dispatch(tool, params)
|
|
118
|
+
return output if @safe_context.redacted_key_values.empty?
|
|
119
|
+
|
|
120
|
+
Redactor.apply(output, typed_redaction_context(tool, params))
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def typed_redaction_context(tool, params)
|
|
124
|
+
types = Hash.new { |hash, key| hash[key] = [] }
|
|
125
|
+
typed_redaction_models(tool, params).each do |name, model|
|
|
126
|
+
@safe_context.redacted_key_values.each do |pattern|
|
|
127
|
+
key = pattern['key_column']
|
|
128
|
+
types[key] << model.type_for_attribute(key) if @model_validator.columns_for(name).include?(key)
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
@safe_context.with_key_value_types(types, raw: %w[sql query].include?(tool))
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def typed_redaction_models(tool, params)
|
|
135
|
+
tables = if tool == 'sql'
|
|
136
|
+
SqlTableScanner.identifiers_in(params['sql'], dialect: sql_dialect, mysql_modes: mysql_quote_modes)
|
|
137
|
+
else
|
|
138
|
+
selected_source_tables(params)
|
|
139
|
+
end
|
|
140
|
+
@model_validator.model_names.filter_map do |name|
|
|
141
|
+
model = resolve_model(name)
|
|
142
|
+
next unless model.respond_to?(:type_for_attribute)
|
|
143
|
+
next unless typed_model_source?(model, name, params, tables)
|
|
144
|
+
|
|
145
|
+
[name, model]
|
|
146
|
+
rescue NameError
|
|
147
|
+
# Injectable registries can describe models without loading Rails.
|
|
148
|
+
next
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def typed_model_source?(model, name, params, tables)
|
|
153
|
+
return true if name == params['model']
|
|
154
|
+
return false unless model.respond_to?(:table_name)
|
|
155
|
+
|
|
156
|
+
# Match the scanner's unquoted final segment, as the identifier gates
|
|
157
|
+
# do. Keep all matching models across schemas: every possible type
|
|
158
|
+
# contributes to masking rather than selecting one ambiguous source.
|
|
159
|
+
tables.any? { |table| table.split('.').last.casecmp?(model.table_name.split('.').last) }
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def selected_source_tables(params)
|
|
163
|
+
Array(params['select'] || params['columns']).filter_map do |column|
|
|
164
|
+
column.split('.')[0...-1].join('.') if column.include?('.')
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
|
|
115
168
|
def sanitize_execution_error(error)
|
|
116
169
|
klass = error.class.name
|
|
117
170
|
# Well-known AR wrappers that contain the adapter error as their cause —
|
|
@@ -530,7 +583,7 @@ module Woods
|
|
|
530
583
|
raise ValidationError, 'columns must contain at least one item' if columns && columns.empty?
|
|
531
584
|
|
|
532
585
|
@model_validator.validate_columns!(params['model'], columns) if columns
|
|
533
|
-
refuse_orphan_eav_value_selection!(Array(columns)) if columns
|
|
586
|
+
refuse_orphan_eav_value_selection!(Array(columns), params['model']) if columns
|
|
534
587
|
model = resolve_model(params['model'])
|
|
535
588
|
limit = params.fetch('limit', 100)
|
|
536
589
|
scope = apply_scope(model, params['scope'], model_name: params['model'])
|
|
@@ -752,7 +805,7 @@ module Woods
|
|
|
752
805
|
limit = params.fetch('limit', 10)
|
|
753
806
|
|
|
754
807
|
@model_validator.validate_column!(params['model'], order_by)
|
|
755
|
-
|
|
808
|
+
refuse_protected_predicate_column!(order_by)
|
|
756
809
|
unless %w[asc desc].include?(direction)
|
|
757
810
|
raise ValidationError, "direction must be asc or desc (got #{direction.inspect})"
|
|
758
811
|
end
|
|
@@ -782,12 +835,8 @@ module Woods
|
|
|
782
835
|
sql = params['sql']
|
|
783
836
|
raise ValidationError, 'Missing required parameter: sql' unless sql
|
|
784
837
|
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
validate_protected_sql_usage!(sql)
|
|
788
|
-
# Post-validation, pre-execution TableGate — blocks every configured
|
|
789
|
-
# table even if the sql is otherwise well-formed.
|
|
790
|
-
gate_sql!(sql)
|
|
838
|
+
require_sql_dialect!
|
|
839
|
+
validate_sql_policy!(sql)
|
|
791
840
|
|
|
792
841
|
limit = params['limit']
|
|
793
842
|
# EXPLAIN's output is plan rows, not the query's own row set: wrapping
|
|
@@ -800,14 +849,61 @@ module Woods
|
|
|
800
849
|
'Resubmit without limit.'
|
|
801
850
|
end
|
|
802
851
|
|
|
803
|
-
query_sql = limit ? "SELECT * FROM (#{sql}) AS _limited LIMIT #{limit}" : sql
|
|
852
|
+
query_sql = limit ? "SELECT * FROM (\n#{sql}\n) AS _limited LIMIT #{limit}" : sql
|
|
853
|
+
validate_sql_policy!(query_sql) if limit
|
|
804
854
|
result = active_connection.select_all(query_sql)
|
|
855
|
+
validate_sql_result_types!(result)
|
|
805
856
|
|
|
806
857
|
{ 'columns' => result.columns, 'rows' => result.rows, 'count' => result.rows.size }
|
|
807
858
|
rescue SqlValidationError => e
|
|
808
859
|
raise ValidationError, e.message
|
|
809
860
|
end
|
|
810
861
|
|
|
862
|
+
def require_sql_dialect!
|
|
863
|
+
return if sql_dialect
|
|
864
|
+
|
|
865
|
+
raise ValidationError, 'Rejected: console_sql requires a recognized database adapter family; ' \
|
|
866
|
+
'use a structured Console tool.'
|
|
867
|
+
end
|
|
868
|
+
|
|
869
|
+
# A row cap changes the statement. Check both the caller's complete
|
|
870
|
+
# statement and the exact SQL sent to the adapter, including the cap.
|
|
871
|
+
def validate_sql_policy!(sql)
|
|
872
|
+
# Alias-list identity is its own policy, even when an alias also
|
|
873
|
+
# looks like a function name to the SQL validator.
|
|
874
|
+
sql_security_views(sql).each { |stripped| refuse_sql_column_alias_lists!(stripped) }
|
|
875
|
+
SqlValidator.new(dialect: sql_dialect, mysql_modes: mysql_quote_modes).validate!(sql)
|
|
876
|
+
validate_protected_sql_usage!(sql)
|
|
877
|
+
gate_sql!(sql)
|
|
878
|
+
end
|
|
879
|
+
|
|
880
|
+
# PostgreSQL represents unregistered composite/opaque types (including
|
|
881
|
+
# arrays of them) with a nil type identity. Their output headers cannot
|
|
882
|
+
# tell the redactor which nested fields are protected. This final fence
|
|
883
|
+
# complements pre-execution checks without relying on alias spelling.
|
|
884
|
+
# Legitimate custom types can also refuse while redaction is active.
|
|
885
|
+
def validate_sql_result_types!(result)
|
|
886
|
+
return unless sql_dialect == :postgres
|
|
887
|
+
return if @safe_context.redacted_columns.empty? && redacted_kv_columns.empty?
|
|
888
|
+
|
|
889
|
+
return if recognized_sql_result_types?(result)
|
|
890
|
+
|
|
891
|
+
raise ValidationError,
|
|
892
|
+
'Rejected: PostgreSQL returned an unrecognized result type that cannot preserve ' \
|
|
893
|
+
'protected field identity. ' \
|
|
894
|
+
'Select ordinary scalar columns or use a structured Console tool.'
|
|
895
|
+
end
|
|
896
|
+
|
|
897
|
+
def recognized_sql_result_types?(result)
|
|
898
|
+
types = result.column_types if result.respond_to?(:column_types)
|
|
899
|
+
return false unless types
|
|
900
|
+
|
|
901
|
+
result.columns.each_with_index.all? do |column, index|
|
|
902
|
+
type = types[index] || types[column]
|
|
903
|
+
type&.type
|
|
904
|
+
end
|
|
905
|
+
end
|
|
906
|
+
|
|
811
907
|
# @param sql [String] Validated SQL (already passed SqlValidator)
|
|
812
908
|
# @return [Boolean] true when the statement starts with EXPLAIN
|
|
813
909
|
def explain_statement?(sql)
|
|
@@ -819,16 +915,11 @@ module Woods
|
|
|
819
915
|
# and PostgreSQL quote/comment grammars differ (`\'` escapes, `#`
|
|
820
916
|
# comments); validating with the matching dialect accepts dialect-valid
|
|
821
917
|
# literals and still rejects every known bypass form. Unknown adapters
|
|
822
|
-
#
|
|
918
|
+
# remain available to structured tools but refuse raw console_sql requests.
|
|
823
919
|
#
|
|
824
920
|
# @return [Symbol, nil]
|
|
825
921
|
def sql_dialect
|
|
826
|
-
|
|
827
|
-
return :mysql if adapter.include?('mysql')
|
|
828
|
-
return :postgres if adapter.include?('postgre')
|
|
829
|
-
return :sqlite if adapter.include?('sqlite')
|
|
830
|
-
|
|
831
|
-
nil
|
|
922
|
+
AdapterFamily.for(active_connection)
|
|
832
923
|
end
|
|
833
924
|
|
|
834
925
|
# Keep session reads local to this request and execution context. Restore
|
|
@@ -899,8 +990,10 @@ module Woods
|
|
|
899
990
|
# @return [Hash] normalized, validated query clauses
|
|
900
991
|
def validated_query_clauses(model, params)
|
|
901
992
|
model_name = params['model']
|
|
993
|
+
# Keep execution and typed source inference on the same validated projection.
|
|
994
|
+
params['select'] = validated_select(params['select'], model_name) if params['select']
|
|
902
995
|
{
|
|
903
|
-
select: params['select']
|
|
996
|
+
select: params['select'],
|
|
904
997
|
joins: validated_query_joins(model, params['joins']),
|
|
905
998
|
scope: params.key?('scope') ? validated_query_scope(params['scope'], model_name, model) : nil,
|
|
906
999
|
group_by: params['group_by']&.any? ? validated_columns(params['group_by'], model_name) : nil,
|
|
@@ -961,7 +1054,7 @@ module Woods
|
|
|
961
1054
|
expressions = Array(select).map do |expr|
|
|
962
1055
|
validate_select_expression!(expr.strip, model_name)
|
|
963
1056
|
end
|
|
964
|
-
refuse_orphan_eav_value_selection!(expressions)
|
|
1057
|
+
refuse_orphan_eav_value_selection!(expressions, model_name)
|
|
965
1058
|
expressions
|
|
966
1059
|
end
|
|
967
1060
|
|
|
@@ -976,21 +1069,41 @@ module Woods
|
|
|
976
1069
|
#
|
|
977
1070
|
# @param expressions [Array<String>] validated select expressions
|
|
978
1071
|
# @raise [ValidationError] when a value column is selected without its key
|
|
979
|
-
def refuse_orphan_eav_value_selection!(expressions)
|
|
1072
|
+
def refuse_orphan_eav_value_selection!(expressions, model_name = nil)
|
|
980
1073
|
selected = directly_selected_columns(expressions)
|
|
981
1074
|
|
|
982
1075
|
@safe_context.redacted_key_values.each do |pattern|
|
|
983
|
-
|
|
984
|
-
|
|
1076
|
+
values = selected.select { |column| base_column_name(column).casecmp?(pattern['value_column']) }
|
|
1077
|
+
values.select! { |column| eav_source_has_key?(column, pattern, model_name) }
|
|
1078
|
+
next if values.empty?
|
|
1079
|
+
|
|
1080
|
+
keys = selected.select { |column| base_column_name(column).casecmp?(pattern['key_column']) }
|
|
1081
|
+
next if paired_eav_sources?(keys, values)
|
|
985
1082
|
|
|
986
1083
|
raise ValidationError,
|
|
987
1084
|
"Rejected: selecting EAV value column '#{pattern['value_column']}' without its paired " \
|
|
988
1085
|
"key column '#{pattern['key_column']}' bypasses redaction; select both columns so the " \
|
|
989
|
-
'value can be masked.'
|
|
1086
|
+
'value can be masked. Both columns must have the same unambiguous source.'
|
|
990
1087
|
end
|
|
991
1088
|
end
|
|
992
1089
|
|
|
993
|
-
|
|
1090
|
+
def eav_source_has_key?(column, pattern, model_name)
|
|
1091
|
+
columns = if column.include?('.')
|
|
1092
|
+
@model_validator.columns_for_table(column.split('.')[0...-1].join('.'))
|
|
1093
|
+
elsif model_name
|
|
1094
|
+
@model_validator.columns_for(model_name)
|
|
1095
|
+
end
|
|
1096
|
+
columns.nil? || columns.include?(pattern['key_column'])
|
|
1097
|
+
end
|
|
1098
|
+
|
|
1099
|
+
def paired_eav_sources?(keys, values)
|
|
1100
|
+
return false unless keys.one? && values.one?
|
|
1101
|
+
|
|
1102
|
+
keys.first.split('.')[0...-1] == values.first.split('.')[0...-1]
|
|
1103
|
+
end
|
|
1104
|
+
|
|
1105
|
+
# Unaliased column references, retaining source qualifiers so a joined
|
|
1106
|
+
# key cannot stand in for the key paired with another source's value.
|
|
994
1107
|
# Aggregates and aliases cannot appear here — the per-expression
|
|
995
1108
|
# validation refuses them over protected columns before this runs.
|
|
996
1109
|
#
|
|
@@ -1004,7 +1117,7 @@ module Woods
|
|
|
1004
1117
|
fn_arg, bare_col, alias_name = match.captures[1..]
|
|
1005
1118
|
next if fn_arg || alias_name
|
|
1006
1119
|
|
|
1007
|
-
|
|
1120
|
+
bare_col
|
|
1008
1121
|
end
|
|
1009
1122
|
end
|
|
1010
1123
|
|
|
@@ -1513,8 +1626,7 @@ module Woods
|
|
|
1513
1626
|
# `SqlNoiseStripper` is the same module SqlValidator uses. The
|
|
1514
1627
|
# combined single-pass strip_noise resolves comments and literals
|
|
1515
1628
|
# together so a comment marker inside a literal can't hide a keyword.
|
|
1516
|
-
|
|
1517
|
-
if SCOPE_TEMPLATE_FORBIDDEN.match?(stripped)
|
|
1629
|
+
if sql_security_views(template).any? { |stripped| SCOPE_TEMPLATE_FORBIDDEN.match?(stripped) }
|
|
1518
1630
|
raise ValidationError,
|
|
1519
1631
|
'scope template contains forbidden SQL keywords ' \
|
|
1520
1632
|
'(subqueries, UNION, time-based functions, DML/DDL are not allowed). ' \
|
|
@@ -1543,7 +1655,7 @@ module Woods
|
|
|
1543
1655
|
return unless params['columns']
|
|
1544
1656
|
|
|
1545
1657
|
@model_validator.validate_columns!(params['model'], params['columns'])
|
|
1546
|
-
refuse_orphan_eav_value_selection!(params['columns'])
|
|
1658
|
+
refuse_orphan_eav_value_selection!(params['columns'], params['model'])
|
|
1547
1659
|
end
|
|
1548
1660
|
|
|
1549
1661
|
# Raw SQL preserves redaction identity only for direct, unaliased
|
|
@@ -1551,14 +1663,29 @@ module Woods
|
|
|
1551
1663
|
# predicates, CTEs, and other result shapes can rename a protected
|
|
1552
1664
|
# value or turn it into an oracle, so they fail closed before execution.
|
|
1553
1665
|
def validate_protected_sql_usage!(sql)
|
|
1666
|
+
# This exact wrapper only preserves the inner statement's output
|
|
1667
|
+
# headers. SqlValidator has already checked balanced delimiters.
|
|
1668
|
+
limited = /\ASELECT \* FROM \(\n(.*)\n\) AS _limited LIMIT \d+\z/m.match(sql)
|
|
1669
|
+
if limited
|
|
1670
|
+
SqlValidator.new(dialect: sql_dialect, mysql_modes: mysql_quote_modes).validate!(limited[1])
|
|
1671
|
+
return validate_protected_sql_usage!(limited[1])
|
|
1672
|
+
end
|
|
1673
|
+
|
|
1674
|
+
sql_security_views(sql).each { |stripped| validate_protected_sql_view!(stripped) }
|
|
1675
|
+
end
|
|
1676
|
+
|
|
1677
|
+
def validate_protected_sql_view!(stripped)
|
|
1678
|
+
refuse_sql_column_alias_lists!(stripped)
|
|
1679
|
+
refuse_composite_sql_projection!(stripped)
|
|
1680
|
+
refuse_ambiguous_eav_sql!(stripped)
|
|
1681
|
+
refuse_protected_sql_positions!(stripped)
|
|
1554
1682
|
protected = (@safe_context.redacted_columns + redacted_kv_columns).uniq
|
|
1555
|
-
referenced = protected.select { |column| sql_identifier_referenced?(
|
|
1683
|
+
referenced = protected.select { |column| sql_identifier_referenced?(stripped, column) }
|
|
1556
1684
|
return if referenced.empty?
|
|
1557
1685
|
|
|
1558
|
-
stripped = SqlNoiseStripper.strip_noise(sql, dialect: sql_dialect || :postgres, **mysql_quote_modes)
|
|
1559
1686
|
expressions, tail = protected_sql_projection(stripped)
|
|
1560
1687
|
selected = expressions.filter_map { |expression| direct_sql_column_name(expression) }
|
|
1561
|
-
unsafe = unsafe_protected_sql_column(referenced, expressions, selected, tail)
|
|
1688
|
+
unsafe = unsafe_protected_sql_column(referenced, expressions, selected, tail, stripped)
|
|
1562
1689
|
return unless unsafe
|
|
1563
1690
|
|
|
1564
1691
|
raise ValidationError,
|
|
@@ -1567,17 +1694,225 @@ module Woods
|
|
|
1567
1694
|
'and unaliased, or use a structured Console tool.'
|
|
1568
1695
|
end
|
|
1569
1696
|
|
|
1570
|
-
|
|
1697
|
+
# Relation/CTE column lists rename fields by position, before output
|
|
1698
|
+
# headers are available. Refuse them whenever either policy is active,
|
|
1699
|
+
# independently of protected-name references or function validation.
|
|
1700
|
+
def refuse_sql_column_alias_lists!(stripped)
|
|
1701
|
+
return if @safe_context.redacted_columns.empty? && redacted_kv_columns.empty?
|
|
1702
|
+
|
|
1703
|
+
identifier = /(?:[A-Za-z_\u0080-\u{10ffff}][A-Za-z0-9_$\u0080-\u{10ffff}]*|"(?:""|[^"])+"|`(?:``|[^`])+`)/u
|
|
1704
|
+
list = /#{identifier}\s*\(\s*#{identifier}(?:\s*,\s*#{identifier})*\s*\)/
|
|
1705
|
+
cte = /#{list}\s*AS\s*(?:(?:NOT\s+)?MATERIALIZED\s*)?\(/i
|
|
1706
|
+
relation = /\A\s*(?:AS\s+)?#{list}/i
|
|
1707
|
+
renamed = stripped.match?(cte) || SqlTableScanner.relation_factors(stripped).any? do |factor|
|
|
1708
|
+
sql_relation_alias_tail(factor).match?(relation)
|
|
1709
|
+
end
|
|
1710
|
+
return unless renamed
|
|
1711
|
+
|
|
1712
|
+
raise ValidationError,
|
|
1713
|
+
'Rejected: SQL relation or CTE column alias lists cannot preserve redaction identity. ' \
|
|
1714
|
+
'Select protected columns directly and unaliased, or use a structured Console tool.'
|
|
1715
|
+
end
|
|
1716
|
+
|
|
1717
|
+
# Consume only the leading source, leaving its optional alias. Balanced
|
|
1718
|
+
# sources include subqueries, ONLY(table), and table-function arguments;
|
|
1719
|
+
# scalar functions elsewhere in the clause are not alias declarations.
|
|
1720
|
+
def sql_relation_alias_tail(factor)
|
|
1721
|
+
tokens = sql_policy_tokens(factor)
|
|
1722
|
+
index = %w[ONLY LATERAL].include?(sql_token_text(tokens, 0)) ? 1 : 0
|
|
1723
|
+
unless sql_token_text(tokens, index) == '('
|
|
1724
|
+
index += 1
|
|
1725
|
+
index += 2 while sql_token_text(tokens, index) == '.'
|
|
1726
|
+
end
|
|
1727
|
+
index = sql_after_parentheses(tokens, index) if sql_token_text(tokens, index) == '('
|
|
1728
|
+
index += 1 if sql_token_text(tokens, index) == '*'
|
|
1729
|
+
tokens[index] ? factor[tokens[index][:start]..] : ''
|
|
1730
|
+
end
|
|
1731
|
+
|
|
1732
|
+
def sql_after_parentheses(tokens, index)
|
|
1733
|
+
depth = tokens[index][:depth]
|
|
1734
|
+
closing = ((index + 1)...tokens.length).find do |offset|
|
|
1735
|
+
tokens[offset][:text] == ')' && tokens[offset][:depth] == depth
|
|
1736
|
+
end
|
|
1737
|
+
closing ? closing + 1 : tokens.length
|
|
1738
|
+
end
|
|
1739
|
+
|
|
1740
|
+
# Result metadata provides column names, not their table provenance.
|
|
1741
|
+
# A joined/derived EAV row can supply a key from a different source.
|
|
1742
|
+
# Keep direct single-source SQL reads and qualified structured joins;
|
|
1743
|
+
# refuse ambiguous SQL shapes before fetching their values.
|
|
1744
|
+
def refuse_ambiguous_eav_sql!(stripped)
|
|
1745
|
+
return if sql_eav_patterns(stripped).empty?
|
|
1746
|
+
return if SqlTableScanner.relation_factors(stripped).size <= 1
|
|
1747
|
+
return unless stripped.include?('*') || redacted_eav_value_columns.any? do |column|
|
|
1748
|
+
sql_identifier_referenced?(stripped, column)
|
|
1749
|
+
end
|
|
1750
|
+
|
|
1751
|
+
raise ValidationError,
|
|
1752
|
+
'Rejected: SQL EAV values require one unambiguous source; use a structured Console query.'
|
|
1753
|
+
end
|
|
1754
|
+
|
|
1755
|
+
# PostgreSQL can return a table alias as one composite-valued column.
|
|
1756
|
+
# Its header carries no identities for the protected fields inside it.
|
|
1757
|
+
# Reject source names used as projection values, including nested forms.
|
|
1758
|
+
def refuse_composite_sql_projection!(stripped)
|
|
1759
|
+
return if @safe_context.redacted_columns.empty? && redacted_kv_columns.empty?
|
|
1760
|
+
|
|
1761
|
+
sources = SqlTableScanner.relation_factors(stripped).flat_map { |factor| sql_relation_names(factor) }
|
|
1762
|
+
tokens = sql_policy_tokens(stripped)
|
|
1763
|
+
return unless whole_row_value?(tokens, sources)
|
|
1764
|
+
|
|
1765
|
+
raise ValidationError,
|
|
1766
|
+
'Rejected: whole-row SQL values cannot preserve protected field identity; select columns.'
|
|
1767
|
+
end
|
|
1768
|
+
|
|
1769
|
+
# Track each query's clause separately, including correlated expressions
|
|
1770
|
+
# without a FROM clause. Relation declarations and qualified columns are
|
|
1771
|
+
# not values; a source token anywhere else may be a composite row.
|
|
1772
|
+
def whole_row_value?(tokens, sources)
|
|
1773
|
+
clauses = {}
|
|
1774
|
+
query_levels = {}
|
|
1775
|
+
tokens.each_with_index.any? do |token, index|
|
|
1776
|
+
clause = sql_clause_at_token(token, clauses, query_levels)
|
|
1777
|
+
next false unless clause && !%w[FROM JOIN STRAIGHT_JOIN].include?(clause)
|
|
1778
|
+
next false unless sources.any? { |source| sql_unquote(token[:text]).casecmp?(source) }
|
|
1779
|
+
|
|
1780
|
+
!sql_nonvalue_reference?(tokens, index)
|
|
1781
|
+
end
|
|
1782
|
+
end
|
|
1783
|
+
|
|
1784
|
+
def sql_clause_at_token(token, clauses, query_levels)
|
|
1785
|
+
word, depth = token.values_at(:text, :depth)
|
|
1786
|
+
keyword = word.upcase
|
|
1787
|
+
if word == '('
|
|
1788
|
+
clauses[depth + 1] = 'EXPRESSION'
|
|
1789
|
+
query_levels.delete(depth + 1)
|
|
1790
|
+
end
|
|
1791
|
+
if word == ')'
|
|
1792
|
+
clauses.delete(depth + 1)
|
|
1793
|
+
query_levels.delete(depth + 1)
|
|
1794
|
+
end
|
|
1795
|
+
query_levels[depth] = true if keyword == 'SELECT'
|
|
1796
|
+
boundaries = %w[SELECT FROM JOIN STRAIGHT_JOIN ON WHERE HAVING ORDER GROUP LIMIT OFFSET]
|
|
1797
|
+
clauses[depth] = keyword if query_levels[depth] && boundaries.include?(keyword)
|
|
1798
|
+
clauses[depth]
|
|
1799
|
+
end
|
|
1800
|
+
|
|
1801
|
+
def sql_nonvalue_reference?(tokens, index)
|
|
1802
|
+
previous = index.positive? ? tokens[index - 1][:text].upcase : nil
|
|
1803
|
+
return true if ['.', 'AS'].include?(previous)
|
|
1804
|
+
return true if sql_token_text(tokens, index + 1) == '.' && sql_token_text(tokens, index + 2) != '*'
|
|
1805
|
+
|
|
1806
|
+
direct_sql_wildcard?(tokens, index)
|
|
1807
|
+
end
|
|
1808
|
+
|
|
1809
|
+
def sql_token_text(tokens, index)
|
|
1810
|
+
tokens[index]&.fetch(:text)&.upcase
|
|
1811
|
+
end
|
|
1812
|
+
|
|
1813
|
+
def direct_sql_wildcard?(tokens, index)
|
|
1814
|
+
return false unless sql_token_text(tokens, index + 1) == '.' && sql_token_text(tokens, index + 2) == '*'
|
|
1815
|
+
|
|
1816
|
+
previous = index.positive? ? sql_token_text(tokens, index - 1) : nil
|
|
1817
|
+
following = sql_token_text(tokens, index + 3)
|
|
1818
|
+
%w[SELECT ,].include?(previous) && [nil, ',', 'FROM'].include?(following)
|
|
1819
|
+
end
|
|
1820
|
+
|
|
1821
|
+
# Comments and literal bodies have already been removed. Token offsets
|
|
1822
|
+
# let projection and ordinal checks retain complete nested expressions.
|
|
1823
|
+
def sql_policy_tokens(stripped)
|
|
1824
|
+
depth = 0
|
|
1825
|
+
stripped.to_enum(:scan, /"(?:[^"]|"")*"|`(?:[^`]|``)*`|''|
|
|
1826
|
+
[A-Za-z_\u0080-\u{10ffff}][A-Za-z0-9_$\u0080-\u{10ffff}]*|[0-9]+|::|[^\s]/ux).map do
|
|
1827
|
+
match = Regexp.last_match
|
|
1828
|
+
word = match[0]
|
|
1829
|
+
depth -= 1 if word == ')'
|
|
1830
|
+
token = { text: word, depth: depth, start: match.begin(0), finish: match.end(0) }
|
|
1831
|
+
depth += 1 if word == '('
|
|
1832
|
+
token
|
|
1833
|
+
end
|
|
1834
|
+
end
|
|
1835
|
+
|
|
1836
|
+
def sql_unquote(identifier)
|
|
1837
|
+
identifier.sub(/\A["`]/, '').sub(/["`]\z/, '').gsub('""', '"').gsub('``', '`')
|
|
1838
|
+
end
|
|
1839
|
+
|
|
1840
|
+
def sql_select_lists(stripped)
|
|
1841
|
+
tokens = sql_policy_tokens(stripped)
|
|
1842
|
+
tokens.each_with_index.filter_map do |token, index|
|
|
1843
|
+
next unless token[:text].casecmp?('SELECT')
|
|
1844
|
+
|
|
1845
|
+
finish = tokens[(index + 1)..].find { |candidate| sql_projection_end?(candidate, token) }
|
|
1846
|
+
finish_at = finish ? finish[:start] : stripped.length
|
|
1847
|
+
{ expressions: sql_projection_expressions(stripped[token[:finish]...finish_at]),
|
|
1848
|
+
depth: token[:depth], start: token[:start], finish: finish_at }
|
|
1849
|
+
end
|
|
1850
|
+
end
|
|
1851
|
+
|
|
1852
|
+
def sql_projection_end?(candidate, token)
|
|
1853
|
+
candidate[:depth] < token[:depth] ||
|
|
1854
|
+
(candidate[:depth] == token[:depth] &&
|
|
1855
|
+
%w[FROM WHERE GROUP HAVING ORDER LIMIT UNION INTERSECT EXCEPT].include?(candidate[:text].upcase))
|
|
1856
|
+
end
|
|
1857
|
+
|
|
1858
|
+
def refuse_protected_sql_positions!(stripped)
|
|
1859
|
+
lists = sql_select_lists(stripped)
|
|
1860
|
+
sql_policy_tokens(stripped).each_cons(3) do |clause, by, position|
|
|
1861
|
+
next unless %w[ORDER GROUP].include?(clause[:text].upcase) && by[:text].casecmp?('BY')
|
|
1862
|
+
|
|
1863
|
+
list = sql_projection_before(lists, clause)
|
|
1864
|
+
next unless list
|
|
1865
|
+
next unless sql_projection_expressions(stripped[position[:start]..]).any? do |item|
|
|
1866
|
+
protected_sql_ordinal?(item, list[:expressions])
|
|
1867
|
+
end
|
|
1868
|
+
|
|
1869
|
+
raise ValidationError,
|
|
1870
|
+
'Rejected: positional SQL ordering or grouping cannot reference protected output columns.'
|
|
1871
|
+
end
|
|
1872
|
+
end
|
|
1873
|
+
|
|
1874
|
+
def sql_projection_before(lists, clause)
|
|
1875
|
+
lists.reverse.find do |candidate|
|
|
1876
|
+
candidate[:depth] == clause[:depth] && candidate[:start] < clause[:start]
|
|
1877
|
+
end
|
|
1878
|
+
end
|
|
1879
|
+
|
|
1880
|
+
def protected_sql_ordinal?(item, expressions)
|
|
1881
|
+
ordinal = /\A\s*\(*\s*([0-9]+)\s*\)*(?:\s|\z)/.match(item)
|
|
1882
|
+
return false unless ordinal && ordinal[1].to_i.positive?
|
|
1883
|
+
|
|
1884
|
+
expression = expressions[ordinal[1].to_i - 1]
|
|
1885
|
+
return false unless expression
|
|
1886
|
+
return true if expression.include?('*')
|
|
1887
|
+
|
|
1888
|
+
(@safe_context.redacted_columns + redacted_eav_value_columns).any? do |column|
|
|
1889
|
+
sql_identifier_referenced?(expression, column)
|
|
1890
|
+
end
|
|
1891
|
+
end
|
|
1892
|
+
|
|
1893
|
+
def sql_relation_names(factor)
|
|
1894
|
+
identifier = /(?:[A-Za-z_\u0080-\u{10ffff}][A-Za-z0-9_$\u0080-\u{10ffff}]*|"(?:""|[^"])+"|`(?:``|[^`])+`)/u
|
|
1895
|
+
source = /\A\s*(?:ONLY\b\s*\(?\s*)?(#{identifier})(?:\s*\.\s*(#{identifier}))?/i
|
|
1896
|
+
match = source.match(factor)
|
|
1897
|
+
names = match ? [match[2] || match[1]] : []
|
|
1898
|
+
rest = match ? factor[match.end(0)..].sub(/\A\s*\*/, '') : factor
|
|
1899
|
+
aliases = /(?:\A|\))\s*(?:AS\s+)?(#{identifier})/i
|
|
1900
|
+
names.concat(rest.scan(aliases).flatten)
|
|
1901
|
+
|
|
1902
|
+
names.map { |name| sql_unquote(name) }
|
|
1903
|
+
end
|
|
1904
|
+
|
|
1905
|
+
def unsafe_protected_sql_column(referenced, expressions, selected, tail, stripped)
|
|
1571
1906
|
referenced.find do |column|
|
|
1572
1907
|
unsafe_protected_sql_reference?(column, expressions, selected, tail)
|
|
1573
|
-
end || orphan_eav_sql_value(selected)
|
|
1908
|
+
end || orphan_eav_sql_value(selected, stripped)
|
|
1574
1909
|
end
|
|
1575
1910
|
|
|
1576
1911
|
def protected_sql_projection(stripped)
|
|
1577
|
-
|
|
1578
|
-
return [[], stripped] unless
|
|
1912
|
+
list = sql_select_lists(stripped).first
|
|
1913
|
+
return [[], stripped] unless list && list[:start].zero?
|
|
1579
1914
|
|
|
1580
|
-
[
|
|
1915
|
+
[list[:expressions], stripped[list[:finish]..]]
|
|
1581
1916
|
end
|
|
1582
1917
|
|
|
1583
1918
|
def unsafe_protected_sql_reference?(column, expressions, selected, tail)
|
|
@@ -1592,15 +1927,31 @@ module Woods
|
|
|
1592
1927
|
@safe_context.redacted_columns.include?(column) || redacted_eav_value_columns.include?(column)
|
|
1593
1928
|
end
|
|
1594
1929
|
|
|
1595
|
-
def orphan_eav_sql_value(selected)
|
|
1596
|
-
pattern =
|
|
1930
|
+
def orphan_eav_sql_value(selected, stripped)
|
|
1931
|
+
pattern = sql_eav_patterns(stripped).find do |candidate|
|
|
1597
1932
|
selected.include?(candidate['value_column']) && !selected.include?(candidate['key_column'])
|
|
1598
1933
|
end
|
|
1599
1934
|
pattern&.fetch('value_column')
|
|
1600
1935
|
end
|
|
1601
1936
|
|
|
1937
|
+
def sql_eav_patterns(stripped)
|
|
1938
|
+
tables = SqlTableScanner.identifiers_in(stripped, dialect: sql_dialect, mysql_modes: mysql_quote_modes)
|
|
1939
|
+
@safe_context.redacted_key_values.select do |pattern|
|
|
1940
|
+
tables.any? do |table|
|
|
1941
|
+
columns = @model_validator.columns_for_table(table)
|
|
1942
|
+
columns.nil? || [pattern['key_column'], pattern['value_column']].all? { |column| columns.include?(column) }
|
|
1943
|
+
end
|
|
1944
|
+
end
|
|
1945
|
+
end
|
|
1946
|
+
|
|
1602
1947
|
def sql_projection_expressions(projection)
|
|
1603
|
-
projection.
|
|
1948
|
+
commas = sql_policy_tokens(projection).select { |token| token[:text] == ',' && token[:depth].zero? }
|
|
1949
|
+
start = 0
|
|
1950
|
+
commas.map do |token|
|
|
1951
|
+
expression = projection[start...token[:start]].strip
|
|
1952
|
+
start = token[:finish]
|
|
1953
|
+
expression
|
|
1954
|
+
end + [projection[start..].strip]
|
|
1604
1955
|
end
|
|
1605
1956
|
|
|
1606
1957
|
def direct_sql_column_name(expression)
|
|
@@ -1621,8 +1972,16 @@ module Woods
|
|
|
1621
1972
|
end
|
|
1622
1973
|
|
|
1623
1974
|
def sql_identifier_referenced?(sql, column)
|
|
1624
|
-
|
|
1625
|
-
|
|
1975
|
+
sql_security_views(sql).any? do |stripped|
|
|
1976
|
+
stripped.match?(/(?<![A-Za-z0-9_$])#{Regexp.escape(column)}(?![A-Za-z0-9_$])/i)
|
|
1977
|
+
end
|
|
1978
|
+
end
|
|
1979
|
+
|
|
1980
|
+
def sql_security_views(sql)
|
|
1981
|
+
dialects = sql_dialect ? [sql_dialect] : SqlValidator::KNOWN_DIALECTS
|
|
1982
|
+
dialects.flat_map do |dialect|
|
|
1983
|
+
SqlNoiseStripper.security_views(sql, dialect: dialect, mysql_modes: sql_dialect ? mysql_quote_modes : nil)
|
|
1984
|
+
end.uniq
|
|
1626
1985
|
end
|
|
1627
1986
|
|
|
1628
1987
|
# Apply column selection to a relation.
|
|
@@ -1662,8 +2021,7 @@ module Woods
|
|
|
1662
2021
|
#
|
|
1663
2022
|
# @return [Arel::Nodes::SqlLiteral]
|
|
1664
2023
|
def random_function
|
|
1665
|
-
|
|
1666
|
-
func = adapter.include?('mysql') ? 'RAND' : 'RANDOM'
|
|
2024
|
+
func = sql_dialect == :mysql ? 'RAND' : 'RANDOM'
|
|
1667
2025
|
Arel.sql("#{func}()")
|
|
1668
2026
|
end
|
|
1669
2027
|
|
|
@@ -87,6 +87,14 @@ module Woods
|
|
|
87
87
|
validate_column!(model_name, column_name)
|
|
88
88
|
end
|
|
89
89
|
|
|
90
|
+
# Return known table columns without guessing an unknown table's shape.
|
|
91
|
+
# @param table_name [String]
|
|
92
|
+
# @return [Array<String>, nil] nil when this table is not registered
|
|
93
|
+
def columns_for_table(table_name)
|
|
94
|
+
model = @model_by_table[table_name.to_s]
|
|
95
|
+
@registry[model] if model
|
|
96
|
+
end
|
|
97
|
+
|
|
90
98
|
# List all known model names.
|
|
91
99
|
#
|
|
92
100
|
# @return [Array<String>]
|