woods 2.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +108 -0
  3. data/CONTRIBUTING.md +135 -10
  4. data/README.md +1 -1
  5. data/docs/AGENT_GUIDE.md +19 -0
  6. data/docs/AGENT_SETUP.md +22 -2
  7. data/docs/BACKEND_MATRIX.md +7 -0
  8. data/docs/CLIENT_HOOKS.md +6 -0
  9. data/docs/CONFIGURATION_REFERENCE.md +133 -18
  10. data/docs/CONSOLE_MCP_SETUP.md +141 -12
  11. data/docs/EMBEDDING_MODELS.md +16 -19
  12. data/docs/EXTRACTOR_REFERENCE.md +219 -21
  13. data/docs/FAQ.md +11 -25
  14. data/docs/GETTING_STARTED.md +7 -1
  15. data/docs/INCREMENTAL_EXTRACTION.md +261 -19
  16. data/docs/INDEX_LAYOUT.md +5 -0
  17. data/docs/INTERNALS.md +9 -0
  18. data/docs/MCP_HTTP_TRANSPORT.md +59 -2
  19. data/docs/MCP_SERVERS.md +87 -8
  20. data/docs/MCP_TOOL_COOKBOOK.md +13 -55
  21. data/docs/NOTION_INTEGRATION.md +7 -1
  22. data/docs/PUBLISHED_INDEX.md +6 -0
  23. data/docs/README.md +6 -1
  24. data/docs/RETRIEVAL_GUIDE.md +17 -0
  25. data/docs/SOURCE_FRESHNESS.md +157 -5
  26. data/docs/TOKEN_BENCHMARK.md +10 -18
  27. data/docs/TROUBLESHOOTING.md +70 -14
  28. data/docs/UNBLOCKED_INTEGRATION.md +60 -8
  29. data/docs/UPGRADING_TO_2.md +184 -9
  30. data/docs/WATCH_DAEMON.md +97 -14
  31. data/exe/woods-console-mcp +2 -2
  32. data/exe/woods-mcp-http +16 -9
  33. data/lib/generators/woods/templates/woods.rb.tt +2 -1
  34. data/lib/tasks/woods.rake +23 -7
  35. data/lib/tasks/woods_checks.rake +2 -2
  36. data/lib/woods/agent_configuration/cli.rb +1 -1
  37. data/lib/woods/agent_configuration/layout.rb +16 -2
  38. data/lib/woods/agent_configuration/plan.rb +13 -3
  39. data/lib/woods/agent_configuration/planner_validation.rb +4 -2
  40. data/lib/woods/agent_configuration/preflight.rb +5 -3
  41. data/lib/woods/builder.rb +17 -57
  42. data/lib/woods/cache/cache_middleware.rb +68 -41
  43. data/lib/woods/chunking/contributor_chunks.rb +119 -0
  44. data/lib/woods/chunking/semantic_chunker.rb +44 -21
  45. data/lib/woods/console/adapter_family.rb +39 -0
  46. data/lib/woods/console/connection_manager.rb +56 -3
  47. data/lib/woods/console/credential_index.rb +33 -3
  48. data/lib/woods/console/embedded_executor.rb +431 -48
  49. data/lib/woods/console/model_validator.rb +8 -0
  50. data/lib/woods/console/rack_middleware.rb +68 -11
  51. data/lib/woods/console/redactor.rb +24 -10
  52. data/lib/woods/console/safe_context.rb +44 -7
  53. data/lib/woods/console/sql_noise_stripper.rb +41 -12
  54. data/lib/woods/console/sql_table_scanner.rb +45 -34
  55. data/lib/woods/console/sql_validator.rb +37 -2
  56. data/lib/woods/dependency_graph.rb +34 -10
  57. data/lib/woods/embedding/fake.rb +12 -0
  58. data/lib/woods/embedding/indexer.rb +195 -98
  59. data/lib/woods/embedding/input_budget.rb +67 -0
  60. data/lib/woods/embedding/openai.rb +70 -20
  61. data/lib/woods/embedding/provider.rb +37 -25
  62. data/lib/woods/embedding/text_preparer.rb +76 -32
  63. data/lib/woods/embedding/token_counter.rb +18 -81
  64. data/lib/woods/embedding/vector_configuration.rb +48 -0
  65. data/lib/woods/extraction_identities.rb +175 -0
  66. data/lib/woods/extractor.rb +304 -107
  67. data/lib/woods/extractors/action_cable_extractor.rb +8 -3
  68. data/lib/woods/extractors/assigned_value_discovery.rb +74 -0
  69. data/lib/woods/extractors/class_declarations.rb +121 -0
  70. data/lib/woods/extractors/configuration_extractor.rb +11 -3
  71. data/lib/woods/extractors/declaration_ancestry.rb +92 -0
  72. data/lib/woods/extractors/event_extractor.rb +8 -0
  73. data/lib/woods/extractors/graphql_extractor.rb +134 -77
  74. data/lib/woods/extractors/job_extractor.rb +5 -1
  75. data/lib/woods/extractors/lib_extractor.rb +132 -15
  76. data/lib/woods/extractors/mailer_extractor.rb +3 -5
  77. data/lib/woods/extractors/manager_extractor.rb +7 -21
  78. data/lib/woods/extractors/migration_declaration.rb +87 -0
  79. data/lib/woods/extractors/migration_extractor.rb +5 -39
  80. data/lib/woods/extractors/phlex_extractor.rb +6 -2
  81. data/lib/woods/extractors/policy_extractor.rb +9 -5
  82. data/lib/woods/extractors/poro_extractor.rb +112 -53
  83. data/lib/woods/extractors/pundit_extractor.rb +11 -6
  84. data/lib/woods/extractors/scheduled_job_extractor.rb +45 -4
  85. data/lib/woods/extractors/serializer_extractor.rb +34 -22
  86. data/lib/woods/extractors/shared_utility_methods.rb +18 -1
  87. data/lib/woods/extractors/source_nesting.rb +142 -106
  88. data/lib/woods/extractors/standalone_module_discovery.rb +123 -0
  89. data/lib/woods/extractors/state_machine_extractor.rb +46 -40
  90. data/lib/woods/extractors/view_component_extractor.rb +9 -7
  91. data/lib/woods/flow_assembler.rb +4 -1
  92. data/lib/woods/generation.rb +25 -0
  93. data/lib/woods/hooks/context_hint.rb +7 -2
  94. data/lib/woods/mcp/bearer_auth.rb +1 -1
  95. data/lib/woods/mcp/bootstrapper.rb +33 -7
  96. data/lib/woods/mcp/config_resolver.rb +26 -7
  97. data/lib/woods/mcp/index_reader.rb +125 -24
  98. data/lib/woods/mcp/index_reader_pinning.rb +16 -0
  99. data/lib/woods/mcp/origin_guard.rb +24 -77
  100. data/lib/woods/mcp/origin_policy.rb +124 -0
  101. data/lib/woods/mcp/renderers/markdown_renderer.rb +7 -1
  102. data/lib/woods/mcp/renderers/plain_renderer.rb +3 -1
  103. data/lib/woods/mcp/search_results.rb +7 -1
  104. data/lib/woods/mcp/server.rb +24 -4
  105. data/lib/woods/module_reconciliation.rb +151 -0
  106. data/lib/woods/path_dispatcher.rb +7 -2
  107. data/lib/woods/railtie_support.rb +8 -0
  108. data/lib/woods/rake_helpers.rb +43 -11
  109. data/lib/woods/release.rb +1 -1
  110. data/lib/woods/resilience/index_validator.rb +8 -3
  111. data/lib/woods/resilience/retryable_provider.rb +18 -1
  112. data/lib/woods/resolved_config.rb +68 -8
  113. data/lib/woods/retrieval/context_assembler.rb +3 -3
  114. data/lib/woods/retrieval/lexical_assembler.rb +3 -2
  115. data/lib/woods/retrieval/scope.rb +18 -2
  116. data/lib/woods/retrieval/source_evidence.rb +14 -2
  117. data/lib/woods/source_contributor_validation.rb +78 -0
  118. data/lib/woods/source_contributors.rb +116 -0
  119. data/lib/woods/source_inputs/handoff.rb +37 -0
  120. data/lib/woods/source_inputs/launcher.rb +53 -13
  121. data/lib/woods/source_inputs/manifest.rb +84 -3
  122. data/lib/woods/source_inputs/private_key.rb +44 -12
  123. data/lib/woods/source_inputs/scanner.rb +98 -27
  124. data/lib/woods/source_inputs/scopes.rb +1 -1
  125. data/lib/woods/source_inputs/session.rb +147 -15
  126. data/lib/woods/source_inputs/stable_reader.rb +127 -0
  127. data/lib/woods/source_inputs/status.rb +40 -8
  128. data/lib/woods/source_inputs/verifier.rb +28 -5
  129. data/lib/woods/source_path_encoding.rb +33 -0
  130. data/lib/woods/source_references/cache.rb +284 -0
  131. data/lib/woods/source_references/collector.rb +120 -0
  132. data/lib/woods/source_references/extraction.rb +185 -0
  133. data/lib/woods/source_references/inputs.rb +134 -0
  134. data/lib/woods/source_references/parser_adapter.rb +134 -0
  135. data/lib/woods/source_references/pass.rb +152 -0
  136. data/lib/woods/source_references/prism_adapter.rb +116 -0
  137. data/lib/woods/source_references/registry.rb +178 -0
  138. data/lib/woods/source_references/runtime_lookup.rb +127 -0
  139. data/lib/woods/source_references/value_class.rb +82 -0
  140. data/lib/woods/storage/metadata_store.rb +4 -1
  141. data/lib/woods/storage/qdrant.rb +2 -2
  142. data/lib/woods/unblocked/client.rb +12 -7
  143. data/lib/woods/unblocked/document_builder.rb +4 -1
  144. data/lib/woods/unblocked/exporter.rb +127 -37
  145. data/lib/woods/unblocked/sync_manifest.rb +137 -21
  146. data/lib/woods/unblocked/uri_migration.rb +105 -0
  147. data/lib/woods/util/host_guard.rb +3 -2
  148. data/lib/woods/version.rb +1 -1
  149. data/lib/woods/watch/catch_up.rb +138 -0
  150. data/lib/woods/watch/claim_lease.rb +150 -0
  151. data/lib/woods/watch/cli.rb +26 -2
  152. data/lib/woods/watch/daemon.rb +80 -59
  153. data/lib/woods/watch/installation/options.rb +1 -1
  154. data/lib/woods/watch/installation/receipt.rb +6 -1
  155. data/lib/woods/watch/managed_child.rb +1 -1
  156. data/lib/woods/watch/supervisor.rb +1 -1
  157. data/lib/woods/watch/tree_scan.rb +14 -2
  158. data/plugin/.claude-plugin/plugin.json +1 -1
  159. data/plugin/hooks/adapters/normalize.rb +3 -2
  160. data/plugin/hooks/woods-input-rules.sh +4 -0
  161. data/plugin/hooks/woods-refresh.sh +15 -7
  162. data/plugin/hooks/woods-session-start.sh +60 -3
  163. data/plugin/skills/woods-diagnose/SKILL.md +336 -2
  164. data/plugin/skills/woods-investigate/SKILL.md +11 -0
  165. data/plugin/skills/woods-mcp-config/SKILL.md +86 -0
  166. data/plugin/skills/woods-setup/SKILL.md +56 -1
  167. metadata +34 -5
@@ -10,6 +10,7 @@ require_relative 'eval_guard'
10
10
  require_relative 'input_contract'
11
11
  require_relative 'model_validator'
12
12
  require_relative 'safe_context'
13
+ require_relative 'redactor'
13
14
  require_relative 'scope_predicate_parser'
14
15
  require_relative 'sql_validator'
15
16
  require_relative 'sql_noise_stripper'
@@ -92,7 +93,7 @@ module Woods
92
93
 
93
94
  normalize_params!(tool, params)
94
95
  start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
95
- result = @safe_context.execute { with_mysql_quote_modes { dispatch(tool, params) } }
96
+ result = @safe_context.execute { with_mysql_quote_modes { dispatch_with_key_redaction(tool, params) } }
96
97
  elapsed = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round(1)
97
98
 
98
99
  { 'ok' => true, 'result' => result, 'timing_ms' => elapsed }
@@ -112,6 +113,58 @@ module Woods
112
113
 
113
114
  private
114
115
 
116
+ def dispatch_with_key_redaction(tool, params)
117
+ output = dispatch(tool, params)
118
+ return output if @safe_context.redacted_key_values.empty?
119
+
120
+ Redactor.apply(output, typed_redaction_context(tool, params))
121
+ end
122
+
123
+ def typed_redaction_context(tool, params)
124
+ types = Hash.new { |hash, key| hash[key] = [] }
125
+ typed_redaction_models(tool, params).each do |name, model|
126
+ @safe_context.redacted_key_values.each do |pattern|
127
+ key = pattern['key_column']
128
+ types[key] << model.type_for_attribute(key) if @model_validator.columns_for(name).include?(key)
129
+ end
130
+ end
131
+ @safe_context.with_key_value_types(types, raw: %w[sql query].include?(tool))
132
+ end
133
+
134
+ def typed_redaction_models(tool, params)
135
+ tables = if tool == 'sql'
136
+ SqlTableScanner.identifiers_in(params['sql'], dialect: sql_dialect, mysql_modes: mysql_quote_modes)
137
+ else
138
+ selected_source_tables(params)
139
+ end
140
+ @model_validator.model_names.filter_map do |name|
141
+ model = resolve_model(name)
142
+ next unless model.respond_to?(:type_for_attribute)
143
+ next unless typed_model_source?(model, name, params, tables)
144
+
145
+ [name, model]
146
+ rescue NameError
147
+ # Injectable registries can describe models without loading Rails.
148
+ next
149
+ end
150
+ end
151
+
152
+ def typed_model_source?(model, name, params, tables)
153
+ return true if name == params['model']
154
+ return false unless model.respond_to?(:table_name)
155
+
156
+ # Match the scanner's unquoted final segment, as the identifier gates
157
+ # do. Keep all matching models across schemas: every possible type
158
+ # contributes to masking rather than selecting one ambiguous source.
159
+ tables.any? { |table| table.split('.').last.casecmp?(model.table_name.split('.').last) }
160
+ end
161
+
162
+ def selected_source_tables(params)
163
+ Array(params['select'] || params['columns']).filter_map do |column|
164
+ column.split('.')[0...-1].join('.') if column.include?('.')
165
+ end
166
+ end
167
+
115
168
  def sanitize_execution_error(error)
116
169
  klass = error.class.name
117
170
  # Well-known AR wrappers that contain the adapter error as their cause —
@@ -530,7 +583,7 @@ module Woods
530
583
  raise ValidationError, 'columns must contain at least one item' if columns && columns.empty?
531
584
 
532
585
  @model_validator.validate_columns!(params['model'], columns) if columns
533
- refuse_orphan_eav_value_selection!(Array(columns)) if columns
586
+ refuse_orphan_eav_value_selection!(Array(columns), params['model']) if columns
534
587
  model = resolve_model(params['model'])
535
588
  limit = params.fetch('limit', 100)
536
589
  scope = apply_scope(model, params['scope'], model_name: params['model'])
@@ -567,9 +620,9 @@ module Woods
567
620
 
568
621
  def handle_association_count(params)
569
622
  model = resolve_model(params['model'])
570
- association_name = params['association']
623
+ association_name = params['association'].to_sym
571
624
  requested_scope = params['scope']
572
- reflection = model.reflect_on_association(association_name.to_sym)
625
+ reflection = model.reflect_on_association(association_name)
573
626
 
574
627
  raise ValidationError, "Unknown association '#{association_name}' on #{params['model']}" unless reflection
575
628
 
@@ -584,13 +637,38 @@ module Woods
584
637
  # association's own model before any database I/O runs (not just
585
638
  # before the association is read): `model.find` below is itself a
586
639
  # query, and a request with a bad scope should never reach it.
587
- validate_scope_columns!(requested_scope, reflection.klass.name) if requested_scope
640
+ validate_association_scope!(requested_scope, reflection)
588
641
 
589
642
  record = checked_relation(model).find(params['id'])
590
- scope = record.public_send(association_name)
643
+ association = record.association(association_name)
644
+ return { 'count' => 0 } if empty_polymorphic_association?(association, reflection)
645
+
646
+ scope = association.scope
647
+ return { 'count' => 0 } unless scope
648
+
591
649
  scope = apply_scope(scope, requested_scope, model_name: reflection.klass.name) if requested_scope
592
650
  gate_association_sql!(scope)
593
- { 'count' => scope.count }
651
+ { 'count' => association_count(scope, reflection) }
652
+ end
653
+
654
+ def validate_association_scope!(scope, reflection)
655
+ return unless scope
656
+
657
+ if reflection.respond_to?(:polymorphic?) && reflection.polymorphic?
658
+ raise ValidationError, 'Rejected: scoped polymorphic association counts require a concrete target model.'
659
+ end
660
+
661
+ validate_scope_columns!(scope, reflection.klass.name)
662
+ end
663
+
664
+ def empty_polymorphic_association?(association, reflection)
665
+ reflection.respond_to?(:polymorphic?) && reflection.polymorphic? && association.klass.nil?
666
+ end
667
+
668
+ def association_count(scope, reflection)
669
+ return scope.count if reflection.collection?
670
+
671
+ scope.exists? ? 1 : 0
594
672
  end
595
673
 
596
674
  # Materialize a model's default scope once, inspect the resolved SQL,
@@ -752,7 +830,7 @@ module Woods
752
830
  limit = params.fetch('limit', 10)
753
831
 
754
832
  @model_validator.validate_column!(params['model'], order_by)
755
- refuse_redacted_column!(order_by)
833
+ refuse_protected_predicate_column!(order_by)
756
834
  unless %w[asc desc].include?(direction)
757
835
  raise ValidationError, "direction must be asc or desc (got #{direction.inspect})"
758
836
  end
@@ -782,12 +860,8 @@ module Woods
782
860
  sql = params['sql']
783
861
  raise ValidationError, 'Missing required parameter: sql' unless sql
784
862
 
785
- require_relative 'sql_validator'
786
- SqlValidator.new(dialect: sql_dialect, mysql_modes: mysql_quote_modes).validate!(sql)
787
- validate_protected_sql_usage!(sql)
788
- # Post-validation, pre-execution TableGate — blocks every configured
789
- # table even if the sql is otherwise well-formed.
790
- gate_sql!(sql)
863
+ require_sql_dialect!
864
+ validate_sql_policy!(sql)
791
865
 
792
866
  limit = params['limit']
793
867
  # EXPLAIN's output is plan rows, not the query's own row set: wrapping
@@ -800,14 +874,61 @@ module Woods
800
874
  'Resubmit without limit.'
801
875
  end
802
876
 
803
- query_sql = limit ? "SELECT * FROM (#{sql}) AS _limited LIMIT #{limit}" : sql
877
+ query_sql = limit ? "SELECT * FROM (\n#{sql}\n) AS _limited LIMIT #{limit}" : sql
878
+ validate_sql_policy!(query_sql) if limit
804
879
  result = active_connection.select_all(query_sql)
880
+ validate_sql_result_types!(result)
805
881
 
806
882
  { 'columns' => result.columns, 'rows' => result.rows, 'count' => result.rows.size }
807
883
  rescue SqlValidationError => e
808
884
  raise ValidationError, e.message
809
885
  end
810
886
 
887
+ def require_sql_dialect!
888
+ return if sql_dialect
889
+
890
+ raise ValidationError, 'Rejected: console_sql requires a recognized database adapter family; ' \
891
+ 'use a structured Console tool.'
892
+ end
893
+
894
+ # A row cap changes the statement. Check both the caller's complete
895
+ # statement and the exact SQL sent to the adapter, including the cap.
896
+ def validate_sql_policy!(sql)
897
+ # Alias-list identity is its own policy, even when an alias also
898
+ # looks like a function name to the SQL validator.
899
+ sql_security_views(sql).each { |stripped| refuse_sql_column_alias_lists!(stripped) }
900
+ SqlValidator.new(dialect: sql_dialect, mysql_modes: mysql_quote_modes).validate!(sql)
901
+ validate_protected_sql_usage!(sql)
902
+ gate_sql!(sql)
903
+ end
904
+
905
+ # PostgreSQL represents unregistered composite/opaque types (including
906
+ # arrays of them) with a nil type identity. Their output headers cannot
907
+ # tell the redactor which nested fields are protected. This final fence
908
+ # complements pre-execution checks without relying on alias spelling.
909
+ # Legitimate custom types can also refuse while redaction is active.
910
+ def validate_sql_result_types!(result)
911
+ return unless sql_dialect == :postgres
912
+ return if @safe_context.redacted_columns.empty? && redacted_kv_columns.empty?
913
+
914
+ return if recognized_sql_result_types?(result)
915
+
916
+ raise ValidationError,
917
+ 'Rejected: PostgreSQL returned an unrecognized result type that cannot preserve ' \
918
+ 'protected field identity. ' \
919
+ 'Select ordinary scalar columns or use a structured Console tool.'
920
+ end
921
+
922
+ def recognized_sql_result_types?(result)
923
+ types = result.column_types if result.respond_to?(:column_types)
924
+ return false unless types
925
+
926
+ result.columns.each_with_index.all? do |column, index|
927
+ type = types[index] || types[column]
928
+ type&.type
929
+ end
930
+ end
931
+
811
932
  # @param sql [String] Validated SQL (already passed SqlValidator)
812
933
  # @return [Boolean] true when the statement starts with EXPLAIN
813
934
  def explain_statement?(sql)
@@ -819,16 +940,11 @@ module Woods
819
940
  # and PostgreSQL quote/comment grammars differ (`\'` escapes, `#`
820
941
  # comments); validating with the matching dialect accepts dialect-valid
821
942
  # literals and still rejects every known bypass form. Unknown adapters
822
- # return nil and get the conservative union of supported dialects.
943
+ # remain available to structured tools but refuse raw console_sql requests.
823
944
  #
824
945
  # @return [Symbol, nil]
825
946
  def sql_dialect
826
- adapter = active_connection.adapter_name.to_s.downcase
827
- return :mysql if adapter.include?('mysql')
828
- return :postgres if adapter.include?('postgre')
829
- return :sqlite if adapter.include?('sqlite')
830
-
831
- nil
947
+ AdapterFamily.for(active_connection)
832
948
  end
833
949
 
834
950
  # Keep session reads local to this request and execution context. Restore
@@ -899,8 +1015,10 @@ module Woods
899
1015
  # @return [Hash] normalized, validated query clauses
900
1016
  def validated_query_clauses(model, params)
901
1017
  model_name = params['model']
1018
+ # Keep execution and typed source inference on the same validated projection.
1019
+ params['select'] = validated_select(params['select'], model_name) if params['select']
902
1020
  {
903
- select: params['select'] ? validated_select(params['select'], model_name) : nil,
1021
+ select: params['select'],
904
1022
  joins: validated_query_joins(model, params['joins']),
905
1023
  scope: params.key?('scope') ? validated_query_scope(params['scope'], model_name, model) : nil,
906
1024
  group_by: params['group_by']&.any? ? validated_columns(params['group_by'], model_name) : nil,
@@ -961,7 +1079,7 @@ module Woods
961
1079
  expressions = Array(select).map do |expr|
962
1080
  validate_select_expression!(expr.strip, model_name)
963
1081
  end
964
- refuse_orphan_eav_value_selection!(expressions)
1082
+ refuse_orphan_eav_value_selection!(expressions, model_name)
965
1083
  expressions
966
1084
  end
967
1085
 
@@ -976,21 +1094,41 @@ module Woods
976
1094
  #
977
1095
  # @param expressions [Array<String>] validated select expressions
978
1096
  # @raise [ValidationError] when a value column is selected without its key
979
- def refuse_orphan_eav_value_selection!(expressions)
1097
+ def refuse_orphan_eav_value_selection!(expressions, model_name = nil)
980
1098
  selected = directly_selected_columns(expressions)
981
1099
 
982
1100
  @safe_context.redacted_key_values.each do |pattern|
983
- next unless selected.include?(pattern['value_column'])
984
- next if selected.include?(pattern['key_column'])
1101
+ values = selected.select { |column| base_column_name(column).casecmp?(pattern['value_column']) }
1102
+ values.select! { |column| eav_source_has_key?(column, pattern, model_name) }
1103
+ next if values.empty?
1104
+
1105
+ keys = selected.select { |column| base_column_name(column).casecmp?(pattern['key_column']) }
1106
+ next if paired_eav_sources?(keys, values)
985
1107
 
986
1108
  raise ValidationError,
987
1109
  "Rejected: selecting EAV value column '#{pattern['value_column']}' without its paired " \
988
1110
  "key column '#{pattern['key_column']}' bypasses redaction; select both columns so the " \
989
- 'value can be masked.'
1111
+ 'value can be masked. Both columns must have the same unambiguous source.'
990
1112
  end
991
1113
  end
992
1114
 
993
- # The bare, unaliased columns referenced by a validated select list.
1115
+ def eav_source_has_key?(column, pattern, model_name)
1116
+ columns = if column.include?('.')
1117
+ @model_validator.columns_for_table(column.split('.')[0...-1].join('.'))
1118
+ elsif model_name
1119
+ @model_validator.columns_for(model_name)
1120
+ end
1121
+ columns.nil? || columns.include?(pattern['key_column'])
1122
+ end
1123
+
1124
+ def paired_eav_sources?(keys, values)
1125
+ return false unless keys.one? && values.one?
1126
+
1127
+ keys.first.split('.')[0...-1] == values.first.split('.')[0...-1]
1128
+ end
1129
+
1130
+ # Unaliased column references, retaining source qualifiers so a joined
1131
+ # key cannot stand in for the key paired with another source's value.
994
1132
  # Aggregates and aliases cannot appear here — the per-expression
995
1133
  # validation refuses them over protected columns before this runs.
996
1134
  #
@@ -1004,7 +1142,7 @@ module Woods
1004
1142
  fn_arg, bare_col, alias_name = match.captures[1..]
1005
1143
  next if fn_arg || alias_name
1006
1144
 
1007
- base_column_name(bare_col)
1145
+ bare_col
1008
1146
  end
1009
1147
  end
1010
1148
 
@@ -1513,8 +1651,7 @@ module Woods
1513
1651
  # `SqlNoiseStripper` is the same module SqlValidator uses. The
1514
1652
  # combined single-pass strip_noise resolves comments and literals
1515
1653
  # together so a comment marker inside a literal can't hide a keyword.
1516
- stripped = SqlNoiseStripper.strip_noise(template, dialect: sql_dialect || :postgres, **mysql_quote_modes)
1517
- if SCOPE_TEMPLATE_FORBIDDEN.match?(stripped)
1654
+ if sql_security_views(template).any? { |stripped| SCOPE_TEMPLATE_FORBIDDEN.match?(stripped) }
1518
1655
  raise ValidationError,
1519
1656
  'scope template contains forbidden SQL keywords ' \
1520
1657
  '(subqueries, UNION, time-based functions, DML/DDL are not allowed). ' \
@@ -1543,7 +1680,7 @@ module Woods
1543
1680
  return unless params['columns']
1544
1681
 
1545
1682
  @model_validator.validate_columns!(params['model'], params['columns'])
1546
- refuse_orphan_eav_value_selection!(params['columns'])
1683
+ refuse_orphan_eav_value_selection!(params['columns'], params['model'])
1547
1684
  end
1548
1685
 
1549
1686
  # Raw SQL preserves redaction identity only for direct, unaliased
@@ -1551,14 +1688,29 @@ module Woods
1551
1688
  # predicates, CTEs, and other result shapes can rename a protected
1552
1689
  # value or turn it into an oracle, so they fail closed before execution.
1553
1690
  def validate_protected_sql_usage!(sql)
1691
+ # This exact wrapper only preserves the inner statement's output
1692
+ # headers. SqlValidator has already checked balanced delimiters.
1693
+ limited = /\ASELECT \* FROM \(\n(.*)\n\) AS _limited LIMIT \d+\z/m.match(sql)
1694
+ if limited
1695
+ SqlValidator.new(dialect: sql_dialect, mysql_modes: mysql_quote_modes).validate!(limited[1])
1696
+ return validate_protected_sql_usage!(limited[1])
1697
+ end
1698
+
1699
+ sql_security_views(sql).each { |stripped| validate_protected_sql_view!(stripped) }
1700
+ end
1701
+
1702
+ def validate_protected_sql_view!(stripped)
1703
+ refuse_sql_column_alias_lists!(stripped)
1704
+ refuse_composite_sql_projection!(stripped)
1705
+ refuse_ambiguous_eav_sql!(stripped)
1706
+ refuse_protected_sql_positions!(stripped)
1554
1707
  protected = (@safe_context.redacted_columns + redacted_kv_columns).uniq
1555
- referenced = protected.select { |column| sql_identifier_referenced?(sql, column) }
1708
+ referenced = protected.select { |column| sql_identifier_referenced?(stripped, column) }
1556
1709
  return if referenced.empty?
1557
1710
 
1558
- stripped = SqlNoiseStripper.strip_noise(sql, dialect: sql_dialect || :postgres, **mysql_quote_modes)
1559
1711
  expressions, tail = protected_sql_projection(stripped)
1560
1712
  selected = expressions.filter_map { |expression| direct_sql_column_name(expression) }
1561
- unsafe = unsafe_protected_sql_column(referenced, expressions, selected, tail)
1713
+ unsafe = unsafe_protected_sql_column(referenced, expressions, selected, tail, stripped)
1562
1714
  return unless unsafe
1563
1715
 
1564
1716
  raise ValidationError,
@@ -1567,17 +1719,225 @@ module Woods
1567
1719
  'and unaliased, or use a structured Console tool.'
1568
1720
  end
1569
1721
 
1570
- def unsafe_protected_sql_column(referenced, expressions, selected, tail)
1722
+ # Relation/CTE column lists rename fields by position, before output
1723
+ # headers are available. Refuse them whenever either policy is active,
1724
+ # independently of protected-name references or function validation.
1725
+ def refuse_sql_column_alias_lists!(stripped)
1726
+ return if @safe_context.redacted_columns.empty? && redacted_kv_columns.empty?
1727
+
1728
+ identifier = /(?:[A-Za-z_\u0080-\u{10ffff}][A-Za-z0-9_$\u0080-\u{10ffff}]*|"(?:""|[^"])+"|`(?:``|[^`])+`)/u
1729
+ list = /#{identifier}\s*\(\s*#{identifier}(?:\s*,\s*#{identifier})*\s*\)/
1730
+ cte = /#{list}\s*AS\s*(?:(?:NOT\s+)?MATERIALIZED\s*)?\(/i
1731
+ relation = /\A\s*(?:AS\s+)?#{list}/i
1732
+ renamed = stripped.match?(cte) || SqlTableScanner.relation_factors(stripped).any? do |factor|
1733
+ sql_relation_alias_tail(factor).match?(relation)
1734
+ end
1735
+ return unless renamed
1736
+
1737
+ raise ValidationError,
1738
+ 'Rejected: SQL relation or CTE column alias lists cannot preserve redaction identity. ' \
1739
+ 'Select protected columns directly and unaliased, or use a structured Console tool.'
1740
+ end
1741
+
1742
+ # Consume only the leading source, leaving its optional alias. Balanced
1743
+ # sources include subqueries, ONLY(table), and table-function arguments;
1744
+ # scalar functions elsewhere in the clause are not alias declarations.
1745
+ def sql_relation_alias_tail(factor)
1746
+ tokens = sql_policy_tokens(factor)
1747
+ index = %w[ONLY LATERAL].include?(sql_token_text(tokens, 0)) ? 1 : 0
1748
+ unless sql_token_text(tokens, index) == '('
1749
+ index += 1
1750
+ index += 2 while sql_token_text(tokens, index) == '.'
1751
+ end
1752
+ index = sql_after_parentheses(tokens, index) if sql_token_text(tokens, index) == '('
1753
+ index += 1 if sql_token_text(tokens, index) == '*'
1754
+ tokens[index] ? factor[tokens[index][:start]..] : ''
1755
+ end
1756
+
1757
+ def sql_after_parentheses(tokens, index)
1758
+ depth = tokens[index][:depth]
1759
+ closing = ((index + 1)...tokens.length).find do |offset|
1760
+ tokens[offset][:text] == ')' && tokens[offset][:depth] == depth
1761
+ end
1762
+ closing ? closing + 1 : tokens.length
1763
+ end
1764
+
1765
+ # Result metadata provides column names, not their table provenance.
1766
+ # A joined/derived EAV row can supply a key from a different source.
1767
+ # Keep direct single-source SQL reads and qualified structured joins;
1768
+ # refuse ambiguous SQL shapes before fetching their values.
1769
+ def refuse_ambiguous_eav_sql!(stripped)
1770
+ return if sql_eav_patterns(stripped).empty?
1771
+ return if SqlTableScanner.relation_factors(stripped).size <= 1
1772
+ return unless stripped.include?('*') || redacted_eav_value_columns.any? do |column|
1773
+ sql_identifier_referenced?(stripped, column)
1774
+ end
1775
+
1776
+ raise ValidationError,
1777
+ 'Rejected: SQL EAV values require one unambiguous source; use a structured Console query.'
1778
+ end
1779
+
1780
+ # PostgreSQL can return a table alias as one composite-valued column.
1781
+ # Its header carries no identities for the protected fields inside it.
1782
+ # Reject source names used as projection values, including nested forms.
1783
+ def refuse_composite_sql_projection!(stripped)
1784
+ return if @safe_context.redacted_columns.empty? && redacted_kv_columns.empty?
1785
+
1786
+ sources = SqlTableScanner.relation_factors(stripped).flat_map { |factor| sql_relation_names(factor) }
1787
+ tokens = sql_policy_tokens(stripped)
1788
+ return unless whole_row_value?(tokens, sources)
1789
+
1790
+ raise ValidationError,
1791
+ 'Rejected: whole-row SQL values cannot preserve protected field identity; select columns.'
1792
+ end
1793
+
1794
+ # Track each query's clause separately, including correlated expressions
1795
+ # without a FROM clause. Relation declarations and qualified columns are
1796
+ # not values; a source token anywhere else may be a composite row.
1797
+ def whole_row_value?(tokens, sources)
1798
+ clauses = {}
1799
+ query_levels = {}
1800
+ tokens.each_with_index.any? do |token, index|
1801
+ clause = sql_clause_at_token(token, clauses, query_levels)
1802
+ next false unless clause && !%w[FROM JOIN STRAIGHT_JOIN].include?(clause)
1803
+ next false unless sources.any? { |source| sql_unquote(token[:text]).casecmp?(source) }
1804
+
1805
+ !sql_nonvalue_reference?(tokens, index)
1806
+ end
1807
+ end
1808
+
1809
+ def sql_clause_at_token(token, clauses, query_levels)
1810
+ word, depth = token.values_at(:text, :depth)
1811
+ keyword = word.upcase
1812
+ if word == '('
1813
+ clauses[depth + 1] = 'EXPRESSION'
1814
+ query_levels.delete(depth + 1)
1815
+ end
1816
+ if word == ')'
1817
+ clauses.delete(depth + 1)
1818
+ query_levels.delete(depth + 1)
1819
+ end
1820
+ query_levels[depth] = true if keyword == 'SELECT'
1821
+ boundaries = %w[SELECT FROM JOIN STRAIGHT_JOIN ON WHERE HAVING ORDER GROUP LIMIT OFFSET]
1822
+ clauses[depth] = keyword if query_levels[depth] && boundaries.include?(keyword)
1823
+ clauses[depth]
1824
+ end
1825
+
1826
+ def sql_nonvalue_reference?(tokens, index)
1827
+ previous = index.positive? ? tokens[index - 1][:text].upcase : nil
1828
+ return true if ['.', 'AS'].include?(previous)
1829
+ return true if sql_token_text(tokens, index + 1) == '.' && sql_token_text(tokens, index + 2) != '*'
1830
+
1831
+ direct_sql_wildcard?(tokens, index)
1832
+ end
1833
+
1834
+ def sql_token_text(tokens, index)
1835
+ tokens[index]&.fetch(:text)&.upcase
1836
+ end
1837
+
1838
+ def direct_sql_wildcard?(tokens, index)
1839
+ return false unless sql_token_text(tokens, index + 1) == '.' && sql_token_text(tokens, index + 2) == '*'
1840
+
1841
+ previous = index.positive? ? sql_token_text(tokens, index - 1) : nil
1842
+ following = sql_token_text(tokens, index + 3)
1843
+ %w[SELECT ,].include?(previous) && [nil, ',', 'FROM'].include?(following)
1844
+ end
1845
+
1846
+ # Comments and literal bodies have already been removed. Token offsets
1847
+ # let projection and ordinal checks retain complete nested expressions.
1848
+ def sql_policy_tokens(stripped)
1849
+ depth = 0
1850
+ stripped.to_enum(:scan, /"(?:[^"]|"")*"|`(?:[^`]|``)*`|''|
1851
+ [A-Za-z_\u0080-\u{10ffff}][A-Za-z0-9_$\u0080-\u{10ffff}]*|[0-9]+|::|[^\s]/ux).map do
1852
+ match = Regexp.last_match
1853
+ word = match[0]
1854
+ depth -= 1 if word == ')'
1855
+ token = { text: word, depth: depth, start: match.begin(0), finish: match.end(0) }
1856
+ depth += 1 if word == '('
1857
+ token
1858
+ end
1859
+ end
1860
+
1861
+ def sql_unquote(identifier)
1862
+ identifier.sub(/\A["`]/, '').sub(/["`]\z/, '').gsub('""', '"').gsub('``', '`')
1863
+ end
1864
+
1865
+ def sql_select_lists(stripped)
1866
+ tokens = sql_policy_tokens(stripped)
1867
+ tokens.each_with_index.filter_map do |token, index|
1868
+ next unless token[:text].casecmp?('SELECT')
1869
+
1870
+ finish = tokens[(index + 1)..].find { |candidate| sql_projection_end?(candidate, token) }
1871
+ finish_at = finish ? finish[:start] : stripped.length
1872
+ { expressions: sql_projection_expressions(stripped[token[:finish]...finish_at]),
1873
+ depth: token[:depth], start: token[:start], finish: finish_at }
1874
+ end
1875
+ end
1876
+
1877
+ def sql_projection_end?(candidate, token)
1878
+ candidate[:depth] < token[:depth] ||
1879
+ (candidate[:depth] == token[:depth] &&
1880
+ %w[FROM WHERE GROUP HAVING ORDER LIMIT UNION INTERSECT EXCEPT].include?(candidate[:text].upcase))
1881
+ end
1882
+
1883
+ def refuse_protected_sql_positions!(stripped)
1884
+ lists = sql_select_lists(stripped)
1885
+ sql_policy_tokens(stripped).each_cons(3) do |clause, by, position|
1886
+ next unless %w[ORDER GROUP].include?(clause[:text].upcase) && by[:text].casecmp?('BY')
1887
+
1888
+ list = sql_projection_before(lists, clause)
1889
+ next unless list
1890
+ next unless sql_projection_expressions(stripped[position[:start]..]).any? do |item|
1891
+ protected_sql_ordinal?(item, list[:expressions])
1892
+ end
1893
+
1894
+ raise ValidationError,
1895
+ 'Rejected: positional SQL ordering or grouping cannot reference protected output columns.'
1896
+ end
1897
+ end
1898
+
1899
+ def sql_projection_before(lists, clause)
1900
+ lists.reverse.find do |candidate|
1901
+ candidate[:depth] == clause[:depth] && candidate[:start] < clause[:start]
1902
+ end
1903
+ end
1904
+
1905
+ def protected_sql_ordinal?(item, expressions)
1906
+ ordinal = /\A\s*\(*\s*([0-9]+)\s*\)*(?:\s|\z)/.match(item)
1907
+ return false unless ordinal && ordinal[1].to_i.positive?
1908
+
1909
+ expression = expressions[ordinal[1].to_i - 1]
1910
+ return false unless expression
1911
+ return true if expression.include?('*')
1912
+
1913
+ (@safe_context.redacted_columns + redacted_eav_value_columns).any? do |column|
1914
+ sql_identifier_referenced?(expression, column)
1915
+ end
1916
+ end
1917
+
1918
+ def sql_relation_names(factor)
1919
+ identifier = /(?:[A-Za-z_\u0080-\u{10ffff}][A-Za-z0-9_$\u0080-\u{10ffff}]*|"(?:""|[^"])+"|`(?:``|[^`])+`)/u
1920
+ source = /\A\s*(?:ONLY\b\s*\(?\s*)?(#{identifier})(?:\s*\.\s*(#{identifier}))?/i
1921
+ match = source.match(factor)
1922
+ names = match ? [match[2] || match[1]] : []
1923
+ rest = match ? factor[match.end(0)..].sub(/\A\s*\*/, '') : factor
1924
+ aliases = /(?:\A|\))\s*(?:AS\s+)?(#{identifier})/i
1925
+ names.concat(rest.scan(aliases).flatten)
1926
+
1927
+ names.map { |name| sql_unquote(name) }
1928
+ end
1929
+
1930
+ def unsafe_protected_sql_column(referenced, expressions, selected, tail, stripped)
1571
1931
  referenced.find do |column|
1572
1932
  unsafe_protected_sql_reference?(column, expressions, selected, tail)
1573
- end || orphan_eav_sql_value(selected)
1933
+ end || orphan_eav_sql_value(selected, stripped)
1574
1934
  end
1575
1935
 
1576
1936
  def protected_sql_projection(stripped)
1577
- match = /\ASELECT\s+(.*?)\s+FROM\b/im.match(stripped)
1578
- return [[], stripped] unless match
1937
+ list = sql_select_lists(stripped).first
1938
+ return [[], stripped] unless list && list[:start].zero?
1579
1939
 
1580
- [sql_projection_expressions(match[1]), stripped[match.end(1)..]]
1940
+ [list[:expressions], stripped[list[:finish]..]]
1581
1941
  end
1582
1942
 
1583
1943
  def unsafe_protected_sql_reference?(column, expressions, selected, tail)
@@ -1592,15 +1952,31 @@ module Woods
1592
1952
  @safe_context.redacted_columns.include?(column) || redacted_eav_value_columns.include?(column)
1593
1953
  end
1594
1954
 
1595
- def orphan_eav_sql_value(selected)
1596
- pattern = @safe_context.redacted_key_values.find do |candidate|
1955
+ def orphan_eav_sql_value(selected, stripped)
1956
+ pattern = sql_eav_patterns(stripped).find do |candidate|
1597
1957
  selected.include?(candidate['value_column']) && !selected.include?(candidate['key_column'])
1598
1958
  end
1599
1959
  pattern&.fetch('value_column')
1600
1960
  end
1601
1961
 
1962
+ def sql_eav_patterns(stripped)
1963
+ tables = SqlTableScanner.identifiers_in(stripped, dialect: sql_dialect, mysql_modes: mysql_quote_modes)
1964
+ @safe_context.redacted_key_values.select do |pattern|
1965
+ tables.any? do |table|
1966
+ columns = @model_validator.columns_for_table(table)
1967
+ columns.nil? || [pattern['key_column'], pattern['value_column']].all? { |column| columns.include?(column) }
1968
+ end
1969
+ end
1970
+ end
1971
+
1602
1972
  def sql_projection_expressions(projection)
1603
- projection.split(',').map(&:strip)
1973
+ commas = sql_policy_tokens(projection).select { |token| token[:text] == ',' && token[:depth].zero? }
1974
+ start = 0
1975
+ commas.map do |token|
1976
+ expression = projection[start...token[:start]].strip
1977
+ start = token[:finish]
1978
+ expression
1979
+ end + [projection[start..].strip]
1604
1980
  end
1605
1981
 
1606
1982
  def direct_sql_column_name(expression)
@@ -1621,8 +1997,16 @@ module Woods
1621
1997
  end
1622
1998
 
1623
1999
  def sql_identifier_referenced?(sql, column)
1624
- stripped = SqlNoiseStripper.strip_noise(sql, dialect: sql_dialect || :postgres, **mysql_quote_modes)
1625
- stripped.match?(/(?<![A-Za-z0-9_$])#{Regexp.escape(column)}(?![A-Za-z0-9_$])/i)
2000
+ sql_security_views(sql).any? do |stripped|
2001
+ stripped.match?(/(?<![A-Za-z0-9_$])#{Regexp.escape(column)}(?![A-Za-z0-9_$])/i)
2002
+ end
2003
+ end
2004
+
2005
+ def sql_security_views(sql)
2006
+ dialects = sql_dialect ? [sql_dialect] : SqlValidator::KNOWN_DIALECTS
2007
+ dialects.flat_map do |dialect|
2008
+ SqlNoiseStripper.security_views(sql, dialect: dialect, mysql_modes: sql_dialect ? mysql_quote_modes : nil)
2009
+ end.uniq
1626
2010
  end
1627
2011
 
1628
2012
  # Apply column selection to a relation.
@@ -1662,8 +2046,7 @@ module Woods
1662
2046
  #
1663
2047
  # @return [Arel::Nodes::SqlLiteral]
1664
2048
  def random_function
1665
- adapter = active_connection.adapter_name.downcase
1666
- func = adapter.include?('mysql') ? 'RAND' : 'RANDOM'
2049
+ func = sql_dialect == :mysql ? 'RAND' : 'RANDOM'
1667
2050
  Arel.sql("#{func}()")
1668
2051
  end
1669
2052
 
@@ -87,6 +87,14 @@ module Woods
87
87
  validate_column!(model_name, column_name)
88
88
  end
89
89
 
90
+ # Return known table columns without guessing an unknown table's shape.
91
+ # @param table_name [String]
92
+ # @return [Array<String>, nil] nil when this table is not registered
93
+ def columns_for_table(table_name)
94
+ model = @model_by_table[table_name.to_s]
95
+ @registry[model] if model
96
+ end
97
+
90
98
  # List all known model names.
91
99
  #
92
100
  # @return [Array<String>]