aws_advanced_ruby_driver_wrapper 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +48 -2
  3. data/README.md +115 -6
  4. data/THIRD-PARTY-LICENSES +70 -11
  5. data/aws_advanced_ruby_driver_wrapper.gemspec +9 -0
  6. data/lib/aws_advanced_ruby_driver_wrapper/active_record/aws_action_cable_postgresql_support.rb +40 -0
  7. data/lib/aws_advanced_ruby_driver_wrapper/active_record/aws_connection_handler.rb +47 -0
  8. data/lib/aws_advanced_ruby_driver_wrapper/active_record/aws_mysql2_adapter.rb +23 -9
  9. data/lib/aws_advanced_ruby_driver_wrapper/active_record/aws_postgresql_adapter.rb +65 -10
  10. data/lib/aws_advanced_ruby_driver_wrapper/active_record/aws_trilogy_adapter.rb +80 -0
  11. data/lib/aws_advanced_ruby_driver_wrapper/active_record/type_adapter_alias.rb +61 -0
  12. data/lib/aws_advanced_ruby_driver_wrapper/driver_dialects/driver_dialect.rb +48 -0
  13. data/lib/aws_advanced_ruby_driver_wrapper/driver_dialects/driver_dialect_manager.rb +11 -4
  14. data/lib/aws_advanced_ruby_driver_wrapper/driver_dialects/mysql2_driver_dialect.rb +129 -0
  15. data/lib/aws_advanced_ruby_driver_wrapper/driver_dialects/mysql_driver_dialect.rb +9 -81
  16. data/lib/aws_advanced_ruby_driver_wrapper/driver_dialects/pg_driver_dialect.rb +28 -0
  17. data/lib/aws_advanced_ruby_driver_wrapper/driver_dialects/trilogy_driver_dialect.rb +118 -0
  18. data/lib/aws_advanced_ruby_driver_wrapper/errors/error_handler.rb +16 -0
  19. data/lib/aws_advanced_ruby_driver_wrapper/errors/mysql_error_handler.rb +10 -5
  20. data/lib/aws_advanced_ruby_driver_wrapper/errors/trilogy_error_handler.rb +127 -0
  21. data/lib/aws_advanced_ruby_driver_wrapper/errors.rb +1 -0
  22. data/lib/aws_advanced_ruby_driver_wrapper/monitoring/cluster_topology_monitor.rb +11 -0
  23. data/lib/aws_advanced_ruby_driver_wrapper/monitoring/monitor.rb +14 -0
  24. data/lib/aws_advanced_ruby_driver_wrapper/monitoring/monitor_connection.rb +6 -0
  25. data/lib/aws_advanced_ruby_driver_wrapper/mysql.rb +27 -9
  26. data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/blue_green_plugin.rb +125 -7
  27. data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/drop_blue_connections_marker.rb +84 -0
  28. data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/host_mapper.rb +3 -1
  29. data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/routing/base_routing.rb +7 -0
  30. data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/routing/substitute_connect_routing.rb +68 -41
  31. data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/routing/suspend_connect_routing.rb +4 -2
  32. data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/routing/suspend_execute_routing.rb +5 -3
  33. data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/routing/suspend_until_corresponding_host_found_connect_routing.rb +7 -1
  34. data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/status_builder.rb +1 -1
  35. data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/status_monitor.rb +10 -13
  36. data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/status_provider.rb +95 -3
  37. data/lib/aws_advanced_ruby_driver_wrapper/plugins/custom_endpoint/custom_endpoint_monitor.rb +11 -1
  38. data/lib/aws_advanced_ruby_driver_wrapper/plugins/custom_endpoint/custom_endpoint_plugin.rb +4 -1
  39. data/lib/aws_advanced_ruby_driver_wrapper/plugins/default_plugin.rb +14 -3
  40. data/lib/aws_advanced_ruby_driver_wrapper/plugins/failover_plugin.rb +68 -3
  41. data/lib/aws_advanced_ruby_driver_wrapper/plugins/federated_auth/base_saml_auth_plugin.rb +266 -0
  42. data/lib/aws_advanced_ruby_driver_wrapper/plugins/federated_auth/federated_auth_plugin.rb +134 -0
  43. data/lib/aws_advanced_ruby_driver_wrapper/plugins/federated_auth/okta_auth_plugin.rb +113 -0
  44. data/lib/aws_advanced_ruby_driver_wrapper/plugins/gdb/gdb_failover_plugin.rb +30 -9
  45. data/lib/aws_advanced_ruby_driver_wrapper/plugins/iam_auth_plugin.rb +16 -2
  46. data/lib/aws_advanced_ruby_driver_wrapper/plugins/initial_connection_strategy_plugin.rb +4 -1
  47. data/lib/aws_advanced_ruby_driver_wrapper/plugins/kms_encryption/key_management_utility.rb +6 -3
  48. data/lib/aws_advanced_ruby_driver_wrapper/plugins/kms_encryption/key_manager.rb +13 -4
  49. data/lib/aws_advanced_ruby_driver_wrapper/plugins/kms_encryption/kms_encryption_plugin.rb +29 -6
  50. data/lib/aws_advanced_ruby_driver_wrapper/plugins/kms_encryption/sql/trigger_mysql.sql +108 -0
  51. data/lib/aws_advanced_ruby_driver_wrapper/plugins/kms_encryption/sql/trigger_pg.sql +54 -0
  52. data/lib/aws_advanced_ruby_driver_wrapper/plugins/secrets_manager_plugin.rb +23 -1
  53. data/lib/aws_advanced_ruby_driver_wrapper/postgresql.rb +49 -23
  54. data/lib/aws_advanced_ruby_driver_wrapper/property_definition.rb +63 -1
  55. data/lib/aws_advanced_ruby_driver_wrapper/ruby_method.rb +3 -0
  56. data/lib/aws_advanced_ruby_driver_wrapper/services/dialect_service.rb +20 -7
  57. data/lib/aws_advanced_ruby_driver_wrapper/services/host_service.rb +34 -8
  58. data/lib/aws_advanced_ruby_driver_wrapper/services/monitor_service.rb +23 -4
  59. data/lib/aws_advanced_ruby_driver_wrapper/services/plugin_manager.rb +80 -29
  60. data/lib/aws_advanced_ruby_driver_wrapper/services/service_container.rb +10 -2
  61. data/lib/aws_advanced_ruby_driver_wrapper/services/service_utility.rb +4 -1
  62. data/lib/aws_advanced_ruby_driver_wrapper/trilogy.rb +167 -0
  63. data/lib/aws_advanced_ruby_driver_wrapper/utils/connection_config.rb +1 -1
  64. data/lib/aws_advanced_ruby_driver_wrapper/utils/connection_config_parser.rb +25 -15
  65. data/lib/aws_advanced_ruby_driver_wrapper/utils/events/batching_event_publisher.rb +18 -7
  66. data/lib/aws_advanced_ruby_driver_wrapper/utils/global_aurora_topology_utils.rb +2 -2
  67. data/lib/aws_advanced_ruby_driver_wrapper/utils/iam_auth_utils.rb +13 -4
  68. data/lib/aws_advanced_ruby_driver_wrapper/utils/parser/encryption_annotation_parser.rb +15 -31
  69. data/lib/aws_advanced_ruby_driver_wrapper/utils/parser/mysql_statement_analyzer.rb +55 -76
  70. data/lib/aws_advanced_ruby_driver_wrapper/utils/parser/mysql_text.rb +215 -0
  71. data/lib/aws_advanced_ruby_driver_wrapper/utils/parser/pg_statement_analyzer.rb +11 -20
  72. data/lib/aws_advanced_ruby_driver_wrapper/utils/parser/query_analysis.rb +2 -3
  73. data/lib/aws_advanced_ruby_driver_wrapper/utils/parser/sql_parser.rb +21 -14
  74. data/lib/aws_advanced_ruby_driver_wrapper/utils/retry_util.rb +2 -1
  75. data/lib/aws_advanced_ruby_driver_wrapper/utils/saml_utils.rb +133 -0
  76. data/lib/aws_advanced_ruby_driver_wrapper/utils/storage/storage_service.rb +29 -8
  77. data/lib/aws_advanced_ruby_driver_wrapper/utils/telemetry/null_telemetry.rb +89 -0
  78. data/lib/aws_advanced_ruby_driver_wrapper/utils/telemetry/open_telemetry_context.rb +124 -0
  79. data/lib/aws_advanced_ruby_driver_wrapper/utils/{ar_constants.rb → telemetry/open_telemetry_counter.rb} +19 -5
  80. data/lib/aws_advanced_ruby_driver_wrapper/utils/telemetry/open_telemetry_factory.rb +122 -0
  81. data/lib/aws_advanced_ruby_driver_wrapper/utils/telemetry/open_telemetry_gauge.rb +36 -0
  82. data/lib/aws_advanced_ruby_driver_wrapper/utils/telemetry/telemetry_const.rb +50 -0
  83. data/lib/aws_advanced_ruby_driver_wrapper/utils/telemetry/telemetry_factory.rb +143 -0
  84. data/lib/aws_advanced_ruby_driver_wrapper/utils/topology_utils.rb +9 -2
  85. data/lib/aws_advanced_ruby_driver_wrapper/version.rb +1 -1
  86. data/lib/aws_advanced_ruby_driver_wrapper.rb +47 -2
  87. metadata +57 -3
@@ -14,6 +14,7 @@
14
14
  # See the License for the specific language governing permissions and
15
15
  # limitations under the License.
16
16
 
17
+ require_relative 'mysql_text'
17
18
  require_relative 'query_analysis'
18
19
  require_relative 'query_type'
19
20
 
@@ -63,18 +64,17 @@ module AwsAdvancedRubyDriverWrapper
63
64
 
64
65
  INSERT_COLUMNS = /#{INSERT_START}#{IDENTIFIER_NC}\s*\(([^)]+)\)/i
65
66
  # The body of a SET clause, ended at the WHERE that follows it, at the statement terminator, or
66
- # at the end of the text. Matched against a copy with every quoted literal blanked (see
67
- # +mask_quoted_literals+), so a +WHERE+ or +;+ that is only part of a string value does not end
68
- # the clause early; the offsets it reports still index the original text.
67
+ # at the end of the text. Matched against a copy with every quoted literal and comment blanked
68
+ # (see +mask_non_code+), so a +WHERE+ or +;+ that is only part of a string value or a comment
69
+ # does not end the clause early; the offsets it reports still index the original text.
69
70
  SET_CLAUSE = /\bSET\b([^;]+?)(?:\bWHERE\b|\z)/im
70
71
  VALUES_CLAUSE = /\A\s*VALUES?\s*/i
71
72
  ON_DUPLICATE = /\AON\s+DUPLICATE\s+KEY\s+UPDATE\b/i
72
73
  ASSIGNMENT = /\A#{IDENTIFIER_CAP}\s*=\s*(.+)\z/m
73
74
  NULL_VALUE = /\ANULL\z/i
74
- QUOTES = ["'", '"', '`'].freeze
75
75
  # The body of a WHERE clause, ended at the first trailing-clause keyword or at the end of the
76
- # text. Like SET_CLAUSE, matched against a quote-blanked copy so a keyword that is only part of
77
- # a string value - +WHERE note = 'a ORDER BY b'+ - does not end the clause early.
76
+ # text. Like SET_CLAUSE, matched against a blanked copy so a keyword that is only part of a
77
+ # string value or a comment - +WHERE note = 'a ORDER BY b'+ - does not end the clause early.
78
78
  WHERE_CLAUSE = /
79
79
  \bWHERE\b([^;]+?)
80
80
  (?:\bGROUP\s+BY\b|\bHAVING\b|\bORDER\s+BY\b|\bLIMIT\b|
@@ -107,12 +107,10 @@ module AwsAdvancedRubyDriverWrapper
107
107
  CREATE_KEYWORD = /\ACREATE\b/
108
108
  DROP_KEYWORD = /\ADROP\b/
109
109
 
110
- # Whitespace and comments in front of a statement. Query instrumentation and ORMs prepend a
111
- # comment routinely, and it says nothing about what the statement does.
112
- # Wrapped in an atomic group, and matching whitespace one character at a time rather than in
113
- # +\s++ runs, so that leading whitespace and comments cannot be re-partitioned on backtracking
114
- # - which would be quadratic on input like "/* /* /* ...".
115
- LEADING_NOISE = %r{\A(?>(?:\s|/\*.*?\*/|--[^\n]*|#[^\n]*)+)}m
110
+ # Whitespace in front of a statement, which includes what any comment in front of it was
111
+ # blanked to (see +analyze+). Query instrumentation and ORMs prepend a comment routinely, and
112
+ # it says nothing about what the statement does.
113
+ LEADING_NOISE = /\A\s+/
116
114
  CTE_START = /\AWITH\s+(?:RECURSIVE\s+)?/i
117
115
  CTE_NAME = /\A#{IDENTIFIER_NC}\s*/
118
116
  CTE_AS = /\AAS\s+(?:(?:NOT\s+)?MATERIALIZED\s*)?/i
@@ -128,9 +126,15 @@ module AwsAdvancedRubyDriverWrapper
128
126
 
129
127
  module_function
130
128
 
131
- def analyze(sql)
129
+ # @param server_version [Integer, nil] the version of the server the statement is sent to, as
130
+ # MySQL numbers it, which decides whether a versioned +/*!+ comment is part of the statement
131
+ def analyze(sql, server_version: nil)
132
132
  return QueryAnalysis.unknown unless sql.is_a?(String) && !sql.strip.empty?
133
133
 
134
+ # A comment can sit anywhere whitespace can - between a keyword and its table, inside a
135
+ # column list, next to a placeholder - so every pattern below reads the text with its
136
+ # comments blanked out, and every scan of it below looks for literals alone.
137
+ sql = MysqlText.blank_comments(sql, server_version: server_version)
134
138
  body, preceding_parameters = statement_body(sql)
135
139
  return QueryAnalysis.unknown if body.nil?
136
140
 
@@ -210,8 +214,7 @@ module AwsAdvancedRubyDriverWrapper
210
214
  tables: tables.freeze,
211
215
  write_columns: [].freeze,
212
216
  where_columns: where_cols.freeze,
213
- for_update: FOR_UPDATE.match?(sql),
214
- parameterized: sql.include?('?')
217
+ for_update: FOR_UPDATE.match?(sql)
215
218
  )
216
219
  end
217
220
 
@@ -225,7 +228,6 @@ module AwsAdvancedRubyDriverWrapper
225
228
  write_columns: bound.freeze,
226
229
  where_columns: [].freeze,
227
230
  for_update: false,
228
- parameterized: sql.include?('?'),
229
231
  unbound_write_columns: unbound.freeze,
230
232
  write_columns_complete: complete
231
233
  )
@@ -245,7 +247,6 @@ module AwsAdvancedRubyDriverWrapper
245
247
  write_columns: set_cols.freeze,
246
248
  where_columns: where_cols.freeze,
247
249
  for_update: false,
248
- parameterized: sql.include?('?'),
249
250
  unbound_write_columns: unbound.freeze,
250
251
  write_columns_complete: complete
251
252
  )
@@ -279,7 +280,6 @@ module AwsAdvancedRubyDriverWrapper
279
280
  write_columns: [].freeze,
280
281
  where_columns: extract_where_columns(sql, first_index),
281
282
  for_update: false,
282
- parameterized: sql.include?('?'),
283
283
  unbound_write_columns: unbound.freeze,
284
284
  write_columns_complete: false
285
285
  )
@@ -303,8 +303,7 @@ module AwsAdvancedRubyDriverWrapper
303
303
  tables: table ? [table].freeze : [].freeze,
304
304
  write_columns: [].freeze,
305
305
  where_columns: where_cols.freeze,
306
- for_update: false,
307
- parameterized: sql.include?('?')
306
+ for_update: false
308
307
  )
309
308
  end
310
309
 
@@ -315,8 +314,7 @@ module AwsAdvancedRubyDriverWrapper
315
314
  tables: table ? [table].freeze : [].freeze,
316
315
  write_columns: [].freeze,
317
316
  where_columns: [].freeze,
318
- for_update: false,
319
- parameterized: false
317
+ for_update: false
320
318
  )
321
319
  end
322
320
 
@@ -327,8 +325,7 @@ module AwsAdvancedRubyDriverWrapper
327
325
  tables: table ? [table].freeze : [].freeze,
328
326
  write_columns: [].freeze,
329
327
  where_columns: [].freeze,
330
- for_update: false,
331
- parameterized: false
328
+ for_update: false
332
329
  )
333
330
  end
334
331
 
@@ -372,9 +369,10 @@ module AwsAdvancedRubyDriverWrapper
372
369
  complete = false unless values.length == columns.length
373
370
  columns.each_with_index do |column_name, position|
374
371
  value = values[position]
375
- if value == '?'
372
+ code = code_of(value.to_s)
373
+ if code == '?'
376
374
  bound << ColumnInfo.new(table_name: table_name, column_name: column_name, parameter_index: index)
377
- elsif !NULL_VALUE.match?(value.to_s)
375
+ elsif !NULL_VALUE.match?(code)
378
376
  unbound << ColumnInfo.new(table_name: table_name, column_name: column_name)
379
377
  end
380
378
  index += placeholder_count(value.to_s)
@@ -391,7 +389,7 @@ module AwsAdvancedRubyDriverWrapper
391
389
  # @param first_index [Integer] the number the statement's first bind parameter has
392
390
  # @return [Array(Array<ColumnInfo>, Array<ColumnInfo>, Boolean)] as extract_insert_columns
393
391
  def extract_set_columns(sql, table_name, first_index)
394
- match = SET_CLAUSE.match(mask_quoted_literals(sql))
392
+ match = SET_CLAUSE.match(mask_non_code(sql))
395
393
  return [[], [], false] unless match
396
394
 
397
395
  # The offsets come from the masked copy; the clause read from them is the original text.
@@ -440,10 +438,11 @@ module AwsAdvancedRubyDriverWrapper
440
438
  end
441
439
 
442
440
  value = match[2].strip
441
+ code = code_of(value)
443
442
  column = ColumnInfo.new(table_name: table_name, column_name: column_name_of(match[1]), parameter_index: index)
444
- if value == '?'
443
+ if code == '?'
445
444
  bound << column
446
- elsif !NULL_VALUE.match?(value)
445
+ elsif !NULL_VALUE.match?(code)
447
446
  unbound << column.with(parameter_index: nil)
448
447
  end
449
448
  index += placeholder_count(value)
@@ -462,14 +461,18 @@ module AwsAdvancedRubyDriverWrapper
462
461
  #
463
462
  # @param first_index [Integer] the number the statement's first bind parameter has
464
463
  def extract_where_columns(sql, first_index = 1)
465
- match = WHERE_CLAUSE.match(mask_quoted_literals(sql))
464
+ masked = mask_non_code(sql)
465
+ match = WHERE_CLAUSE.match(masked)
466
466
  return [].freeze unless match
467
467
 
468
468
  # The offsets come from the masked copy; the clause read from them is the original text.
469
+ # Its placeholders are counted on the masked copy, so that a question mark inside a string
470
+ # value is not taken for one.
469
471
  where_body = sql[match.begin(1)...match.end(1)]
470
- return [].freeze unless where_body.include?('?')
472
+ masked_body = masked[match.begin(1)...match.end(1)]
473
+ return [].freeze unless masked_body.include?('?')
471
474
 
472
- base = first_index + placeholder_count(sql[0...match.begin(1)])
475
+ base = first_index + masked[0...match.begin(1)].count('?')
473
476
 
474
477
  cols = []
475
478
  pos = 0
@@ -477,10 +480,10 @@ module AwsAdvancedRubyDriverWrapper
477
480
  pos = m.end(0)
478
481
  # Groups: 1=BETWEEN col, 2=BETWEEN sentinel, 3=+col OP ?+, 4=+? OP col+, 5=IN col, 6=LIKE col
479
482
  col = strip_quotes((m[1] || m[3] || m[4] || m[5] || m[6]).to_s)
480
- index = base + placeholder_count(where_body[0...m.begin(0)])
483
+ index = base + masked_body[0...m.begin(0)].count('?')
481
484
  # One entry per bound parameter the predicate consumes: one for +=+ or +LIKE+, two for a
482
485
  # +BETWEEN+, and one per placeholder for an +IN (?, ?, ...)+.
483
- placeholder_count(m[0]).times do |offset|
486
+ masked_body[m.begin(0)...m.end(0)].count('?').times do |offset|
484
487
  cols << ColumnInfo.new(table_name: nil, column_name: col, parameter_index: index + offset)
485
488
  end
486
489
  end
@@ -547,7 +550,7 @@ module AwsAdvancedRubyDriverWrapper
547
550
 
548
551
  depth = 0
549
552
  closed_at = nil
550
- unclosed_quote = each_unquoted_char(group) do |char, index|
553
+ unclosed = MysqlText.each_code_char(group, comments: false) do |char, index|
551
554
  next if closed_at
552
555
 
553
556
  # No else: only parentheses affect the depth, so every other character is ignored.
@@ -558,18 +561,18 @@ module AwsAdvancedRubyDriverWrapper
558
561
  closed_at = index if depth.zero?
559
562
  end
560
563
  end
561
- return [nil, nil] if unclosed_quote || closed_at.nil?
564
+ return [nil, nil] if unclosed || closed_at.nil?
562
565
 
563
566
  [group[1...closed_at], group[(closed_at + 1)..]]
564
567
  end
565
568
 
566
- # Splits a comma separated list, ignoring commas inside quotes or nested parentheses.
569
+ # Splits a comma separated list, ignoring commas inside quotes, comments or nested parentheses.
567
570
  #
568
- # @return [Array<String>, nil] nil when the quotes or parentheses do not balance
571
+ # @return [Array<String>, nil] nil when the quotes, comments or parentheses do not balance
569
572
  def split_top_level(text)
570
573
  depth = 0
571
574
  boundaries = []
572
- unclosed_quote = each_unquoted_char(text) do |char, index|
575
+ unclosed = MysqlText.each_code_char(text, comments: false) do |char, index|
573
576
  # No else: only parentheses and top-level commas matter, so every other character is ignored.
574
577
  case char
575
578
  when '(' then depth += 1
@@ -577,7 +580,7 @@ module AwsAdvancedRubyDriverWrapper
577
580
  when ',' then boundaries << index if depth.zero?
578
581
  end
579
582
  end
580
- return nil if unclosed_quote || !depth.zero?
583
+ return nil if unclosed || !depth.zero?
581
584
 
582
585
  from = 0
583
586
  parts = boundaries.map do |at|
@@ -589,51 +592,27 @@ module AwsAdvancedRubyDriverWrapper
589
592
  end
590
593
 
591
594
  # How many bind parameters a value expression consumes. A question mark inside a quoted
592
- # literal is not one of them.
595
+ # literal is not one of them, and neither was one inside a comment, which +analyze+ blanked.
593
596
  def placeholder_count(text)
594
597
  count = 0
595
- each_unquoted_char(text) { |char, _index| count += 1 if char == '?' }
598
+ MysqlText.each_code_char(text, comments: false) { |char, _index| count += 1 if char == '?' }
596
599
  count
597
600
  end
598
601
 
599
- # The text with the contents of every quoted literal, and the quote marks themselves, replaced
600
- # by spaces. Length and every offset are left unchanged, so a clause boundary found here with a
601
- # pattern slices the original text exactly, while a keyword or terminator that was only part of
602
- # a string value is no longer there to be found.
603
- def mask_quoted_literals(text)
602
+ # The text with every quoted literal, quotes included, replaced by spaces, on top of the
603
+ # comments +analyze+ blanked. Length and every offset are left unchanged, so a clause boundary
604
+ # found here with a pattern slices the original text exactly, while a keyword or terminator
605
+ # that was only part of a string value or a comment is no longer there to be found.
606
+ def mask_non_code(text)
604
607
  masked = ' ' * text.length
605
- each_unquoted_char(text) { |char, index| masked[index] = char }
608
+ MysqlText.each_code_char(text, comments: false) { |char, index| masked[index] = char }
606
609
  masked
607
610
  end
608
611
 
609
- # Walks the text once, yielding every character that is not inside a quoted literal along
610
- # with its index. Backslash escapes are skipped, and a doubled quote reads as one closing and
611
- # one opening quote, which leaves the state right either way.
612
- #
613
- # @return [String, nil] the quote character still open when the text ran out, nil when the
614
- # text ended outside a quoted literal
615
- def each_unquoted_char(text)
616
- index = 0
617
- quote = nil
618
-
619
- while index < text.length
620
- char = text[index]
621
- if quote
622
- if char == '\\' && quote != '`'
623
- index += 2
624
- next
625
- end
626
-
627
- quote = nil if char == quote
628
- elsif QUOTES.include?(char)
629
- quote = char
630
- else
631
- yield(char, index)
632
- end
633
- index += 1
634
- end
635
-
636
- quote
612
+ # A value expression with its literals taken out and the space a blanked comment left trimmed,
613
+ # so that a comment next to a lone placeholder or NULL does not hide what the value is.
614
+ def code_of(value)
615
+ mask_non_code(value).strip
637
616
  end
638
617
  end
639
618
  end
@@ -0,0 +1,215 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License").
6
+ # You may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+
17
+ module AwsAdvancedRubyDriverWrapper
18
+ module Utils
19
+ module Parser
20
+ # Reads MySQL statement text the way the server's lexer separates code from what is not code:
21
+ # quoted literals and comments. Anything that counts the +?+ placeholders of a statement has to
22
+ # skip both, since a +?+ inside either is not a placeholder, and miscounting one binds a value
23
+ # to the wrong column.
24
+ module MysqlText
25
+ # The quote characters a literal can be wrapped in. A backtick quotes an identifier rather than
26
+ # a value, but is included so a +?+ inside a backtick-quoted name is not miscounted either.
27
+ QUOTES = ["'", '"', '`'].freeze
28
+ # What a block comment that never closes is reported as by {each_code_char}.
29
+ UNCLOSED_COMMENT = '/*'
30
+
31
+ # The server version from which MySQL reads a six-digit version in a +/*!+ comment.
32
+ SIX_DIGIT_VERSIONS_FROM = 80_100
33
+
34
+ module_function
35
+
36
+ # Yields every character that is code, along with its index, skipping quoted literals and
37
+ # comments (see {each_segment}).
38
+ #
39
+ # @param server_version [Integer, nil] see {each_segment}
40
+ # @param comments [Boolean] see {each_segment}
41
+ # @return [String, nil] the quote character, or {UNCLOSED_COMMENT}, still open when the text
42
+ # ran out; nil when the text ended in code or in a line comment
43
+ def each_code_char(text, server_version: nil, comments: true)
44
+ each_segment(text, server_version: server_version, comments: comments) do |kind, from, to|
45
+ (from...to).each { |index| yield(text[index], index) } if kind == :code
46
+ end
47
+ end
48
+
49
+ # The text with every comment replaced by spaces, and its literals left as they are. Length and
50
+ # every offset are unchanged, so a pattern that has to see quoted names, which a comment can
51
+ # sit between, can be matched against it and its offsets used on the original text.
52
+ #
53
+ # What is left is to be read for its literals alone (+comments: false+), not scanned for
54
+ # comments again: blanking +0--/**/?+ leaves +0-- ?+, where the dashes now look like the
55
+ # start of a comment the server never saw.
56
+ #
57
+ # @param server_version [Integer, nil] see {each_segment}
58
+ def blank_comments(text, server_version: nil)
59
+ blanked = text.dup
60
+ each_segment(text, server_version: server_version) do |kind, from, to|
61
+ blanked[from...to] = ' ' * (to - from) if kind == :comment
62
+ end
63
+ blanked
64
+ end
65
+
66
+ # Walks the text once, yielding each run of code, quoted literal or comment as its kind and the
67
+ # range of indices it covers. A literal or block comment that never closes runs to the end.
68
+ #
69
+ # Inside a literal a backslash escapes the next character, except inside a backtick, and a
70
+ # doubled quote reads as one closing and one opening quote, which leaves the state right
71
+ # either way. A comment is +/* ... */+, +#+ to the end of the line, or +--+ to the end of the
72
+ # line when whitespace or a control character follows the dashes, as MySQL requires; without
73
+ # it the dashes are two minus signs.
74
+ #
75
+ # A +/*! ... */+ comment holds code that MySQL runs, so only its opening +/*!+ and its closing
76
+ # +*/+ are comment and what is between them is read as code. When the +!+ is followed by a
77
+ # version, +/*!80023 ... */+, MySQL runs the code only on a server of that version or later,
78
+ # and on an older one the whole of it is a comment. Which of the two it is depends on the
79
+ # server the statement is sent to, so the comment is read as code when that is not known.
80
+ #
81
+ # @param server_version [Integer, nil] the version of the server the statement is sent to, as
82
+ # MySQL numbers it (8.0.39 is 80039), or nil when it is not known
83
+ # @param comments [Boolean] whether to look for comments at all; false for text whose comments
84
+ # have already been blanked (see {blank_comments}), so that only literals are skipped
85
+ # @yieldparam kind [Symbol] +:code+, +:literal+ or +:comment+
86
+ # @return [String, nil] as {each_code_char}
87
+ def each_segment(text, server_version: nil, comments: true)
88
+ index = 0
89
+ code_from = 0
90
+ executable_open = false
91
+
92
+ while index < text.length
93
+ kind, finish = if comments
94
+ segment_at(text, index, server_version, executable_open)
95
+ elsif QUOTES.include?(text[index])
96
+ [:literal, literal_end(text, index)]
97
+ end
98
+ if kind.nil?
99
+ index += 1
100
+ next
101
+ end
102
+
103
+ case kind
104
+ when :executable_open then executable_open = true
105
+ when :executable_close then executable_open = false
106
+ end
107
+ yield(:code, code_from, index) if code_from < index
108
+ yield(kind == :literal ? :literal : :comment, index, finish || text.length)
109
+ return kind == :literal ? text[index] : UNCLOSED_COMMENT if finish.nil?
110
+
111
+ index = code_from = finish
112
+ end
113
+
114
+ yield(:code, code_from, index) if code_from < index
115
+ executable_open ? UNCLOSED_COMMENT : nil
116
+ end
117
+
118
+ # What starts at +index+, if it is not code.
119
+ #
120
+ # @return [Array(Symbol, Integer), nil] its kind and the index just past it, which is nil when it
121
+ # never ends; nil when the character is code
122
+ def segment_at(text, index, server_version, executable_open)
123
+ char = text[index]
124
+ if QUOTES.include?(char)
125
+ [:literal, literal_end(text, index)]
126
+ elsif text[index, 3] == '/*!'
127
+ executable_comment(text, index, server_version)
128
+ elsif executable_open && text[index, 2] == '*/'
129
+ [:executable_close, index + 2]
130
+ elsif text[index, 2] == '/*'
131
+ close = text.index('*/', index + 2)
132
+ [:comment, close && (close + 2)]
133
+ elsif char == '#' || dash_comment?(text, index)
134
+ [:comment, text.index("\n", index) || text.length]
135
+ end
136
+ end
137
+
138
+ # A +/*!+ comment: its opener and version when MySQL runs what it holds, or the whole of it when
139
+ # the version it names is newer than the server's.
140
+ def executable_comment(text, index, server_version)
141
+ version, digits = comment_version(text, index + 3, server_version)
142
+ return [:executable_open, index + 3 + digits] unless version && server_version && version > server_version
143
+
144
+ [:comment, skipped_comment_end(text, index + 3 + digits)]
145
+ end
146
+
147
+ # The version a +/*!+ comment names, read as MySQL reads it: six digits followed by whitespace
148
+ # on a server that takes six, otherwise five digits, and no version at all with fewer. The end
149
+ # of the comment right after six digits is not whitespace, so MySQL reads five of them then.
150
+ #
151
+ # @return [Array(Integer, Integer), Array(nil, Integer)] the version, or nil, and how many
152
+ # digits it took up
153
+ def comment_version(text, from, server_version)
154
+ six = text[from, 7].to_s
155
+ if server_version.to_i >= SIX_DIGIT_VERSIONS_FROM && six.match?(/\A\d{6}(?:\s|\z)/)
156
+ [six[0, 6].to_i, 6]
157
+ elsif text[from, 5].to_s.match?(/\A\d{5}\z/)
158
+ [text[from, 5].to_i, 5]
159
+ else
160
+ [nil, 0]
161
+ end
162
+ end
163
+
164
+ # Where a +/*!+ comment MySQL does not run ends. MySQL lets one comment be nested inside one of
165
+ # these, so the first +*/+ after a nested +/*+ closes the nested one rather than this one.
166
+ #
167
+ # @return [Integer, nil] the index just past its close, or nil when it never closes
168
+ def skipped_comment_end(text, from)
169
+ nested = false
170
+ index = from
171
+
172
+ while index < text.length
173
+ pair = text[index, 2]
174
+ if pair == '*/'
175
+ return index + 2 unless nested
176
+
177
+ nested = false
178
+ index += 2
179
+ elsif pair == '/*' && !nested
180
+ nested = true
181
+ index += 2
182
+ else
183
+ index += 1
184
+ end
185
+ end
186
+
187
+ nil
188
+ end
189
+
190
+ # @return [Integer, nil] the index just past the quote that closes the literal opened at
191
+ # +start+, or nil when it is never closed
192
+ def literal_end(text, start)
193
+ quote = text[start]
194
+ index = start + 1
195
+
196
+ while index < text.length
197
+ char = text[index]
198
+ return index + 1 if char == quote
199
+
200
+ index += char == '\\' && quote != '`' ? 2 : 1
201
+ end
202
+
203
+ nil
204
+ end
205
+
206
+ def dash_comment?(text, index)
207
+ return false unless text[index, 2] == '--'
208
+
209
+ following = text[index + 2]
210
+ following.nil? || following.match?(/[\s[:cntrl:]]/)
211
+ end
212
+ end
213
+ end
214
+ end
215
+ end
@@ -81,18 +81,19 @@ module AwsAdvancedRubyDriverWrapper
81
81
 
82
82
  # Dispatches a parsed statement hash to the appropriate extract_* method.
83
83
  # @param stmt [Hash] top-level statement hash from pg_query
84
- # @param parameterized [Boolean]
84
+ # @param parameterized [Boolean] whether the statement has bind parameters; a WHERE clause is
85
+ # only read for the columns it compares against them when it does
85
86
  # @return [QueryAnalysis]
86
87
  def extract_from_stmt(stmt, parameterized)
87
88
  case stmt
88
89
  in { select_stmt: inner_stmt } then extract_select(inner_stmt, parameterized)
89
- in { insert_stmt: inner_stmt } then extract_insert(inner_stmt, parameterized)
90
+ in { insert_stmt: inner_stmt } then extract_insert(inner_stmt)
90
91
  in { update_stmt: inner_stmt } then extract_update(inner_stmt, parameterized)
91
92
  in { delete_stmt: inner_stmt } then extract_delete(inner_stmt, parameterized)
92
93
  in { create_stmt: inner_stmt } then extract_create(inner_stmt)
93
94
  in { drop_stmt: inner_stmt } then extract_drop(inner_stmt)
94
95
  in { copy_stmt: inner_stmt } then extract_copy(inner_stmt, parameterized)
95
- in { merge_stmt: inner_stmt } then extract_merge(inner_stmt, parameterized)
96
+ in { merge_stmt: inner_stmt } then extract_merge(inner_stmt)
96
97
  in { prepare_stmt: inner_stmt } then extract_prepare(inner_stmt, parameterized)
97
98
  else QueryAnalysis.unknown
98
99
  end
@@ -121,7 +122,6 @@ module AwsAdvancedRubyDriverWrapper
121
122
  write_columns: analyses.flat_map(&:write_columns).freeze,
122
123
  where_columns: [].freeze,
123
124
  for_update: false,
124
- parameterized: parameterized,
125
125
  unbound_write_columns: analyses.flat_map(&:unbound_write_columns).freeze,
126
126
  write_columns_complete: analyses.all?(&:write_columns_complete)
127
127
  )
@@ -133,7 +133,7 @@ module AwsAdvancedRubyDriverWrapper
133
133
  # that only deletes stores nothing. A value written with something other than a bind parameter
134
134
  # is reported unbound, and an INSERT clause without a column list is reported as not
135
135
  # enumerable, so a write that cannot be read fails closed rather than every MERGE being refused.
136
- def extract_merge(stmt, parameterized)
136
+ def extract_merge(stmt)
137
137
  table = stmt.dig(:relation, :relname)
138
138
  bound = []
139
139
  unbound = []
@@ -166,7 +166,6 @@ module AwsAdvancedRubyDriverWrapper
166
166
  write_columns: bound.freeze,
167
167
  where_columns: [].freeze,
168
168
  for_update: false,
169
- parameterized: parameterized,
170
169
  unbound_write_columns: unbound.freeze,
171
170
  write_columns_complete: complete
172
171
  )
@@ -197,12 +196,11 @@ module AwsAdvancedRubyDriverWrapper
197
196
  tables: tables.freeze,
198
197
  write_columns: [].freeze,
199
198
  where_columns: where_cols.freeze,
200
- for_update: for_update,
201
- parameterized: parameterized
199
+ for_update: for_update
202
200
  )
203
201
  end
204
202
 
205
- def extract_insert(stmt, parameterized)
203
+ def extract_insert(stmt)
206
204
  table = stmt.dig(:relation, :relname)
207
205
  declared = Array(stmt[:cols]).filter_map { |column_entry| column_entry.dig(:res_target, :name) }
208
206
  value_rows = Array(stmt.dig(:select_stmt, :select_stmt, :values_lists))
@@ -215,7 +213,6 @@ module AwsAdvancedRubyDriverWrapper
215
213
  write_columns: (bound + upsert_bound).freeze,
216
214
  where_columns: [].freeze,
217
215
  for_update: false,
218
- parameterized: parameterized,
219
216
  unbound_write_columns: (unbound + upsert_unbound).freeze,
220
217
  write_columns_complete: complete
221
218
  )
@@ -232,7 +229,6 @@ module AwsAdvancedRubyDriverWrapper
232
229
  write_columns: set_cols.freeze,
233
230
  where_columns: where_cols.freeze,
234
231
  for_update: false,
235
- parameterized: parameterized,
236
232
  unbound_write_columns: unbound.freeze
237
233
  )
238
234
  end
@@ -305,8 +301,7 @@ module AwsAdvancedRubyDriverWrapper
305
301
  tables: table ? [table].freeze : [].freeze,
306
302
  write_columns: [].freeze,
307
303
  where_columns: where_cols.freeze,
308
- for_update: false,
309
- parameterized: parameterized
304
+ for_update: false
310
305
  )
311
306
  end
312
307
 
@@ -317,8 +312,7 @@ module AwsAdvancedRubyDriverWrapper
317
312
  tables: table ? [table].freeze : [].freeze,
318
313
  write_columns: [].freeze,
319
314
  where_columns: [].freeze,
320
- for_update: false,
321
- parameterized: false
315
+ for_update: false
322
316
  )
323
317
  end
324
318
 
@@ -332,8 +326,7 @@ module AwsAdvancedRubyDriverWrapper
332
326
  tables: tables.freeze,
333
327
  write_columns: [].freeze,
334
328
  where_columns: [].freeze,
335
- for_update: false,
336
- parameterized: false
329
+ for_update: false
337
330
  )
338
331
  end
339
332
 
@@ -355,7 +348,7 @@ module AwsAdvancedRubyDriverWrapper
355
348
  return extract_from_stmt(query, parameterized) if query.is_a?(Hash) && !query.empty?
356
349
 
357
350
  return QueryAnalysis.new(query_type: QueryType::SELECT, tables: tables, write_columns: [].freeze,
358
- where_columns: [].freeze, for_update: false, parameterized: false)
351
+ where_columns: [].freeze, for_update: false)
359
352
  end
360
353
 
361
354
  declared = Array(stmt[:attlist]).filter_map { |column_entry| column_entry.dig(:string, :sval) }
@@ -365,7 +358,6 @@ module AwsAdvancedRubyDriverWrapper
365
358
  write_columns: [].freeze,
366
359
  where_columns: [].freeze,
367
360
  for_update: false,
368
- parameterized: false,
369
361
  unbound_write_columns: declared.map { |name| ColumnInfo.new(table_name: table, column_name: name) }.freeze,
370
362
  write_columns_complete: declared.any?
371
363
  )
@@ -492,7 +484,6 @@ module AwsAdvancedRubyDriverWrapper
492
484
  write_columns: [].freeze,
493
485
  where_columns: [].freeze,
494
486
  for_update: false,
495
- parameterized: sql.match?(/\$\d+/), # fallback only; AST unavailable here
496
487
  write_columns_complete: false
497
488
  )
498
489
  end
@@ -29,8 +29,8 @@ module AwsAdvancedRubyDriverWrapper
29
29
  # at all, which is what an INSERT with no column list, an +INSERT ... SELECT+, and a statement
30
30
  # that would not parse all look like.
31
31
  QueryAnalysis = Data.define(:query_type, :tables, :write_columns, :where_columns, :for_update,
32
- :parameterized, :unbound_write_columns, :write_columns_complete) do
33
- def initialize(query_type:, tables:, write_columns:, where_columns:, for_update:, parameterized:,
32
+ :unbound_write_columns, :write_columns_complete) do
33
+ def initialize(query_type:, tables:, write_columns:, where_columns:, for_update:,
34
34
  unbound_write_columns: [].freeze, write_columns_complete: true)
35
35
  super
36
36
  end
@@ -44,7 +44,6 @@ module AwsAdvancedRubyDriverWrapper
44
44
  write_columns: [].freeze,
45
45
  where_columns: [].freeze,
46
46
  for_update: false,
47
- parameterized: false,
48
47
  write_columns_complete: false
49
48
  )
50
49
  end