regexp_parser 2.12.0 → 2.13.0

data/lib/regexp_parser/scanner.rb

@@ -11,43 +11,10 @@ require_relative 'scanner/errors/premature_end_error'
 require_relative 'scanner/errors/validation_error'
 
 class Regexp::Scanner
-  # Scans the given regular expression text, or Regexp object and collects the
-  # emitted token into an array that gets returned at the end. If a block is
-  # given, it gets called for each emitted token.
-  #
-  # This method may raise errors if a syntax error is encountered.
-  # --------------------------------------------------------------------------
-  def self.scan(input_object, options: nil, collect_tokens: true, &block)
-    new.scan(input_object, options: options, collect_tokens: collect_tokens, &block)
-  end
-
-  def scan(input_object, options: nil, collect_tokens: true, &block)
-    self.collect_tokens = collect_tokens
-    self.literal_run = nil
-    stack = []
+  # Emit fixed machine data once. The ragel task converts its generated
+  # accessors and tables to constants with frozen values.
 
-    input = input_object.is_a?(Regexp) ? input_object.source : input_object
-    self.free_spacing = free_spacing?(input_object, options)
-    self.regexp_encoding = extract_encoding(input_object, options)
-    self.spacing_stack = [{:free_spacing => free_spacing, :depth => 0}]
-
-    data = input.unpack("c*")
-    eof = data.length
-
-    self.tokens = []
-    self.block = block
-
-    self.set_depth = 0
-    self.group_depth = 0
-    self.capturing_group_count = 0
-    self.conditional_stack = []
-    self.char_pos = 0
-
-class << self
-	attr_accessor :_re_scanner_trans_keys
-	private :_re_scanner_trans_keys, :_re_scanner_trans_keys=
-end
-self._re_scanner_trans_keys = [
+RE_SCANNER_TRANS_KEYS = [
 0,0,-128,-65,-128,-65,
 -128,-65,41,41,39,
 57,39,39,33,62,
@@ -98,13 +65,9 @@ self._re_scanner_trans_keys = [
 92,39,60,39,57,
 48,57,41,57,33,62,
 0
-]
+].freeze
 
-class << self
-	attr_accessor :_re_scanner_key_spans
-	private :_re_scanner_key_spans, :_re_scanner_key_spans=
-end
-self._re_scanner_key_spans = [
+RE_SCANNER_KEY_SPANS = [
 0,64,64,64,1,19,1,30,
 1,22,19,1,10,19,10,19,
 30,1,10,20,10,15,22,19,
@@ -122,13 +85,9 @@ self._re_scanner_key_spans = [
 10,10,10,10,1,1,0,33,
 1,0,1,55,55,1,22,19,
 10,17,30
-]
+].freeze
 
-class << self
-	attr_accessor :_re_scanner_index_offsets
-	private :_re_scanner_index_offsets, :_re_scanner_index_offsets=
-end
-self._re_scanner_index_offsets = [
+RE_SCANNER_INDEX_OFFSETS = [
 0,0,65,130,195,197,217,219,
 250,252,275,295,297,308,328,339,
 359,390,392,403,424,435,451,474,
@@ -146,13 +105,9 @@ self._re_scanner_index_offsets = [
 5196,5207,5218,5229,5240,5242,5244,5245,
 5279,5281,5282,5284,5340,5396,5398,5421,
 5441,5452,5470
-]
+].freeze
 
-class << self
-	attr_accessor :_re_scanner_indicies
-	private :_re_scanner_indicies, :_re_scanner_indicies=
-end
-self._re_scanner_indicies = [
+RE_SCANNER_INDICIES = [
 1,1,1,1,1,1,1,1,
 1,1,1,1,1,1,1,1,
 1,1,1,1,1,1,1,1,
@@ -841,13 +796,9 @@ self._re_scanner_indicies = [
 96,96,211,96,96,211,211,211,
 211,211,211,211,211,211,211,96,
 96,96,211,93,96,0
-]
+].freeze
 
-class << self
-	attr_accessor :_re_scanner_trans_targs
-	private :_re_scanner_trans_targs, :_re_scanner_trans_targs=
-end
-self._re_scanner_trans_targs = [
+RE_SCANNER_TRANS_TARGS = [
 76,77,1,2,76,4,76,6,
 76,8,76,76,10,16,11,76,
 12,13,14,15,17,18,19,20,
@@ -875,13 +826,9 @@ self._re_scanner_trans_targs = [
 114,106,117,106,53,106,120,122,
 106,56,106,106,106,69,126,127,
 128,129,130,126
-]
+].freeze
 
-class << self
-	attr_accessor :_re_scanner_trans_actions
-	private :_re_scanner_trans_actions, :_re_scanner_trans_actions=
-end
-self._re_scanner_trans_actions = [
+RE_SCANNER_TRANS_ACTIONS = [
 1,2,0,0,3,0,4,0,
 5,0,6,7,0,0,0,8,
 0,0,0,0,0,0,0,0,
@@ -909,13 +856,9 @@ self._re_scanner_trans_actions = [
 0,89,33,90,0,91,33,0,
 92,0,93,94,95,0,96,33,
 33,33,33,97
-]
+].freeze
 
-class << self
-	attr_accessor :_re_scanner_to_state_actions
-	private :_re_scanner_to_state_actions, :_re_scanner_to_state_actions=
-end
-self._re_scanner_to_state_actions = [
+RE_SCANNER_TO_STATE_ACTIONS = [
 0,0,0,0,0,0,0,0,
 0,0,0,0,0,0,0,0,
 0,0,0,0,0,0,0,0,
@@ -933,13 +876,9 @@ self._re_scanner_to_state_actions = [
 0,0,0,0,0,0,0,0,
 0,0,0,0,0,0,64,0,
 0,0,0
-]
+].freeze
 
-class << self
-	attr_accessor :_re_scanner_from_state_actions
-	private :_re_scanner_from_state_actions, :_re_scanner_from_state_actions=
-end
-self._re_scanner_from_state_actions = [
+RE_SCANNER_FROM_STATE_ACTIONS = [
 0,0,0,0,0,0,0,0,
 0,0,0,0,0,0,0,0,
 0,0,0,0,0,0,0,0,
@@ -957,13 +896,9 @@ self._re_scanner_from_state_actions = [
 0,0,0,0,0,0,0,0,
 0,0,0,0,0,0,37,0,
 0,0,0
-]
+].freeze
 
-class << self
-	attr_accessor :_re_scanner_eof_actions
-	private :_re_scanner_eof_actions, :_re_scanner_eof_actions=
-end
-self._re_scanner_eof_actions = [
+RE_SCANNER_EOF_ACTIONS = [
 0,0,0,0,0,0,0,0,
 0,0,0,0,0,0,0,0,
 0,0,0,0,0,0,0,0,
@@ -981,13 +916,9 @@ self._re_scanner_eof_actions = [
 0,0,0,0,0,0,0,0,
 0,0,0,0,0,0,0,0,
 0,0,0
-]
+].freeze
 
-class << self
-	attr_accessor :_re_scanner_eof_trans
-	private :_re_scanner_eof_trans, :_re_scanner_eof_trans=
-end
-self._re_scanner_eof_trans = [
+RE_SCANNER_EOF_TRANS = [
 0,1,1,1,5,5,5,5,
 5,12,12,12,12,12,12,12,
 12,12,12,12,12,12,12,12,
@@ -1005,54 +936,58 @@ self._re_scanner_eof_trans = [
 191,191,191,191,194,194,194,198,
 198,198,198,203,203,205,0,212,
 212,212,212
-]
+].freeze
 
-class << self
-	attr_accessor :re_scanner_start
-end
-self.re_scanner_start = 76;
-class << self
-	attr_accessor :re_scanner_first_final
-end
-self.re_scanner_first_final = 76;
-class << self
-	attr_accessor :re_scanner_error
-end
-self.re_scanner_error = 0;
+RE_SCANNER_START = 76.freeze
+RE_SCANNER_FIRST_FINAL = 76.freeze
+RE_SCANNER_ERROR = 0.freeze
 
-class << self
-	attr_accessor :re_scanner_en_char_type
-end
-self.re_scanner_en_char_type = 91;
-class << self
-	attr_accessor :re_scanner_en_unicode_property
-end
-self.re_scanner_en_unicode_property = 92;
-class << self
-	attr_accessor :re_scanner_en_character_set
-end
-self.re_scanner_en_character_set = 93;
-class << self
-	attr_accessor :re_scanner_en_set_escape_sequence
-end
-self.re_scanner_en_set_escape_sequence = 100;
-class << self
-	attr_accessor :re_scanner_en_escape_sequence
-end
-self.re_scanner_en_escape_sequence = 106;
-class << self
-	attr_accessor :re_scanner_en_conditional_expression
-end
-self.re_scanner_en_conditional_expression = 126;
-class << self
-	attr_accessor :re_scanner_en_main
-end
-self.re_scanner_en_main = 76;
+RE_SCANNER_EN_CHAR_TYPE = 91.freeze
+RE_SCANNER_EN_UNICODE_PROPERTY = 92.freeze
+RE_SCANNER_EN_CHARACTER_SET = 93.freeze
+RE_SCANNER_EN_SET_ESCAPE_SEQUENCE = 100.freeze
+RE_SCANNER_EN_ESCAPE_SEQUENCE = 106.freeze
+RE_SCANNER_EN_CONDITIONAL_EXPRESSION = 126.freeze
+RE_SCANNER_EN_MAIN = 76.freeze
+
+  # Scans the given regular expression text, or Regexp object and collects the
+  # emitted token into an array that gets returned at the end. If a block is
+  # given, it gets called for each emitted token.
+  #
+  # This method may raise errors if a syntax error is encountered.
+  # --------------------------------------------------------------------------
+  def self.scan(input_object, options: nil, collect_tokens: true, &block)
+    new.scan(input_object, options: options, collect_tokens: collect_tokens, &block)
+  end
+
+  def scan(input_object, options: nil, collect_tokens: true, &block)
+    self.collect_tokens = collect_tokens
+    self.literal_run = nil
+    stack = []
+
+    input = input_object.is_a?(Regexp) ? input_object.source : input_object
+    self.free_spacing = free_spacing?(input_object, options)
+    self.regexp_encoding = extract_encoding(input_object, options)
+    self.spacing_stack = [{:free_spacing => free_spacing, :depth => 0}]
+
+    # Keep token text stable if a callback mutates the caller's String.
+    source = input.dup.freeze
+    data = source.unpack("c*")
+    eof = data.length
+
+    self.tokens = []
+    self.block = block
+
+    self.set_depth = 0
+    self.group_depth = 0
+    self.capturing_group_count = 0
+    self.conditional_stack = []
+    self.char_pos = 0
 
 begin
 	p ||= 0
 	pe ||= data.length
-	cs = re_scanner_start
+	cs = RE_SCANNER_START
 	top = 0
 	ts = nil
 	te = nil
@@ -1080,29 +1015,29 @@ begin
 	end
 	end
 	if _goto_level <= _resume
-	case _re_scanner_from_state_actions[cs]
+	case RE_SCANNER_FROM_STATE_ACTIONS[cs]
 	when 37
 		begin
 ts = p
 		end
 	end
 	_keys = cs << 1
-	_inds = _re_scanner_index_offsets[cs]
-	_slen = _re_scanner_key_spans[cs]
+	_inds = RE_SCANNER_INDEX_OFFSETS[cs]
+	_slen = RE_SCANNER_KEY_SPANS[cs]
 	_wide = data[p].ord
 	_trans = if ( _slen > 0 &&
-			_re_scanner_trans_keys[_keys] <= _wide &&
-			_wide <= _re_scanner_trans_keys[_keys + 1]
+			RE_SCANNER_TRANS_KEYS[_keys] <= _wide &&
+			_wide <= RE_SCANNER_TRANS_KEYS[_keys + 1]
 		    )
-			_re_scanner_indicies[ _inds + _wide - _re_scanner_trans_keys[_keys] ]
+			RE_SCANNER_INDICIES[ _inds + _wide - RE_SCANNER_TRANS_KEYS[_keys] ]
 		 else
-			_re_scanner_indicies[ _inds + _slen ]
+			RE_SCANNER_INDICIES[ _inds + _slen ]
 		 end
 	end
 	if _goto_level <= _eof_trans
-	cs = _re_scanner_trans_targs[_trans]
-	if _re_scanner_trans_actions[_trans] != 0
-	case _re_scanner_trans_actions[_trans]
+	cs = RE_SCANNER_TRANS_TARGS[_trans]
+	if RE_SCANNER_TRANS_ACTIONS[_trans] != 0
+	case RE_SCANNER_TRANS_ACTIONS[_trans]
 	when 39
 		begin
  self.group_depth = group_depth + 1 		end
@@ -1114,7 +1049,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      case text = copy(data, ts-1,te)
+      case text = copy(source, ts-1,te)
       when '\d'; emit(:type, :digit, text)
       when '\D'; emit(:type, :nondigit, text)
       when '\h'; emit(:type, :hex, text)
@@ -1139,7 +1074,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      text = copy(data, ts-1,te)
+      text = copy(source, ts-1,te)
       type = (text[1] == 'P') ^ (text[3] == '^') ? :nonproperty : :property
 
       name = text[3..-2].gsub(/[\^\s_\-]/, '').downcase
@@ -1202,14 +1137,14 @@ te = p+1
 		begin
 te = p+1
  begin
-      emit(:literal, :literal, copy(data, ts, te))
+      emit(:literal, :literal, copy(source, ts, te))
      end
 		end
 	when 15
 		begin
 te = p+1
  begin
-      text = copy(data, ts, te)
+      text = copy(source, ts, te)
       emit(:literal, :literal, text)
      end
 		end
@@ -1245,7 +1180,7 @@ p = p - 1; begin
 		begin
 te = p
 p = p - 1; begin
-      text = copy(data, ts, te)
+      text = copy(source, ts, te)
       emit(:literal, :literal, text)
      end
 		end
@@ -1281,7 +1216,7 @@ p = p - 1; begin
 		begin
  begin p = ((te))-1; end
  begin
-      text = copy(data, ts, te)
+      text = copy(source, ts, te)
       emit(:literal, :literal, text)
      end
 		end
@@ -1289,7 +1224,7 @@ p = p - 1; begin
 		begin
 te = p+1
  begin
-      emit(:escape, :octal, copy(data, ts-1,te))
+      emit(:escape, :octal, copy(source, ts-1,te))
       	begin
 		top -= 1
 		cs = stack[top]
@@ -1319,7 +1254,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      emit(:escape, :literal, copy(data, ts-1,te))
+      emit(:escape, :literal, copy(source, ts-1,te))
       	begin
 		top -= 1
 		cs = stack[top]
@@ -1333,7 +1268,7 @@ te = p+1
 		begin
 te = p
 p = p - 1; begin
-      emit(:escape, :octal, copy(data, ts-1,te))
+      emit(:escape, :octal, copy(source, ts-1,te))
       	begin
 		top -= 1
 		cs = stack[top]
@@ -1347,7 +1282,7 @@ p = p - 1; begin
 		begin
 te = p
 p = p - 1; begin
-      emit(:escape, :literal, copy(data, ts-1,te))
+      emit(:escape, :literal, copy(source, ts-1,te))
       	begin
 		top -= 1
 		cs = stack[top]
@@ -1361,7 +1296,7 @@ p = p - 1; begin
 		begin
  begin p = ((te))-1; end
  begin
-      emit(:escape, :literal, copy(data, ts-1,te))
+      emit(:escape, :literal, copy(source, ts-1,te))
       	begin
 		top -= 1
 		cs = stack[top]
@@ -1375,7 +1310,7 @@ p = p - 1; begin
 		begin
 te = p+1
  begin
-      emit(:escape, :octal, copy(data, ts-1,te))
+      emit(:escape, :octal, copy(source, ts-1,te))
       	begin
 		top -= 1
 		cs = stack[top]
@@ -1389,7 +1324,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      case text = copy(data, ts-1,te)
+      case text = copy(source, ts-1,te)
       when '\.'; emit(:escape, :dot, text)
       when '\|'; emit(:escape, :alternation, text)
       when '\^'; emit(:escape, :bol, text)
@@ -1421,7 +1356,7 @@ te = p+1
  begin
       # \b is emitted as backspace only when inside a character set, otherwise
       # it is a word boundary anchor. A syntax might "normalize" it if needed.
-      case text = copy(data, ts-1,te)
+      case text = copy(source, ts-1,te)
       when '\a'; emit(:escape, :bell, text)
       when '\b'; emit(:escape, :backspace, text)
       when '\e'; emit(:escape, :escape, text)
@@ -1444,7 +1379,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      text = copy(data, ts-1,te)
+      text = copy(source, ts-1,te)
       if text[2] == '{'
         emit(:escape, :codepoint_list, text)
       else
@@ -1463,7 +1398,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      emit(:escape, :hex, copy(data, ts-1,te))
+      emit(:escape, :hex, copy(source, ts-1,te))
       	begin
 		top -= 1
 		cs = stack[top]
@@ -1477,7 +1412,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      emit_meta_control_sequence(data, ts, te, :control)
+      emit_meta_control_sequence(data, source, ts, te, :control)
       	begin
 		top -= 1
 		cs = stack[top]
@@ -1491,7 +1426,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      emit_meta_control_sequence(data, ts, te, :meta_sequence)
+      emit_meta_control_sequence(data, source, ts, te, :meta_sequence)
       	begin
 		top -= 1
 		cs = stack[top]
@@ -1537,7 +1472,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      emit(:escape, :literal, copy(data, ts-1,te))
+      emit(:escape, :literal, copy(source, ts-1,te))
       	begin
 		top -= 1
 		cs = stack[top]
@@ -1551,7 +1486,7 @@ te = p+1
 		begin
 te = p
 p = p - 1; begin
-      text = copy(data, ts-1,te)
+      text = copy(source, ts-1,te)
 
       # If not enough groups have been opened, there is a fallback to either an
       # octal or literal interpretation for 2+ digit numerical escapes.
@@ -1578,7 +1513,7 @@ p = p - 1; begin
 		begin
 te = p
 p = p - 1; begin
-      emit(:escape, :octal, copy(data, ts-1,te))
+      emit(:escape, :octal, copy(source, ts-1,te))
       	begin
 		top -= 1
 		cs = stack[top]
@@ -1592,7 +1527,7 @@ p = p - 1; begin
 		begin
 te = p
 p = p - 1; begin
-      text = copy(data, ts-1,te)
+      text = copy(source, ts-1,te)
       if regexp_encoding == Encoding::BINARY
         text.split(/(?=\\)/).each { |part| emit(:escape, :hex, part) }
       else
@@ -1611,7 +1546,7 @@ p = p - 1; begin
 		begin
 te = p
 p = p - 1; begin
-      emit(:escape, :hex, copy(data, ts-1,te))
+      emit(:escape, :hex, copy(source, ts-1,te))
       	begin
 		top -= 1
 		cs = stack[top]
@@ -1625,7 +1560,7 @@ p = p - 1; begin
 		begin
 te = p
 p = p - 1; begin
-      emit_meta_control_sequence(data, ts, te, :control)
+      emit_meta_control_sequence(data, source, ts, te, :control)
       	begin
 		top -= 1
 		cs = stack[top]
@@ -1639,7 +1574,7 @@ p = p - 1; begin
 		begin
 te = p
 p = p - 1; begin
-      emit_meta_control_sequence(data, ts, te, :meta_sequence)
+      emit_meta_control_sequence(data, source, ts, te, :meta_sequence)
       	begin
 		top -= 1
 		cs = stack[top]
@@ -1653,7 +1588,7 @@ p = p - 1; begin
 		begin
 te = p
 p = p - 1; begin
-      emit(:escape, :literal, copy(data, ts-1,te))
+      emit(:escape, :literal, copy(source, ts-1,te))
       	begin
 		top -= 1
 		cs = stack[top]
@@ -1667,7 +1602,7 @@ p = p - 1; begin
 		begin
  begin p = ((te))-1; end
  begin
-      text = copy(data, ts-1,te)
+      text = copy(source, ts-1,te)
       if regexp_encoding == Encoding::BINARY
         text.split(/(?=\\)/).each { |part| emit(:escape, :hex, part) }
       else
@@ -1686,7 +1621,7 @@ p = p - 1; begin
 		begin
  begin p = ((te))-1; end
  begin
-      emit(:escape, :literal, copy(data, ts-1,te))
+      emit(:escape, :literal, copy(source, ts-1,te))
       	begin
 		top -= 1
 		cs = stack[top]
@@ -1700,7 +1635,7 @@ p = p - 1; begin
 		begin
 te = p+1
  begin
-      text = copy(data, ts, te-1)
+      text = copy(source, ts, te-1)
       text =~ /[^0]/ or raise ValidationError.for(:backref, 'condition', 'invalid ref ID')
       emit(:conditional, :condition, text)
       emit(:conditional, :condition_close, ')')
@@ -1755,7 +1690,7 @@ p = p - 1; begin
 		begin
 te = p+1
  begin
-      emit(:meta, :dot, copy(data, ts, te))
+      emit(:meta, :dot, copy(source, ts, te))
      end
 		end
 	when 45
@@ -1763,9 +1698,9 @@ te = p+1
 te = p+1
  begin
       if conditional_stack.last == group_depth
-        emit(:conditional, :separator, copy(data, ts, te))
+        emit(:conditional, :separator, copy(source, ts, te))
       else
-        emit(:meta, :alternation, copy(data, ts, te))
+        emit(:meta, :alternation, copy(source, ts, te))
       end
      end
 		end
@@ -1773,28 +1708,28 @@ te = p+1
 		begin
 te = p+1
  begin
-      emit(:anchor, :bol, copy(data, ts, te))
+      emit(:anchor, :bol, copy(source, ts, te))
      end
 		end
 	when 38
 		begin
 te = p+1
  begin
-      emit(:anchor, :eol, copy(data, ts, te))
+      emit(:anchor, :eol, copy(source, ts, te))
      end
 		end
 	when 62
 		begin
 te = p+1
  begin
-      emit(:keep, :mark, copy(data, ts, te))
+      emit(:keep, :mark, copy(source, ts, te))
      end
 		end
 	when 61
 		begin
 te = p+1
  begin
-      case text = copy(data, ts, te)
+      case text = copy(source, ts, te)
       when '\A'; emit(:anchor, :bos, text)
       when '\z'; emit(:anchor, :eos, text)
       when '\Z'; emit(:anchor, :eos_ob_eol, text)
@@ -1808,14 +1743,14 @@ te = p+1
 		begin
 te = p+1
  begin
-      append_literal(data, ts, te)
+      append_literal(source, ts, te)
      end
 		end
 	when 52
 		begin
 te = p+1
  begin
-      text = copy(data, ts, te)
+      text = copy(source, ts, te)
 
       conditional_stack << group_depth
 
@@ -1835,7 +1770,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      text = copy(data, ts, te)
+      text = copy(source, ts, te)
       if text[2..-1] =~ /([^\-mixdau:]|^$)|-.*([dau])/
         raise ValidationError.for(:group_option, $1 || "-#{$2}", text)
       end
@@ -1846,7 +1781,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      case text = copy(data, ts, te)
+      case text = copy(source, ts, te)
       when '(?='; emit(:assertion, :lookahead, text)
       when '(?!'; emit(:assertion, :nlookahead, text)
       when '(?<='; emit(:assertion, :lookbehind, text)
@@ -1858,7 +1793,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      case text = copy(data, ts, te)
+      case text = copy(source, ts, te)
       when '(?:'; emit(:group, :passive, text)
       when '(?>'; emit(:group, :atomic, text)
       when '(?~'; emit(:group, :absence, text)
@@ -1879,7 +1814,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      case text = copy(data, ts, te)
+      case text = copy(source, ts, te)
       when /^\\k(.)[^0-9\-][^+\-]*['>]$/
         emit(:backref, $1 == '<' ? :name_ref_ab : :name_ref_sq, text)
       when /^\\k(.)0*[1-9]\d*['>]$/
@@ -1899,7 +1834,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      case text = copy(data, ts, te)
+      case text = copy(source, ts, te)
       when /^\\g(.)[^0-9+\-].*['>]$/
         emit(:backref, $1 == '<' ? :name_call_ab : :name_call_sq, text)
       when /^\\g(.)(?:0|0*[1-9]\d*)['>]$/
@@ -1915,7 +1850,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      case text = copy(data, ts, te)
+      case text = copy(source, ts, te)
       when '?' ; emit(:quantifier, :zero_or_one, text)
       when '??'; emit(:quantifier, :zero_or_one_reluctant, text)
       when '?+'; emit(:quantifier, :zero_or_one_possessive, text)
@@ -1926,7 +1861,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      case text = copy(data, ts, te)
+      case text = copy(source, ts, te)
       when '*' ; emit(:quantifier, :zero_or_more, text)
       when '*?'; emit(:quantifier, :zero_or_more_reluctant, text)
       when '*+'; emit(:quantifier, :zero_or_more_possessive, text)
@@ -1937,7 +1872,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      case text = copy(data, ts, te)
+      case text = copy(source, ts, te)
       when '+' ; emit(:quantifier, :one_or_more, text)
       when '+?'; emit(:quantifier, :one_or_more_reluctant, text)
       when '++'; emit(:quantifier, :one_or_more_possessive, text)
@@ -1948,7 +1883,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      emit(:quantifier, :interval, copy(data, ts, te))
+      emit(:quantifier, :interval, copy(source, ts, te))
      end
 		end
 	when 49
@@ -1956,10 +1891,10 @@ te = p+1
 te = p+1
  begin
       if free_spacing
-        emit(:free_space, :comment, copy(data, ts, te))
+        emit(:free_space, :comment, copy(source, ts, te))
       else
         # consume only the pound sign (#) and backtrack to do regular scanning
-        append_literal(data, ts, ts + 1)
+        append_literal(source, ts, ts + 1)
          begin p = (( ts + 1))-1; end
 
       end
@@ -1969,7 +1904,7 @@ te = p+1
 		begin
 te = p
 p = p - 1; begin
-      text = copy(data, ts, te)
+      text = copy(source, ts, te)
       if text[2..-1] =~ /([^\-mixdau:]|^$)|-.*([dau])/
         raise ValidationError.for(:group_option, $1 || "-#{$2}", text)
       end
@@ -1981,7 +1916,7 @@ p = p - 1; begin
 te = p
 p = p - 1; begin
       self.capturing_group_count = capturing_group_count + 1
-      text = copy(data, ts, te)
+      text = copy(source, ts, te)
       emit(:group, :capture, text)
      end
 		end
@@ -1989,7 +1924,7 @@ p = p - 1; begin
 		begin
 te = p
 p = p - 1; begin
-      case text = copy(data, ts, te)
+      case text = copy(source, ts, te)
       when '?' ; emit(:quantifier, :zero_or_one, text)
       when '??'; emit(:quantifier, :zero_or_one_reluctant, text)
       when '?+'; emit(:quantifier, :zero_or_one_possessive, text)
@@ -2000,7 +1935,7 @@ p = p - 1; begin
 		begin
 te = p
 p = p - 1; begin
-      case text = copy(data, ts, te)
+      case text = copy(source, ts, te)
       when '*' ; emit(:quantifier, :zero_or_more, text)
       when '*?'; emit(:quantifier, :zero_or_more_reluctant, text)
       when '*+'; emit(:quantifier, :zero_or_more_possessive, text)
@@ -2011,7 +1946,7 @@ p = p - 1; begin
 		begin
 te = p
 p = p - 1; begin
-      case text = copy(data, ts, te)
+      case text = copy(source, ts, te)
       when '+' ; emit(:quantifier, :one_or_more, text)
       when '+?'; emit(:quantifier, :one_or_more_reluctant, text)
       when '++'; emit(:quantifier, :one_or_more_possessive, text)
@@ -2022,7 +1957,7 @@ p = p - 1; begin
 		begin
 te = p
 p = p - 1; begin
-      append_literal(data, ts, te)
+      append_literal(source, ts, te)
      end
 		end
 	when 60
@@ -2044,10 +1979,10 @@ p = p - 1; begin
 te = p
 p = p - 1; begin
       if free_spacing
-        emit(:free_space, :comment, copy(data, ts, te))
+        emit(:free_space, :comment, copy(source, ts, te))
       else
         # consume only the pound sign (#) and backtrack to do regular scanning
-        append_literal(data, ts, ts + 1)
+        append_literal(source, ts, ts + 1)
          begin p = (( ts + 1))-1; end
 
       end
@@ -2058,9 +1993,9 @@ p = p - 1; begin
 te = p
 p = p - 1; begin
       if free_spacing
-        emit(:free_space, :whitespace, copy(data, ts, te))
+        emit(:free_space, :whitespace, copy(source, ts, te))
       else
-        append_literal(data, ts, te)
+        append_literal(source, ts, te)
       end
      end
 		end
@@ -2068,14 +2003,14 @@ p = p - 1; begin
 		begin
 te = p
 p = p - 1; begin
-      append_literal(data, ts, te)
+      append_literal(source, ts, te)
      end
 		end
 	when 3
 		begin
  begin p = ((te))-1; end
  begin
-      text = copy(data, ts, te)
+      text = copy(source, ts, te)
       if text[2..-1] =~ /([^\-mixdau:]|^$)|-.*([dau])/
         raise ValidationError.for(:group_option, $1 || "-#{$2}", text)
       end
@@ -2086,7 +2021,7 @@ p = p - 1; begin
 		begin
  begin p = ((te))-1; end
  begin
-      append_literal(data, ts, te)
+      append_literal(source, ts, te)
      end
 		end
 	when 7
@@ -2116,14 +2051,14 @@ end
 	when 58
 	begin begin p = ((te))-1; end
 
-      append_literal(data, ts, te)
+      append_literal(source, ts, te)
     end
 end
 			end
 	when 75
 		begin
 
-    text = copy(data, ts ? ts-1 : 0,-1)
+    text = error_text(source, ts, p)
     raise PrematureEndError.new(text)
   		end
 		begin
@@ -2143,7 +2078,7 @@ p = p - 1; begin
 	when 18
 		begin
 
-    text = copy(data, ts ? ts-1 : 0,-1)
+    text = error_text(source, ts, p)
     raise PrematureEndError.new(text)
   		end
 		begin
@@ -2163,13 +2098,13 @@ p = p - 1; begin
 	when 89
 		begin
 
-    text = copy(data, ts ? ts-1 : 0,-1)
+    text = error_text(source, ts, p)
     raise PrematureEndError.new(text)
   		end
 		begin
 te = p
 p = p - 1; begin
-      emit_meta_control_sequence(data, ts, te, :control)
+      emit_meta_control_sequence(data, source, ts, te, :control)
       	begin
 		top -= 1
 		cs = stack[top]
@@ -2182,13 +2117,13 @@ p = p - 1; begin
 	when 91
 		begin
 
-    text = copy(data, ts ? ts-1 : 0,-1)
+    text = error_text(source, ts, p)
     raise PrematureEndError.new(text)
   		end
 		begin
 te = p
 p = p - 1; begin
-      emit_meta_control_sequence(data, ts, te, :meta_sequence)
+      emit_meta_control_sequence(data, source, ts, te, :meta_sequence)
       	begin
 		top -= 1
 		cs = stack[top]
@@ -2201,13 +2136,13 @@ p = p - 1; begin
 	when 27
 		begin
 
-    text = copy(data, ts ? ts-1 : 0,-1)
+    text = error_text(source, ts, p)
     raise PrematureEndError.new(text)
   		end
 		begin
  begin p = ((te))-1; end
  begin
-      emit_meta_control_sequence(data, ts, te, :control)
+      emit_meta_control_sequence(data, source, ts, te, :control)
       	begin
 		top -= 1
 		cs = stack[top]
@@ -2220,13 +2155,13 @@ p = p - 1; begin
 	when 29
 		begin
 
-    text = copy(data, ts ? ts-1 : 0,-1)
+    text = error_text(source, ts, p)
     raise PrematureEndError.new(text)
   		end
 		begin
  begin p = ((te))-1; end
  begin
-      emit_meta_control_sequence(data, ts, te, :meta_sequence)
+      emit_meta_control_sequence(data, source, ts, te, :meta_sequence)
       	begin
 		top -= 1
 		cs = stack[top]
@@ -2239,7 +2174,7 @@ p = p - 1; begin
 	when 31
 		begin
 
-    text = copy(data, ts ? ts-1 : 0,-1)
+    text = error_text(source, ts, p)
     raise ValidationError.for(:sequence, 'sequence', text)
   		end
 		begin
@@ -2260,7 +2195,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      emit(:group, :comment, copy(data, ts, te))
+      emit(:group, :comment, copy(source, ts, te))
      end
 		end
 	when 40
@@ -2291,7 +2226,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      emit(:set, :open, copy(data, ts, te))
+      emit(:set, :open, copy(source, ts, te))
       	begin
 		stack[top] = cs
 		top+= 1
@@ -2308,7 +2243,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      emit(:set, :close, copy(data, ts, te))
+      emit(:set, :close, copy(source, ts, te))
       if in_set?
         	begin
 		top -= 1
@@ -2359,7 +2294,7 @@ te = p+1
 		begin
 te = p+1
  begin
-      text = copy(data, ts, te)
+      text = copy(source, ts, te)
 
       type = :posixclass
       class_name = text[2..-3]
@@ -2391,7 +2326,7 @@ act = 58;		end
 	end
 	end
 	if _goto_level <= _again
-	case _re_scanner_to_state_actions[cs]
+	case RE_SCANNER_TO_STATE_ACTIONS[cs]
 	when 64
 		begin
 ts = nil;		end
@@ -2415,12 +2350,12 @@ act = 0
 	end
 	if _goto_level <= _test_eof
 	if p == eof
-	if _re_scanner_eof_trans[cs] > 0
-		_trans = _re_scanner_eof_trans[cs] - 1;
+	if RE_SCANNER_EOF_TRANS[cs] > 0
+		_trans = RE_SCANNER_EOF_TRANS[cs] - 1;
 		_goto_level = _eof_trans
 		next;
 	end
-	  case _re_scanner_eof_actions[cs]
+	  case RE_SCANNER_EOF_ACTIONS[cs]
 	when 12
 		begin
 
@@ -2429,7 +2364,7 @@ act = 0
 	when 25
 		begin
 
-    text = copy(data, ts ? ts-1 : 0,-1)
+    text = error_text(source, ts, p)
     raise PrematureEndError.new(text)
   		end
 	  end
@@ -2445,9 +2380,9 @@ end
     # to avoid "warning: assigned but unused variable - testEof"
     testEof = testEof
 
-    if cs == re_scanner_error
-      text = copy(data, ts ? ts-1 : 0,-1)
-      raise ScannerError.new("Scan error at '#{text}'")
+    if cs == RE_SCANNER_ERROR
+      text = error_text(source, ts, p)
+      raise ScannerError.new("Scan error at #{text}")
     end
 
     raise PrematureEndError.new("(missing group closing paranthesis) "+
@@ -2545,15 +2480,28 @@ end
     set_depth > 0
   end
 
-  # Copy from ts to te from data as text
-  def copy(data, ts, te)
-    data[ts...te].pack('c*').force_encoding('utf-8')
+  # Ragel offsets are bytes; emitted token positions are characters.
+  def copy(source, ts, te)
+    source.byteslice(ts, te - ts).force_encoding('utf-8')
+  end
+
+  def error_text(source, ts, position)
+    start = ts ? [ts - 1,0].max : 0
+    finish = source.bytesize
+    if position < finish
+      # Include the complete offending UTF-8 character, but no following text.
+      character = source.byteslice(position, 4).force_encoding('utf-8').each_char.first
+      finish = position + character.bytesize
+    end
+    text = copy(source, start, finish)
+		text = "…#{text[-9..-1]}" if text.length > 10
+		text
   end
 
   # Appends one or more characters to the literal buffer, to be emitted later
   # by a call to emit_literal.
-  def append_literal(data, ts, te)
-    (self.literal_run ||= []) << copy(data, ts, te)
+  def append_literal(source, ts, te)
+    (self.literal_run ||= []) << copy(source, ts, te)
   end
 
   # Emits the literal run collected by calls to the append_literal method.
@@ -2592,10 +2540,10 @@ end
     emit(:group, token, text)
   end
 
-  def emit_meta_control_sequence(data, ts, te, token)
+  def emit_meta_control_sequence(data, source, ts, te, token)
     if data.last < 0x00 || data.last > 0x7F
       raise ValidationError.for(:sequence, 'escape', token.to_s)
     end
-    emit(:escape, token, copy(data, ts-1,te))
+    emit(:escape, token, copy(source, ts-1,te))
   end
-end # module Regexp::Scanner
+end