regexp_parser 2.6.1 → 2.7.0

data/lib/regexp_parser/scanner.rb

@@ -1,8 +1,8 @@
 # -*- warn-indent:false;  -*-
 
-# line 1 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 1 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 
-# line 646 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 650 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 
 
 # THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY
@@ -77,23 +77,24 @@ class Regexp::Scanner
   #
   # This method may raise errors if a syntax error is encountered.
   # --------------------------------------------------------------------------
-  def self.scan(input_object, options: nil, &block)
-    new.scan(input_object, options: options, &block)
+  def self.scan(input_object, options: nil, collect_tokens: true, &block)
+    new.scan(input_object, options: options, collect_tokens: collect_tokens, &block)
   end
 
-  def scan(input_object, options: nil, &block)
-    self.literal = nil
+  def scan(input_object, options: nil, collect_tokens: true, &block)
+    self.collect_tokens = collect_tokens
+    self.literal_run = nil
     stack = []
 
     input = input_object.is_a?(Regexp) ? input_object.source : input_object
     self.free_spacing = free_spacing?(input_object, options)
     self.spacing_stack = [{:free_spacing => free_spacing, :depth => 0}]
 
-    data  = input.unpack("c*") if input.is_a?(String)
+    data  = input.unpack("c*")
     eof   = data.length
 
     self.tokens = []
-    self.block  = block_given? ? block : nil
+    self.block  = block
 
     self.set_depth = 0
     self.group_depth = 0
@@ -101,7 +102,7 @@ class Regexp::Scanner
     self.char_pos = 0
 
     
-# line 104 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner.rb"
+# line 105 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner.rb"
 class << self
 	attr_accessor :_re_scanner_trans_keys
 	private :_re_scanner_trans_keys, :_re_scanner_trans_keys=
@@ -143,15 +144,16 @@ self._re_scanner_trans_keys = [
 	44, 57, 68, 119, 80, 112, 
 	-62, 125, -128, -65, -128, 
 	-65, -128, -65, 38, 38, 
-	38, 93, 58, 58, 67, 120, 
-	-62, 125, -128, -65, -128, 
-	-65, -128, -65, 48, 55, 
-	48, 55, 77, 77, 45, 45, 
-	0, 0, 67, 99, 45, 
-	45, 0, 0, 92, 92, 
-	48, 102, 39, 60, 39, 57, 
-	49, 57, 41, 57, 33, 
-	62, 0
+	38, 93, 58, 58, 48, 120, 
+	48, 55, 48, 55, -62, 
+	125, -128, -65, -128, -65, 
+	-128, -65, 48, 55, 48, 55, 
+	77, 77, 45, 45, 0, 
+	0, 67, 99, 45, 45, 
+	0, 0, 92, 92, 48, 102, 
+	39, 60, 39, 57, 49, 
+	57, 41, 57, 33, 62, 
+	0
 ]
 
 class << self
@@ -171,9 +173,9 @@ self._re_scanner_key_spans = [
 	1, 256, 51, 127, 127, 24, 94, 1, 
 	1, 94, 94, 1, 21, 21, 21, 58, 
 	14, 52, 33, 188, 64, 64, 64, 1, 
-	56, 1, 54, 188, 64, 64, 64, 8, 
-	8, 1, 1, 0, 33, 1, 0, 1, 
-	55, 22, 19, 9, 17, 30
+	56, 1, 73, 8, 8, 188, 64, 64, 
+	64, 8, 8, 1, 1, 0, 33, 1, 
+	0, 1, 55, 22, 19, 9, 17, 30
 ]
 
 class << self
@@ -193,9 +195,9 @@ self._re_scanner_index_offsets = [
 	2633, 2635, 2892, 2944, 3072, 3200, 3225, 3320, 
 	3322, 3324, 3419, 3514, 3516, 3538, 3560, 3582, 
 	3641, 3656, 3709, 3743, 3932, 3997, 4062, 4127, 
-	4129, 4186, 4188, 4243, 4432, 4497, 4562, 4627, 
-	4636, 4645, 4647, 4649, 4650, 4684, 4686, 4687, 
-	4689, 4745, 4768, 4788, 4798, 4816
+	4129, 4186, 4188, 4262, 4271, 4280, 4469, 4534, 
+	4599, 4664, 4673, 4682, 4684, 4686, 4687, 4721, 
+	4723, 4724, 4726, 4782, 4805, 4825, 4835, 4853
 ]
 
 class << self
@@ -726,38 +728,42 @@ self._re_scanner_indicies = [
 	152, 152, 152, 152, 152, 152, 152, 152, 
 	152, 152, 152, 152, 152, 152, 152, 152, 
 	152, 152, 152, 152, 152, 152, 152, 152, 
-	154, 152, 56, 156, 158, 158, 157, 157, 
-	157, 158, 157, 157, 157, 157, 158, 157, 
-	157, 158, 157, 157, 158, 157, 157, 157, 
-	158, 157, 157, 157, 158, 158, 158, 157, 
-	157, 157, 158, 158, 158, 158, 158, 158, 
-	157, 158, 157, 157, 157, 157, 157, 158, 
-	157, 158, 157, 158, 158, 158, 158, 158, 
-	158, 158, 157, 159, 159, 159, 159, 159, 
-	159, 159, 159, 159, 159, 159, 159, 159, 
-	159, 159, 159, 159, 159, 159, 159, 159, 
-	159, 159, 159, 159, 159, 159, 159, 159, 
-	159, 160, 160, 160, 160, 160, 160, 160, 
-	160, 160, 160, 160, 160, 160, 160, 160, 
-	160, 161, 161, 161, 161, 161, 61, 61, 
-	61, 61, 61, 61, 61, 61, 61, 61, 
+	154, 152, 56, 156, 158, 158, 158, 158, 
+	158, 158, 158, 158, 157, 157, 157, 157, 
+	157, 157, 157, 157, 157, 157, 157, 159, 
+	159, 157, 157, 157, 159, 157, 157, 157, 
+	157, 159, 157, 157, 159, 157, 157, 159, 
+	157, 157, 157, 159, 157, 157, 157, 159, 
+	159, 159, 157, 157, 157, 159, 159, 159, 
+	159, 159, 159, 157, 159, 157, 157, 157, 
+	157, 157, 159, 157, 159, 157, 159, 159, 
+	159, 159, 159, 159, 159, 157, 161, 161, 
+	161, 161, 161, 161, 161, 161, 160, 162, 
+	162, 162, 162, 162, 162, 162, 162, 160, 
+	163, 163, 163, 163, 163, 163, 163, 163, 
+	163, 163, 163, 163, 163, 163, 163, 163, 
+	163, 163, 163, 163, 163, 163, 163, 163, 
+	163, 163, 163, 163, 163, 163, 164, 164, 
+	164, 164, 164, 164, 164, 164, 164, 164, 
+	164, 164, 164, 164, 164, 164, 165, 165, 
+	165, 165, 165, 61, 61, 61, 61, 61, 
 	61, 61, 61, 61, 61, 61, 61, 61, 
 	61, 61, 61, 61, 61, 61, 61, 61, 
 	61, 61, 61, 61, 61, 61, 61, 61, 
 	61, 61, 61, 61, 61, 61, 61, 61, 
-	61, 61, 61, 61, 61, 162, 61, 61, 
-	61, 162, 162, 162, 162, 61, 61, 162, 
-	61, 163, 164, 164, 164, 164, 164, 164, 
-	164, 165, 165, 61, 61, 61, 61, 61, 
-	162, 61, 46, 46, 166, 167, 61, 61, 
-	46, 167, 61, 61, 46, 61, 168, 61, 
-	61, 169, 61, 167, 167, 61, 61, 61, 
-	167, 167, 61, 46, 162, 162, 162, 162, 
-	61, 61, 170, 170, 63, 167, 170, 170, 
-	61, 167, 61, 61, 61, 61, 61, 170, 
-	61, 169, 61, 170, 167, 170, 171, 170, 
-	167, 172, 61, 46, 162, 162, 162, 61, 
 	61, 61, 61, 61, 61, 61, 61, 61, 
+	61, 61, 166, 61, 61, 61, 166, 166, 
+	166, 166, 61, 61, 166, 61, 167, 168, 
+	168, 168, 168, 168, 168, 168, 169, 169, 
+	61, 61, 61, 61, 61, 166, 61, 46, 
+	46, 170, 171, 61, 61, 46, 171, 61, 
+	61, 46, 61, 172, 61, 61, 173, 61, 
+	171, 171, 61, 61, 61, 171, 171, 61, 
+	46, 166, 166, 166, 166, 61, 61, 174, 
+	174, 63, 171, 174, 174, 61, 171, 61, 
+	61, 61, 61, 61, 174, 61, 173, 61, 
+	174, 171, 174, 175, 174, 171, 176, 61, 
+	46, 166, 166, 166, 61, 61, 61, 61, 
 	61, 61, 61, 61, 61, 61, 61, 61, 
 	61, 61, 61, 61, 61, 61, 61, 61, 
 	61, 61, 61, 61, 61, 61, 61, 61, 
@@ -765,7 +771,7 @@ self._re_scanner_indicies = [
 	61, 61, 61, 61, 61, 61, 61, 61, 
 	61, 61, 61, 61, 61, 61, 61, 61, 
 	61, 61, 61, 61, 61, 61, 61, 61, 
-	173, 62, 62, 62, 62, 62, 62, 62, 
+	61, 61, 61, 61, 61, 177, 62, 62, 
 	62, 62, 62, 62, 62, 62, 62, 62, 
 	62, 62, 62, 62, 62, 62, 62, 62, 
 	62, 62, 62, 62, 62, 62, 62, 62, 
@@ -773,42 +779,43 @@ self._re_scanner_indicies = [
 	62, 62, 62, 62, 62, 62, 62, 62, 
 	62, 62, 62, 62, 62, 62, 62, 62, 
 	62, 62, 62, 62, 62, 62, 62, 62, 
-	62, 173, 174, 174, 174, 174, 174, 174, 
-	174, 174, 174, 174, 174, 174, 174, 174, 
-	174, 174, 174, 174, 174, 174, 174, 174, 
-	174, 174, 174, 174, 174, 174, 174, 174, 
-	174, 174, 174, 174, 174, 174, 174, 174, 
-	174, 174, 174, 174, 174, 174, 174, 174, 
-	174, 174, 174, 174, 174, 174, 174, 174, 
-	174, 174, 174, 174, 174, 174, 174, 174, 
-	174, 174, 173, 176, 176, 176, 176, 176, 
-	176, 176, 176, 175, 178, 178, 178, 178, 
-	178, 178, 178, 178, 177, 180, 64, 182, 
-	181, 64, 184, 69, 69, 69, 69, 69, 
+	62, 62, 62, 62, 62, 62, 177, 178, 
+	178, 178, 178, 178, 178, 178, 178, 178, 
+	178, 178, 178, 178, 178, 178, 178, 178, 
+	178, 178, 178, 178, 178, 178, 178, 178, 
+	178, 178, 178, 178, 178, 178, 178, 178, 
+	178, 178, 178, 178, 178, 178, 178, 178, 
+	178, 178, 178, 178, 178, 178, 178, 178, 
+	178, 178, 178, 178, 178, 178, 178, 178, 
+	178, 178, 178, 178, 178, 178, 178, 177, 
+	180, 180, 180, 180, 180, 180, 180, 180, 
+	179, 182, 182, 182, 182, 182, 182, 182, 
+	182, 181, 184, 64, 186, 185, 64, 188, 
 	69, 69, 69, 69, 69, 69, 69, 69, 
 	69, 69, 69, 69, 69, 69, 69, 69, 
 	69, 69, 69, 69, 69, 69, 69, 69, 
-	69, 69, 185, 69, 187, 186, 69, 72, 
-	69, 189, 189, 189, 189, 189, 189, 189, 
-	189, 189, 189, 188, 188, 188, 188, 188, 
-	188, 188, 189, 189, 189, 189, 189, 189, 
-	188, 188, 188, 188, 188, 188, 188, 188, 
-	188, 188, 188, 188, 188, 188, 188, 188, 
-	188, 188, 188, 188, 188, 188, 188, 188, 
-	188, 188, 189, 189, 189, 189, 189, 189, 
-	188, 191, 190, 190, 190, 190, 190, 192, 
-	190, 190, 190, 193, 193, 193, 193, 193, 
-	193, 193, 193, 193, 190, 190, 194, 190, 
-	88, 87, 87, 87, 87, 87, 195, 87, 
-	87, 195, 195, 195, 195, 195, 195, 195, 
-	195, 195, 195, 87, 90, 90, 90, 90, 
-	90, 90, 90, 90, 90, 195, 89, 195, 
-	195, 195, 195, 195, 195, 90, 90, 90, 
-	90, 90, 90, 90, 90, 90, 90, 195, 
-	195, 91, 91, 91, 91, 91, 91, 91, 
-	91, 91, 91, 91, 195, 91, 91, 195, 
-	195, 195, 195, 195, 195, 195, 195, 195, 
-	195, 91, 91, 91, 91, 88, 91, 0
+	69, 69, 69, 69, 69, 69, 69, 189, 
+	69, 191, 190, 69, 72, 69, 193, 193, 
+	193, 193, 193, 193, 193, 193, 193, 193, 
+	192, 192, 192, 192, 192, 192, 192, 193, 
+	193, 193, 193, 193, 193, 192, 192, 192, 
+	192, 192, 192, 192, 192, 192, 192, 192, 
+	192, 192, 192, 192, 192, 192, 192, 192, 
+	192, 192, 192, 192, 192, 192, 192, 193, 
+	193, 193, 193, 193, 193, 192, 195, 194, 
+	194, 194, 194, 194, 196, 194, 194, 194, 
+	197, 197, 197, 197, 197, 197, 197, 197, 
+	197, 194, 194, 198, 194, 88, 87, 87, 
+	87, 87, 87, 199, 87, 87, 199, 199, 
+	199, 199, 199, 199, 199, 199, 199, 199, 
+	87, 90, 90, 90, 90, 90, 90, 90, 
+	90, 90, 199, 89, 199, 199, 199, 199, 
+	199, 199, 90, 90, 90, 90, 90, 90, 
+	90, 90, 90, 90, 199, 199, 91, 91, 
+	91, 91, 91, 91, 91, 91, 91, 91, 
+	91, 199, 91, 91, 199, 199, 199, 199, 
+	199, 199, 199, 199, 199, 199, 91, 91, 
+	91, 91, 88, 91, 0
 ]
 
 class << self
@@ -823,11 +830,11 @@ self._re_scanner_trans_targs = [
 	27, 29, 28, 30, 32, 33, 35, 34, 
 	36, 73, 38, 73, 39, 41, 0, 42, 
 	43, 90, 91, 91, 44, 91, 91, 91, 
-	47, 48, 91, 91, 99, 99, 49, 52, 
-	99, 105, 99, 107, 55, 99, 108, 99, 
-	110, 58, 61, 59, 60, 99, 62, 63, 
-	64, 65, 66, 67, 99, 112, 113, 69, 
-	70, 113, 71, 72, 3, 75, 76, 77, 
+	47, 48, 91, 91, 101, 101, 49, 52, 
+	101, 107, 101, 109, 55, 101, 110, 101, 
+	112, 58, 61, 59, 60, 101, 62, 63, 
+	64, 65, 66, 67, 101, 114, 115, 69, 
+	70, 115, 71, 72, 3, 75, 76, 77, 
 	78, 79, 73, 80, 73, 84, 85, 73, 
 	86, 73, 87, 73, 73, 88, 73, 73, 
 	73, 73, 73, 73, 81, 73, 82, 5, 
@@ -835,12 +842,12 @@ self._re_scanner_trans_targs = [
 	73, 73, 73, 73, 73, 9, 24, 73, 
 	37, 89, 40, 92, 93, 94, 91, 95, 
 	96, 97, 91, 91, 91, 91, 45, 91, 
-	91, 46, 91, 91, 91, 98, 98, 100, 
-	101, 102, 99, 103, 103, 99, 51, 99, 
-	54, 99, 99, 57, 68, 99, 50, 99, 
-	104, 99, 99, 99, 106, 99, 53, 99, 
-	109, 111, 99, 56, 99, 99, 113, 114, 
-	115, 116, 117, 113
+	91, 46, 91, 91, 91, 98, 99, 98, 
+	98, 100, 98, 102, 103, 104, 101, 105, 
+	105, 101, 51, 101, 54, 101, 101, 57, 
+	68, 101, 50, 101, 106, 101, 101, 101, 
+	108, 101, 53, 101, 111, 113, 101, 56, 
+	101, 101, 115, 116, 117, 118, 119, 115
 ]
 
 class << self
@@ -867,12 +874,12 @@ self._re_scanner_trans_actions = [
 	59, 60, 61, 62, 63, 0, 0, 64, 
 	0, 66, 0, 0, 41, 41, 67, 0, 
 	41, 68, 69, 70, 71, 72, 0, 73, 
-	74, 0, 75, 76, 77, 78, 79, 0, 
-	41, 41, 80, 81, 82, 83, 0, 84, 
-	0, 85, 86, 0, 0, 87, 0, 88, 
-	0, 89, 90, 91, 41, 92, 0, 93, 
-	41, 0, 94, 0, 95, 96, 97, 41, 
-	41, 41, 41, 98
+	74, 0, 75, 76, 77, 78, 0, 79, 
+	80, 0, 81, 0, 41, 41, 82, 83, 
+	84, 85, 0, 86, 0, 87, 88, 0, 
+	0, 89, 0, 90, 0, 91, 92, 93, 
+	41, 94, 0, 95, 41, 0, 96, 0, 
+	97, 98, 99, 41, 41, 41, 41, 100
 ]
 
 class << self
@@ -892,9 +899,9 @@ self._re_scanner_to_state_actions = [
 	0, 34, 0, 0, 0, 0, 0, 0, 
 	0, 0, 0, 0, 0, 0, 0, 0, 
 	0, 65, 65, 65, 0, 0, 0, 0, 
-	0, 0, 65, 65, 0, 0, 0, 0, 
+	0, 0, 65, 0, 0, 65, 0, 0, 
 	0, 0, 0, 0, 0, 0, 0, 0, 
-	0, 65, 0, 0, 0, 0
+	0, 0, 0, 65, 0, 0, 0, 0
 ]
 
 class << self
@@ -914,9 +921,9 @@ self._re_scanner_from_state_actions = [
 	0, 35, 0, 0, 0, 0, 0, 0, 
 	0, 0, 0, 0, 0, 0, 0, 0, 
 	0, 35, 35, 35, 0, 0, 0, 0, 
-	0, 0, 35, 35, 0, 0, 0, 0, 
+	0, 0, 35, 0, 0, 35, 0, 0, 
 	0, 0, 0, 0, 0, 0, 0, 0, 
-	0, 35, 0, 0, 0, 0
+	0, 0, 0, 35, 0, 0, 0, 0
 ]
 
 class << self
@@ -936,9 +943,9 @@ self._re_scanner_eof_actions = [
 	0, 0, 0, 0, 0, 0, 0, 0, 
 	0, 0, 0, 0, 0, 0, 0, 0, 
 	0, 0, 0, 25, 0, 0, 0, 0, 
-	0, 0, 0, 25, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 25, 0, 0, 
 	0, 0, 0, 0, 0, 0, 0, 0, 
-	0, 0, 0, 0, 0, 0
+	0, 0, 0, 0, 0, 0, 0, 0
 ]
 
 class << self
@@ -958,9 +965,9 @@ self._re_scanner_eof_trans = [
 	87, 0, 112, 112, 113, 113, 112, 114, 
 	116, 118, 118, 124, 125, 127, 129, 131, 
 	136, 0, 0, 0, 150, 150, 150, 150, 
-	153, 156, 0, 0, 174, 174, 174, 176, 
-	178, 180, 180, 180, 184, 184, 184, 184, 
-	189, 0, 196, 196, 196, 196
+	153, 156, 0, 161, 161, 0, 178, 178, 
+	178, 180, 182, 184, 184, 184, 188, 188, 
+	188, 188, 193, 0, 200, 200, 200, 200
 ]
 
 class << self
@@ -995,20 +1002,20 @@ self.re_scanner_en_set_escape_sequence = 98;
 class << self
 	attr_accessor :re_scanner_en_escape_sequence
 end
-self.re_scanner_en_escape_sequence = 99;
+self.re_scanner_en_escape_sequence = 101;
 class << self
 	attr_accessor :re_scanner_en_conditional_expression
 end
-self.re_scanner_en_conditional_expression = 113;
+self.re_scanner_en_conditional_expression = 115;
 class << self
 	attr_accessor :re_scanner_en_main
 end
 self.re_scanner_en_main = 73;
 
 
-# line 744 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 749 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
     
-# line 1011 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner.rb"
+# line 1018 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner.rb"
 begin
 	p ||= 0
 	pe ||= data.length
@@ -1019,9 +1026,9 @@ begin
 	act = 0
 end
 
-# line 745 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 750 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
     
-# line 1024 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner.rb"
+# line 1031 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner.rb"
 begin
 	testEof = false
 	_slen, _trans, _keys, _inds, _acts, _nacts = nil
@@ -1049,7 +1056,7 @@ begin
 		begin
 ts = p
 		end
-# line 1052 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner.rb"
+# line 1059 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner.rb"
 	end
 	_keys = cs << 1
 	_inds = _re_scanner_index_offsets[cs]
@@ -1069,11 +1076,11 @@ ts = p
 	if _re_scanner_trans_actions[_trans] != 0
 	case _re_scanner_trans_actions[_trans]
 	when 37 then
-# line 152 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 147 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  self.group_depth = group_depth + 1 		end
 	when 4 then
-# line 153 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 148 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  self.group_depth = group_depth - 1 		end
 	when 41 then
@@ -1082,7 +1089,7 @@ ts = p
 te = p+1
 		end
 	when 66 then
-# line 12 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/char_type.rl"
+# line 12 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/char_type.rl"
 		begin
 te = p+1
  begin 
@@ -1108,14 +1115,14 @@ te = p+1
      end
 		end
 	when 15 then
-# line 16 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/property.rl"
+# line 16 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/property.rl"
 		begin
 te = p+1
  begin 
       text = copy(data, ts-1, te)
       type = (text[1] == 'P') ^ (text[3] == '^') ? :nonproperty : :property
 
-      name = data[ts+2..te-2].pack('c*').gsub(/[\^\s_\-]/, '').downcase
+      name = text[3..-2].gsub(/[\^\s_\-]/, '').downcase
 
       token = self.class.short_prop_map[name] || self.class.long_prop_map[name]
       validation_error(:property, name) unless token
@@ -1132,7 +1139,7 @@ te = p+1
      end
 		end
 	when 19 then
-# line 180 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 175 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin  # special case, emits two tokens
@@ -1141,28 +1148,27 @@ te = p+1
      end
 		end
 	when 71 then
-# line 185 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 180 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
-      text = copy(data, ts, te)
-      if tokens.last[1] == :open
-        emit(:set, :negate, text)
+      if prev_token[1] == :open
+        emit(:set, :negate, '^')
       else
-        emit(:literal, :literal, text)
+        emit(:literal, :literal, '^')
       end
      end
 		end
 	when 73 then
-# line 206 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 200 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
-      emit(:set, :intersection, copy(data, ts, te))
+      emit(:set, :intersection, '&&')
      end
 		end
 	when 69 then
-# line 210 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 204 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1177,7 +1183,7 @@ te = p+1
      end
 		end
 	when 67 then
-# line 244 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 238 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1185,7 +1191,7 @@ te = p+1
      end
 		end
 	when 17 then
-# line 248 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 242 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1194,25 +1200,25 @@ te = p+1
      end
 		end
 	when 74 then
-# line 194 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 188 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
-      text = copy(data, ts, te)
-      # ranges cant start with a subset or intersection/negation/range operator
-      if tokens.last[0] == :set
-        emit(:literal, :literal, text)
+      # ranges cant start with the opening bracket, a subset, or
+      # intersection/negation/range operators
+      if prev_token[0] == :set
+        emit(:literal, :literal, '-')
       else
-        emit(:set, :range, text)
+        emit(:set, :range, '-')
       end
      end
 		end
 	when 77 then
-# line 214 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 208 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
-      emit(:set, :open, copy(data, ts, te))
+      emit(:set, :open, '[')
       	begin
 		stack[top] = cs
 		top+= 1
@@ -1224,7 +1230,7 @@ p = p - 1; begin
      end
 		end
 	when 72 then
-# line 248 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 242 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
@@ -1233,25 +1239,25 @@ p = p - 1; begin
      end
 		end
 	when 18 then
-# line 194 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 188 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  begin p = ((te))-1; end
  begin 
-      text = copy(data, ts, te)
-      # ranges cant start with a subset or intersection/negation/range operator
-      if tokens.last[0] == :set
-        emit(:literal, :literal, text)
+      # ranges cant start with the opening bracket, a subset, or
+      # intersection/negation/range operators
+      if prev_token[0] == :set
+        emit(:literal, :literal, '-')
       else
-        emit(:set, :range, text)
+        emit(:set, :range, '-')
       end
      end
 		end
 	when 21 then
-# line 214 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 208 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  begin p = ((te))-1; end
  begin 
-      emit(:set, :open, copy(data, ts, te))
+      emit(:set, :open, '[')
       	begin
 		stack[top] = cs
 		top+= 1
@@ -1263,16 +1269,31 @@ p = p - 1; begin
      end
 		end
 	when 16 then
-# line 248 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 242 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  begin p = ((te))-1; end
  begin 
       text = copy(data, ts, te)
       emit(:literal, :literal, text)
+     end
+		end
+	when 81 then
+# line 252 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+		begin
+te = p+1
+ begin 
+      emit(:escape, :octal, copy(data, ts-1, te))
+      	begin
+		top -= 1
+		cs = stack[top]
+		_goto_level = _again
+		next
+	end
+
      end
 		end
 	when 79 then
-# line 257 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 258 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1281,7 +1302,7 @@ te = p+1
       	begin
 		stack[top] = cs
 		top+= 1
-		cs = 99
+		cs = 101
 		_goto_level = _again
 		next
 	end
@@ -1289,7 +1310,7 @@ te = p+1
      end
 		end
 	when 78 then
-# line 263 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 267 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1303,8 +1324,23 @@ te = p+1
 
      end
 		end
-	when 83 then
-# line 273 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+	when 80 then
+# line 252 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+		begin
+te = p
+p = p - 1; begin 
+      emit(:escape, :octal, copy(data, ts-1, te))
+      	begin
+		top -= 1
+		cs = stack[top]
+		_goto_level = _again
+		next
+	end
+
+     end
+		end
+	when 85 then
+# line 277 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1319,8 +1355,8 @@ te = p+1
 
      end
 		end
-	when 90 then
-# line 279 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+	when 92 then
+# line 283 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1334,8 +1370,8 @@ te = p+1
 
      end
 		end
-	when 80 then
-# line 284 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+	when 82 then
+# line 288 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1365,8 +1401,8 @@ te = p+1
 
      end
 		end
-	when 86 then
-# line 305 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+	when 88 then
+# line 309 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1392,7 +1428,7 @@ te = p+1
      end
 		end
 	when 30 then
-# line 321 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 325 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1411,8 +1447,8 @@ te = p+1
 
      end
 		end
-	when 96 then
-# line 331 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+	when 98 then
+# line 335 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1427,7 +1463,7 @@ te = p+1
      end
 		end
 	when 26 then
-# line 340 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 344 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1442,7 +1478,7 @@ te = p+1
      end
 		end
 	when 28 then
-# line 345 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 349 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1456,8 +1492,8 @@ te = p+1
 
      end
 		end
-	when 84 then
-# line 350 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+	when 86 then
+# line 354 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1473,8 +1509,8 @@ te = p+1
 
      end
 		end
-	when 85 then
-# line 356 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+	when 87 then
+# line 360 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1491,7 +1527,7 @@ te = p+1
      end
 		end
 	when 24 then
-# line 362 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 366 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1505,8 +1541,8 @@ te = p+1
 
      end
 		end
-	when 89 then
-# line 279 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+	when 91 then
+# line 283 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
@@ -1520,8 +1556,8 @@ p = p - 1; begin
 
      end
 		end
-	when 95 then
-# line 331 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+	when 97 then
+# line 335 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
@@ -1535,8 +1571,8 @@ p = p - 1; begin
 
      end
 		end
-	when 92 then
-# line 340 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+	when 94 then
+# line 344 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
@@ -1550,8 +1586,8 @@ p = p - 1; begin
 
      end
 		end
-	when 94 then
-# line 345 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+	when 96 then
+# line 349 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
@@ -1565,8 +1601,8 @@ p = p - 1; begin
 
      end
 		end
-	when 87 then
-# line 362 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+	when 89 then
+# line 366 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
@@ -1581,7 +1617,7 @@ p = p - 1; begin
      end
 		end
 	when 23 then
-# line 362 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 366 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  begin p = ((te))-1; end
  begin 
@@ -1595,11 +1631,11 @@ p = p - 1; begin
 
      end
 		end
-	when 88 then
+	when 90 then
 # line 1 "NONE"
 		begin
 	case act
-	when 16 then
+	when 17 then
 	begin begin p = ((te))-1; end
 
       text = copy(data, ts-1, te)
@@ -1612,7 +1648,7 @@ p = p - 1; begin
 	end
 
     end
-	when 17 then
+	when 18 then
 	begin begin p = ((te))-1; end
 
       emit(:escape, :octal, copy(data, ts-1, te))
@@ -1627,7 +1663,7 @@ p = p - 1; begin
 end 
 			end
 	when 33 then
-# line 372 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 376 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1636,8 +1672,8 @@ te = p+1
       emit(:conditional, :condition_close, ')')
      end
 		end
-	when 97 then
-# line 378 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+	when 99 then
+# line 382 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1652,8 +1688,8 @@ te = p+1
 
      end
 		end
-	when 98 then
-# line 378 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+	when 100 then
+# line 382 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
@@ -1669,7 +1705,7 @@ p = p - 1; begin
      end
 		end
 	when 32 then
-# line 378 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 382 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  begin p = ((te))-1; end
  begin 
@@ -1685,7 +1721,7 @@ p = p - 1; begin
      end
 		end
 	when 39 then
-# line 391 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 395 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1693,7 +1729,7 @@ te = p+1
      end
 		end
 	when 44 then
-# line 395 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 399 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1705,7 +1741,7 @@ te = p+1
      end
 		end
 	when 43 then
-# line 405 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 409 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1713,7 +1749,7 @@ te = p+1
      end
 		end
 	when 36 then
-# line 409 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 413 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1721,7 +1757,7 @@ te = p+1
      end
 		end
 	when 63 then
-# line 413 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 417 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1729,7 +1765,7 @@ te = p+1
      end
 		end
 	when 62 then
-# line 417 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 421 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1744,7 +1780,7 @@ te = p+1
      end
 		end
 	when 42 then
-# line 428 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 432 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1752,7 +1788,7 @@ te = p+1
      end
 		end
 	when 52 then
-# line 443 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 447 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1765,7 +1801,7 @@ te = p+1
       	begin
 		stack[top] = cs
 		top+= 1
-		cs = 113
+		cs = 115
 		_goto_level = _again
 		next
 	end
@@ -1773,7 +1809,7 @@ te = p+1
      end
 		end
 	when 53 then
-# line 474 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 478 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1785,7 +1821,7 @@ te = p+1
      end
 		end
 	when 7 then
-# line 488 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 492 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1798,7 +1834,7 @@ te = p+1
      end
 		end
 	when 6 then
-# line 505 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 509 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1820,7 +1856,7 @@ te = p+1
      end
 		end
 	when 11 then
-# line 546 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 550 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1841,7 +1877,7 @@ te = p+1
      end
 		end
 	when 10 then
-# line 565 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 569 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1858,7 +1894,7 @@ te = p+1
      end
 		end
 	when 60 then
-# line 581 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 585 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1870,7 +1906,7 @@ te = p+1
      end
 		end
 	when 56 then
-# line 589 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 593 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1882,7 +1918,7 @@ te = p+1
      end
 		end
 	when 58 then
-# line 597 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 601 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1894,7 +1930,7 @@ te = p+1
      end
 		end
 	when 13 then
-# line 605 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 609 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1902,7 +1938,7 @@ te = p+1
      end
 		end
 	when 48 then
-# line 620 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 624 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -1917,7 +1953,7 @@ te = p+1
      end
 		end
 	when 51 then
-# line 474 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 478 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
@@ -1929,7 +1965,7 @@ p = p - 1; begin
      end
 		end
 	when 54 then
-# line 488 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 492 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
@@ -1942,7 +1978,7 @@ p = p - 1; begin
      end
 		end
 	when 49 then
-# line 523 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 527 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
@@ -1951,7 +1987,7 @@ p = p - 1; begin
      end
 		end
 	when 59 then
-# line 581 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 585 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
@@ -1963,7 +1999,7 @@ p = p - 1; begin
      end
 		end
 	when 55 then
-# line 589 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 593 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
@@ -1975,7 +2011,7 @@ p = p - 1; begin
      end
 		end
 	when 57 then
-# line 597 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 601 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
@@ -1987,7 +2023,7 @@ p = p - 1; begin
      end
 		end
 	when 64 then
-# line 610 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 614 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
@@ -1995,14 +2031,14 @@ p = p - 1; begin
      end
 		end
 	when 61 then
-# line 616 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 620 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
       	begin
 		stack[top] = cs
 		top+= 1
-		cs = 99
+		cs = 101
 		_goto_level = _again
 		next
 	end
@@ -2010,7 +2046,7 @@ p = p - 1; begin
      end
 		end
 	when 47 then
-# line 620 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 624 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
@@ -2025,7 +2061,7 @@ p = p - 1; begin
      end
 		end
 	when 46 then
-# line 630 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 634 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
@@ -2037,7 +2073,7 @@ p = p - 1; begin
      end
 		end
 	when 45 then
-# line 641 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 645 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
@@ -2045,7 +2081,7 @@ p = p - 1; begin
      end
 		end
 	when 3 then
-# line 474 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 478 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  begin p = ((te))-1; end
  begin 
@@ -2057,7 +2093,7 @@ p = p - 1; begin
      end
 		end
 	when 12 then
-# line 610 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 614 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  begin p = ((te))-1; end
  begin 
@@ -2065,14 +2101,14 @@ p = p - 1; begin
      end
 		end
 	when 9 then
-# line 616 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 620 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  begin p = ((te))-1; end
  begin 
       	begin
 		stack[top] = cs
 		top+= 1
-		cs = 99
+		cs = 101
 		_goto_level = _again
 		next
 	end
@@ -2090,7 +2126,7 @@ p = p - 1; begin
 		next
 	end
 end
-	when 40 then
+	when 41 then
 	begin begin p = ((te))-1; end
 
       text = copy(data, ts, te)
@@ -2099,7 +2135,7 @@ end
       end
       emit_options(text)
     end
-	when 41 then
+	when 42 then
 	begin begin p = ((te))-1; end
 
       case text = copy(data, ts, te)
@@ -2109,7 +2145,7 @@ end
       when '(?<!'; emit(:assertion, :nlookbehind,  text)
       end
     end
-	when 55 then
+	when 56 then
 	begin begin p = ((te))-1; end
 
       append_literal(data, ts, te)
@@ -2117,17 +2153,17 @@ end
 end 
 			end
 	when 76 then
-# line 140 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 135 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 
     text = copy(data, ts ? ts-1 : 0, -1)
     raise PrematureEndError.new( text )
   		end
-# line 214 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 208 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
-      emit(:set, :open, copy(data, ts, te))
+      emit(:set, :open, '[')
       	begin
 		stack[top] = cs
 		top+= 1
@@ -2139,17 +2175,17 @@ p = p - 1; begin
      end
 		end
 	when 20 then
-# line 140 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 135 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 
     text = copy(data, ts ? ts-1 : 0, -1)
     raise PrematureEndError.new( text )
   		end
-# line 214 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 208 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  begin p = ((te))-1; end
  begin 
-      emit(:set, :open, copy(data, ts, te))
+      emit(:set, :open, '[')
       	begin
 		stack[top] = cs
 		top+= 1
@@ -2160,14 +2196,14 @@ p = p - 1; begin
 
      end
 		end
-	when 91 then
-# line 140 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+	when 93 then
+# line 135 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 
     text = copy(data, ts ? ts-1 : 0, -1)
     raise PrematureEndError.new( text )
   		end
-# line 340 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 344 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
@@ -2181,14 +2217,14 @@ p = p - 1; begin
 
      end
 		end
-	when 93 then
-# line 140 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+	when 95 then
+# line 135 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 
     text = copy(data, ts ? ts-1 : 0, -1)
     raise PrematureEndError.new( text )
   		end
-# line 345 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 349 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p
 p = p - 1; begin 
@@ -2203,13 +2239,13 @@ p = p - 1; begin
      end
 		end
 	when 27 then
-# line 140 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 135 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 
     text = copy(data, ts ? ts-1 : 0, -1)
     raise PrematureEndError.new( text )
   		end
-# line 340 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 344 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  begin p = ((te))-1; end
  begin 
@@ -2224,13 +2260,13 @@ p = p - 1; begin
      end
 		end
 	when 29 then
-# line 140 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 135 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 
     text = copy(data, ts ? ts-1 : 0, -1)
     raise PrematureEndError.new( text )
   		end
-# line 345 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 349 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  begin p = ((te))-1; end
  begin 
@@ -2245,13 +2281,13 @@ p = p - 1; begin
      end
 		end
 	when 31 then
-# line 146 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 141 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 
     text = copy(data, ts ? ts-1 : 0, -1)
     validation_error(:sequence, 'sequence', text)
   		end
-# line 336 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 340 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -2265,10 +2301,10 @@ te = p+1
      end
 		end
 	when 5 then
-# line 153 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 148 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  self.group_depth = group_depth - 1 		end
-# line 459 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 463 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -2276,16 +2312,16 @@ te = p+1
      end
 		end
 	when 38 then
-# line 153 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 148 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  self.group_depth = group_depth - 1 		end
-# line 528 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 532 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
       if conditional_stack.last == group_depth + 1
         conditional_stack.pop
-        emit(:conditional, :close, copy(data, ts, te))
+        emit(:conditional, :close, ')')
       else
         if spacing_stack.length > 1 &&
            spacing_stack.last[:depth] == group_depth + 1
@@ -2293,15 +2329,15 @@ te = p+1
           self.free_spacing = spacing_stack.last[:free_spacing]
         end
 
-        emit(:group, :close, copy(data, ts, te))
+        emit(:group, :close, ')')
       end
      end
 		end
 	when 40 then
-# line 154 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 149 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  self.set_depth   = set_depth   + 1 		end
-# line 434 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 438 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -2317,10 +2353,10 @@ te = p+1
      end
 		end
 	when 70 then
-# line 155 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 150 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  self.set_depth   = set_depth   - 1 		end
-# line 161 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 156 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -2344,15 +2380,15 @@ te = p+1
      end
 		end
 	when 75 then
-# line 155 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 150 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  self.set_depth   = set_depth   - 1 		end
-# line 170 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 165 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin  # special case, emits two tokens
-      emit(:literal, :literal, copy(data, ts, te-1))
-      emit(:set, :close, copy(data, ts+1, te))
+      emit(:literal, :literal, '-')
+      emit(:set, :close, ']')
       if in_set?
         	begin
 		top -= 1
@@ -2372,10 +2408,10 @@ te = p+1
      end
 		end
 	when 22 then
-# line 155 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 150 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  self.set_depth   = set_depth   - 1 		end
-# line 219 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 213 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 te = p+1
  begin 
@@ -2400,56 +2436,56 @@ te = p+1
 		begin
 te = p+1
 		end
-# line 154 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 149 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  self.set_depth   = set_depth   + 1 		end
-	when 82 then
+	when 84 then
 # line 1 "NONE"
 		begin
 te = p+1
 		end
-# line 273 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 277 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
-act = 16;		end
-	when 81 then
+act = 17;		end
+	when 83 then
 # line 1 "NONE"
 		begin
 te = p+1
 		end
-# line 279 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 283 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
-act = 17;		end
+act = 18;		end
 	when 8 then
 # line 1 "NONE"
 		begin
 te = p+1
 		end
-# line 488 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 492 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
-act = 41;		end
+act = 42;		end
 	when 2 then
 # line 1 "NONE"
 		begin
 te = p+1
 		end
-# line 641 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 645 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
-act = 55;		end
+act = 56;		end
 	when 50 then
 # line 1 "NONE"
 		begin
 te = p+1
 		end
-# line 153 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 148 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  self.group_depth = group_depth - 1 		end
-# line 152 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 147 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
  self.group_depth = group_depth + 1 		end
-# line 474 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 478 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
-act = 40;		end
-# line 2452 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner.rb"
+act = 41;		end
+# line 2488 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner.rb"
 	end
 	end
 	end
@@ -2467,7 +2503,7 @@ ts = nil;		end
 		begin
 act = 0
 		end
-# line 2470 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner.rb"
+# line 2506 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner.rb"
 	end
 
 	if cs == 0
@@ -2489,19 +2525,19 @@ act = 0
 	end
 	  case _re_scanner_eof_actions[cs]
 	when 14 then
-# line 8 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/property.rl"
+# line 8 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/property.rl"
 		begin
 
     raise PrematureEndError.new('unicode property')
   		end
 	when 25 then
-# line 140 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 135 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 		begin
 
     text = copy(data, ts ? ts-1 : 0, -1)
     raise PrematureEndError.new( text )
   		end
-# line 2504 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner.rb"
+# line 2540 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner.rb"
 	  end
 	end
 
@@ -2512,7 +2548,7 @@ act = 0
 end
 	end
 
-# line 746 "/Users/janoschmuller/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
+# line 751 "/Users/janosch/code/regexp_parser/tasks/../lib/regexp_parser/scanner/scanner.rl"
 
     # to avoid "warning: assigned but unused variable - testEof"
     testEof = testEof
@@ -2528,7 +2564,7 @@ end
           "[#{set_depth}]") if in_set?
 
     # when the entire expression is a literal run
-    emit_literal if literal
+    emit_literal if literal_run
 
     tokens
   end
@@ -2555,26 +2591,37 @@ end
   def emit(type, token, text)
     #puts "EMIT: type: #{type}, token: #{token}, text: #{text}, ts: #{ts}, te: #{te}"
 
-    emit_literal if literal
+    emit_literal if literal_run
 
     # Ragel runs with byte-based indices (ts, te). These are of little value to
     # end-users, so we keep track of char-based indices and emit those instead.
     ts_char_pos = char_pos
     te_char_pos = char_pos + text.length
 
-    if block
-      block.call type, token, text, ts_char_pos, te_char_pos
-    end
+    tok = [type, token, text, ts_char_pos, te_char_pos]
 
-    tokens << [type, token, text, ts_char_pos, te_char_pos]
+    self.prev_token = tok
 
     self.char_pos = te_char_pos
+
+    if block
+      block.call type, token, text, ts_char_pos, te_char_pos
+      # TODO: in v3.0.0, remove `collect_tokens:` kwarg and only collect if no block given
+      tokens << tok if collect_tokens
+    elsif collect_tokens
+      tokens << tok
+    end
   end
 
+  attr_accessor :literal_run # only public for #||= to work on ruby <= 2.5
+
   private
 
-  attr_accessor :tokens, :literal, :block, :free_spacing, :spacing_stack,
-                :group_depth, :set_depth, :conditional_stack, :char_pos
+  attr_accessor :block,
+                :collect_tokens, :tokens, :prev_token,
+                :free_spacing, :spacing_stack,
+                :group_depth, :set_depth, :conditional_stack,
+                :char_pos
 
   def free_spacing?(input_object, options)
     if options && !input_object.is_a?(String)
@@ -2604,14 +2651,13 @@ end
   # Appends one or more characters to the literal buffer, to be emitted later
   # by a call to emit_literal.
   def append_literal(data, ts, te)
-    self.literal = literal || []
-    literal << copy(data, ts, te)
+    (self.literal_run ||= []) << copy(data, ts, te)
   end
 
   # Emits the literal run collected by calls to the append_literal method.
   def emit_literal
-    text = literal.join
-    self.literal = nil
+    text = literal_run.join
+    self.literal_run = nil
     emit(:literal, :literal, text)
   end