demotape 0.0.2 → 0.0.3

data/lib/demo_tape/parser.rb

@@ -10,199 +10,179 @@ require 'racc/parser.rb'
 module DemoTape
   class Parser < Racc::Parser
 
-module_eval(<<'...end demotape.y/module_eval...', 'demotape.y', 442)
+module_eval(<<'...end demotape.y/module_eval...', 'demotape.y', 228)
+  include Helpers
+
   def parse(str, file: "<unknown>")
     @file = file
     @lexer = DemoTape::Lexer.new
     @tokens = @lexer.tokenize(str)
     @token_index = 0
-    do_parse
-  end
-
-  def next_token
-    token = @tokens.shift
-    @current_token_index = @token_index
-    @token_index += 1
-    token
-  end
-
-  def make_token(type, value, index)
-    line_info = @lexer.line_map[index] || {}
-    token_class = case type
-                  when :identifier then Token::Identifier
-                  when :string then Token::String
-                  when :number then Token::Number
-                  when :duration then Token::Duration
-                  when :regex then Token::Regex
-                  when :time_unit then Token::TimeUnit
-                  when :operator then Token::Operator
-                  else Token::Base
-                  end
-
-    token_class.new(
-      value,
-      line: line_info[:line],
-      column: line_info[:column],
-      raw: line_info[:raw]
-    )
-  end
-
-  def attach_location(command, duration_index: nil, speed_index: nil, timeout_index: nil)
-    line_info = @lexer.line_map[@command_start_index] || {}
-    command.line = line_info[:line]
-    command.column = line_info[:column]
-    command.line_content = line_info[:content]
-    command.file = @file
-
-    # Attach duration/speed/timeout specific columns if provided
-    if duration_index
-      duration_info = @lexer.line_map[duration_index] || {}
-      command.duration_column = duration_info[:column]
-    end
-
-    if speed_index
-      speed_info = @lexer.line_map[speed_index] || {}
-      command.speed_column = speed_info[:column]
-    end
-
-    if timeout_index
-      timeout_info = @lexer.line_map[timeout_index] || {}
-      command.timeout_column = timeout_info[:column]
-    end
-
-    command
-  end
-
-  def on_error(token_id, token_value, value_stack)
-    # Get line metadata from the lexer's line map
-    line_info = @lexer.line_map[@current_token_index] || {}
-    line_num = line_info[:line] || "?"
-    col_num = line_info[:column] || 1
-    line_content = line_info[:content] || ""
-
-    token_name = token_to_str(token_id) || token_id.to_s
-
-    error_msg = "Unexpected token #{token_name.inspect} at #{@file}:#{line_num}:#{col_num}:\n"
-    error_msg += "  #{line_content}\n"
-    error_msg += "  #{' ' * (col_num - 1)}^"
-
-    raise DemoTape::ParseError, error_msg
+    @token_indices = {}  # Map token object_id to its index
+    validate_tree!(do_parse)
   end
 
 ...end demotape.y/module_eval...
 ##### State transition tables begin ###
 
 racc_action_table = [
-    37,    21,    18,    19,    40,    20,    38,    24,    25,    34,
-    24,    25,    43,     4,    44,    45,    24,    25,     6,    24,
-    25,    10,     4,    12,    47,    11,    49,    34,    13,    48,
-    14,    28,    31,    29,    32,     9,    15,     9,    17,    27,
-    30,    33,    34,    35,    36,    50,    51,    52,    53,    54,
-    55,    56,    57,    58,    59,    60 ]
+     5,     4,     6,    22,    17,    10,    11,    12,    13,    14,
+    15,    16,    18,    19,    20,    21,     5,     4,     6,    24,
+    17,    10,    11,    12,    13,    14,    15,    16,    18,    19,
+    20,    21,    44,    41,    42,    30,    17,    10,    11,    12,
+    13,    14,    15,    16,    18,    19,    20,    21,    44,    41,
+    42,    31,    17,    10,    11,    12,    13,    14,    15,    16,
+    18,    19,    20,    21,    44,    41,    42,    45,    17,    10,
+    11,    12,    13,    14,    15,    16,    18,    19,    20,    21,
+    44,    41,    42,    47,    17,    10,    11,    12,    13,    14,
+    15,    16,    18,    19,    20,    21,    44,    41,    42,    51,
+    17,    10,    11,    12,    13,    14,    15,    16,    18,    19,
+    20,    21,    25,    48,    49,    17,    10,    11,    12,    13,
+    14,    15,    16,    18,    19,    20,    21,    54,    56,    57,
+    17,    10,    11,    12,    13,    14,    15,    16,    18,    19,
+    20,    21,    27,    28,    17,    10,    11,    12,    13,    14,
+    15,    16,    18,    19,    20,    21,    32,    33,    17,    10,
+    11,    12,    13,    14,    15,    16,    18,    19,    20,    21,
+    52,    59,    17,    10,    11,    12,    13,    14,    15,    16,
+    18,    19,    20,    21,    61,    60,    17,    10,    11,    12,
+    13,    14,    15,    16,    18,    19,    20,    21,    34,    36,
+    35,    37,    63,    64,    68,    67,    62,    65,    66,    69,
+    70,    71,    72,    73,    74,    75,    76 ]
 
 racc_action_check = [
-    27,    10,    10,    10,    35,    10,    35,    10,    10,    27,
-    35,    35,    36,     0,    36,    37,    36,    36,     1,    37,
-    37,     4,     2,     4,    38,     4,    47,    38,     5,    47,
-     5,    12,    14,    12,    14,     3,     6,     7,     8,    11,
-    13,    16,    19,    21,    26,    48,    49,    50,    51,    52,
-    53,    54,    55,    57,    58,    59 ]
+     0,     0,     0,     1,     0,     0,     0,     0,     0,     0,
+     0,     0,     0,     0,     0,     0,     2,     2,     2,     4,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,    34,    34,    34,    22,    34,    34,    34,    34,
+    34,    34,    34,    34,    34,    34,    34,    34,    36,    36,
+    36,    25,    36,    36,    36,    36,    36,    36,    36,    36,
+    36,    36,    36,    36,    39,    39,    39,    35,    39,    39,
+    39,    39,    39,    39,    39,    39,    39,    39,    39,    39,
+    45,    45,    45,    37,    45,    45,    45,    45,    45,    45,
+    45,    45,    45,    45,    45,    45,    47,    47,    47,    41,
+    47,    47,    47,    47,    47,    47,    47,    47,    47,    47,
+    47,    47,     5,    38,    38,     5,     5,     5,     5,     5,
+     5,     5,     5,     5,     5,     5,     5,    44,    46,    46,
+    44,    44,    44,    44,    44,    44,    44,    44,    44,    44,
+    44,    44,     8,     8,     8,     8,     8,     8,     8,     8,
+     8,     8,     8,     8,     8,     8,    26,    26,    26,    26,
+    26,    26,    26,    26,    26,    26,    26,    26,    26,    26,
+    43,    48,    43,    43,    43,    43,    43,    43,    43,    43,
+    43,    43,    43,    43,    53,    49,    53,    53,    53,    53,
+    53,    53,    53,    53,    53,    53,    53,    53,    28,    33,
+    28,    33,    55,    55,    58,    58,    54,    56,    57,    60,
+    63,    64,    66,    67,    68,    71,    73 ]
 
 racc_action_pointer = [
-     9,    18,    18,    32,    18,    25,    36,    34,    36,   nil,
-    -3,    33,    27,    34,    28,   nil,    39,   nil,   nil,    30,
-   nil,    40,   nil,   nil,   nil,   nil,    41,    -3,   nil,   nil,
-   nil,   nil,   nil,   nil,   nil,     0,     6,     9,    15,   nil,
-   nil,   nil,   nil,   nil,   nil,   nil,   nil,    23,    36,    40,
-    41,    39,    40,    47,    45,    46,   nil,    44,    51,    49,
-   nil ]
+    -3,     3,    13,   nil,    14,   108,   nil,   nil,   137,   nil,
+   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
+   nil,   nil,    35,   nil,   nil,    46,   151,   nil,   193,   nil,
+   nil,   nil,   nil,   194,    29,    62,    45,    78,   111,    61,
+   nil,    94,   nil,   165,   123,    77,   126,    93,   166,   183,
+   nil,   nil,   nil,   179,   201,   200,   202,   206,   202,   nil,
+   204,   nil,   nil,   205,   209,   nil,   207,   211,   209,   nil,
+   nil,   210,   nil,   211,   nil,   nil,   nil ]
 
 racc_action_default = [
-    -2,   -39,    -1,    -6,   -31,    -9,   -39,    -6,   -39,    -5,
-   -39,   -39,   -39,   -39,   -39,    61,   -39,    -3,    -7,   -10,
-   -18,   -39,   -29,   -30,   -36,   -37,   -16,   -15,   -32,   -33,
-    -8,   -34,   -35,    -4,   -38,   -39,   -39,   -39,   -26,   -25,
-   -27,   -28,   -11,   -12,   -17,   -13,   -14,   -39,   -23,   -39,
-   -39,   -24,   -21,   -39,   -39,   -39,   -19,   -22,   -39,   -39,
-   -20 ]
+    -2,   -42,    -1,    -3,   -42,   -42,    -7,    -8,   -42,   -28,
+   -30,   -31,   -32,   -33,   -34,   -35,   -36,   -37,   -38,   -39,
+   -40,   -41,   -42,    -4,    -5,   -42,   -42,    -9,   -42,   -29,
+    77,    -6,   -10,   -42,   -19,   -42,   -19,   -42,   -42,   -20,
+   -21,   -42,   -24,   -42,   -42,   -19,   -42,   -19,   -42,   -42,
+   -22,   -23,   -25,   -42,   -42,   -42,   -42,   -42,   -42,   -11,
+   -42,   -26,   -27,   -42,   -42,   -13,   -42,   -42,   -42,   -12,
+   -15,   -42,   -14,   -42,   -18,   -16,   -17 ]
 
 racc_goto_table = [
-    22,     8,     1,    23,    26,    16,     3,     2,     7,   nil,
+    29,    38,     8,    46,     8,     1,     3,    26,    23,     2,
+    50,   nil,    55,   nil,    58,   nil,   nil,   nil,    29,   nil,
    nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
-   nil,   nil,   nil,   nil,   nil,    41,    42,    46,    39 ]
+   nil,   nil,   nil,   nil,   nil,    29,   nil,   nil,   nil,   nil,
+   nil,   nil,   nil,   nil,   nil,    29,    53 ]
 
 racc_goto_check = [
-     7,     4,     1,     6,     6,     4,     3,     2,     3,   nil,
+     9,     6,     5,     6,     5,     1,     3,     5,     3,     2,
+     8,   nil,     6,   nil,     6,   nil,   nil,   nil,     9,   nil,
    nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
-   nil,   nil,   nil,   nil,   nil,     7,     7,     7,     6 ]
+   nil,   nil,   nil,   nil,   nil,     9,   nil,   nil,   nil,   nil,
+   nil,   nil,   nil,   nil,   nil,     9,     5 ]
 
 racc_goto_pointer = [
-   nil,     2,     7,     6,    -2,   nil,    -7,   -10 ]
+   nil,     5,     9,     6,   nil,     2,   -33,   nil,   -29,    -8 ]
 
 racc_goto_default = [
-   nil,   nil,   nil,   nil,   nil,     5,   nil,   nil ]
+   nil,   nil,   nil,   nil,     7,    43,   nil,    39,    40,     9 ]
 
 racc_reduce_table = [
   0, 0, :racc_error,
-  1, 14, :_reduce_1,
-  0, 14, :_reduce_2,
-  3, 15, :_reduce_3,
-  4, 15, :_reduce_4,
-  1, 17, :_reduce_5,
-  0, 17, :_reduce_6,
-  3, 16, :_reduce_7,
-  3, 16, :_reduce_8,
-  1, 16, :_reduce_9,
-  3, 16, :_reduce_10,
-  5, 16, :_reduce_11,
-  5, 16, :_reduce_12,
-  5, 16, :_reduce_13,
-  5, 16, :_reduce_14,
-  3, 16, :_reduce_15,
-  3, 16, :_reduce_16,
-  5, 16, :_reduce_17,
-  3, 16, :_reduce_18,
-  11, 16, :_reduce_19,
-  14, 16, :_reduce_20,
-  9, 16, :_reduce_21,
-  11, 16, :_reduce_22,
-  7, 16, :_reduce_23,
-  8, 16, :_reduce_24,
-  5, 16, :_reduce_25,
-  5, 16, :_reduce_26,
-  5, 16, :_reduce_27,
-  5, 16, :_reduce_28,
-  3, 16, :_reduce_29,
-  3, 16, :_reduce_30,
-  1, 16, :_reduce_31,
-  3, 18, :_reduce_32,
-  3, 18, :_reduce_33,
-  3, 18, :_reduce_34,
-  3, 18, :_reduce_35,
-  1, 20, :_reduce_36,
-  1, 20, :_reduce_37,
-  2, 19, :_reduce_38 ]
-
-racc_reduce_n = 39
-
-racc_shift_n = 61
+  1, 20, :_reduce_1,
+  0, 20, :_reduce_2,
+  1, 21, :_reduce_3,
+  2, 21, :_reduce_4,
+  2, 22, :_reduce_5,
+  3, 22, :_reduce_6,
+  1, 22, :_reduce_7,
+  1, 22, :_reduce_8,
+  2, 22, :_reduce_9,
+  3, 22, :_reduce_10,
+  6, 23, :_reduce_11,
+  7, 23, :_reduce_12,
+  7, 23, :_reduce_13,
+  8, 23, :_reduce_14,
+  7, 23, :_reduce_15,
+  8, 23, :_reduce_16,
+  9, 23, :_reduce_17,
+  8, 23, :_reduce_18,
+  0, 25, :_reduce_19,
+  1, 25, :_reduce_20,
+  1, 26, :_reduce_21,
+  2, 26, :_reduce_22,
+  2, 27, :_reduce_23,
+  1, 27, :_reduce_24,
+  2, 27, :_reduce_25,
+  3, 27, :_reduce_26,
+  3, 27, :_reduce_27,
+  1, 24, :_reduce_28,
+  2, 24, :_reduce_29,
+  1, 28, :_reduce_30,
+  1, 28, :_reduce_31,
+  1, 28, :_reduce_32,
+  1, 28, :_reduce_33,
+  1, 28, :_reduce_34,
+  1, 28, :_reduce_35,
+  1, 28, :_reduce_36,
+  1, 28, :_reduce_37,
+  1, 28, :_reduce_38,
+  1, 28, :_reduce_39,
+  1, 28, :_reduce_40,
+  1, 28, :_reduce_41 ]
+
+racc_reduce_n = 42
+
+racc_shift_n = 77
 
 racc_token_table = {
   false => 0,
   :error => 1,
-  :NEWLINE => 2,
-  :SPACE => 3,
-  :IDENTIFIER => 4,
-  :PLUS => 5,
-  :NUMBER => 6,
-  :AT => 7,
-  :REGEX => 8,
-  :COMMA => 9,
-  :STRING => 10,
-  :WORD => 11,
-  :TIME_UNIT => 12 }
-
-racc_nt_base = 13
+  :END => 2,
+  :LEADING_SPACE => 3,
+  :COMMENT => 4,
+  :NEWLINE => 5,
+  :DO => 6,
+  :TRAILING_SPACE => 7,
+  :IDENTIFIER => 8,
+  :STRING => 9,
+  :MULTILINE_STRING => 10,
+  :NUMBER => 11,
+  :DURATION => 12,
+  :REGEX => 13,
+  :SPACE => 14,
+  :COMMA => 15,
+  :PLUS => 16,
+  :AT => 17,
+  :TIME_UNIT => 18 }
+
+racc_nt_base = 19
 
 racc_use_result_var = true
 
@@ -226,25 +206,33 @@ Ractor.make_shareable(Racc_arg) if defined?(Ractor)
 Racc_token_to_s_table = [
   "$end",
   "error",
+  "END",
+  "LEADING_SPACE",
+  "COMMENT",
   "NEWLINE",
-  "SPACE",
+  "DO",
+  "TRAILING_SPACE",
   "IDENTIFIER",
-  "PLUS",
+  "STRING",
+  "MULTILINE_STRING",
   "NUMBER",
-  "AT",
+  "DURATION",
   "REGEX",
+  "SPACE",
   "COMMA",
-  "STRING",
-  "WORD",
+  "PLUS",
+  "AT",
   "TIME_UNIT",
   "$start",
-  "tape",
-  "commands",
-  "command",
-  "trailing_space",
-  "key_combo",
-  "duration",
-  "string" ]
+  "document",
+  "lines",
+  "line",
+  "group",
+  "line_tokens",
+  "group_body",
+  "group_lines",
+  "group_line",
+  "line_token" ]
 Ractor.make_shareable(Racc_token_to_s_table) if defined?(Ractor)
 
 Racc_debug_parser = false
@@ -253,628 +241,429 @@ Racc_debug_parser = false
 
 # reduce 0 omitted
 
-module_eval(<<'.,.,', 'demotape.y', 2)
+module_eval(<<'.,.,', 'demotape.y', 11)
   def _reduce_1(val, _values, result)
-     result = val[0]
+     result = val[0].flatten.compact
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 3)
+module_eval(<<'.,.,', 'demotape.y', 12)
   def _reduce_2(val, _values, result)
      result = []
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 5)
+module_eval(<<'.,.,', 'demotape.y', 14)
   def _reduce_3(val, _values, result)
-     result = [val[0]]
+     result = val[0]
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 6)
+module_eval(<<'.,.,', 'demotape.y', 15)
   def _reduce_4(val, _values, result)
-     result = val[0] << val[1]
+     result = val[0] + val[1]
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 8)
+module_eval(<<'.,.,', 'demotape.y', 19)
   def _reduce_5(val, _values, result)
-     result = nil
+              result = [
+            make_token(:comment, val[0], index_for_value(val[0])),
+            make_token(:newline, val[1], index_for_value(val[1]))
+          ]
+
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 9)
+module_eval(<<'.,.,', 'demotape.y', 25)
   def _reduce_6(val, _values, result)
-     result = nil
+              result = [
+            make_token(:leading_space, val[0], index_for_value(val[0])),
+            make_token(:comment, val[1], index_for_value(val[1])),
+            make_token(:newline, val[2], index_for_value(val[2]))
+          ]
+
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 12)
+module_eval(<<'.,.,', 'demotape.y', 31)
   def _reduce_7(val, _values, result)
-                 space_index = @token_index - 2
-             line_info = @lexer.line_map[space_index] || {}
-             error_msg = "Unexpected token \"SPACE\" at #{@file}:#{line_info[:line]}:#{line_info[:column]}:\n"
-             error_msg += "  #{line_info[:content]}\n"
-             error_msg += "  #{' ' * (line_info[:column] - 1)}^"
-             raise DemoTape::ParseError, error_msg
-
+     result = [make_token(:newline, val[0], index_for_value(val[0]))]
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 20)
+module_eval(<<'.,.,', 'demotape.y', 32)
   def _reduce_8(val, _values, result)
-                 @command_start_index = @last_key_index
-             keys = val[0][:keys]
-             key_tokens = val[0][:tokens]
-             command_name = keys.shift
-
-             tokens = key_tokens + [make_token(:number, val[2], @token_index - 1)]
-
-             cmd = DemoTape::Command.new(command_name, "", keys:, count: val[2])
-             cmd.tokens = tokens
-             result = attach_location(cmd).prepare!
-
+     result = [val[0]]
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 32)
+module_eval(<<'.,.,', 'demotape.y', 34)
   def _reduce_9(val, _values, result)
-                 @command_start_index = @last_key_index
-             keys = val[0][:keys]
-             key_tokens = val[0][:tokens]
-             command_name = keys.shift
-
-             cmd = DemoTape::Command.new(command_name, "", keys:)
-             cmd.tokens = key_tokens
-             result = attach_location(cmd).prepare!
+              tokens, start_idx = val[0]
+          result = [
+            { type: :command, tokens: tokens, line: line_for_token_at(start_idx), column: column_for_token_at(start_idx) },
+            make_token(:newline, val[1], index_for_value(val[1]))
+          ]
 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 42)
+module_eval(<<'.,.,', 'demotape.y', 41)
   def _reduce_10(val, _values, result)
-                 @command_start_index = @token_index - 4
-             duration_index = @token_index - 2
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 4),
-               make_token(:number, val[2], @token_index - 2)
-             ]
-
-             # Duration commands (Sleep, Wait) treat bare numbers as seconds
-             # Key commands treat numbers as repeat counts
-             if ["Sleep", "Wait"].include?(val[0])
-               cmd = DemoTape::Command.new(val[0], "#{val[2]}s")
-               cmd.tokens = tokens
-               result = attach_location(cmd, duration_index: duration_index).prepare!
-             else
-               cmd = DemoTape::Command.new(val[0], "", count: val[2])
-               cmd.tokens = tokens
-               result = attach_location(cmd).prepare!
-             end
+              idx_lead = index_for_value(val[0])
+          tokens, _ = val[1]
+          result = [
+            { type: :command, tokens: [make_token(:leading_space, val[0], idx_lead)] + tokens, line: line_for_token_at(idx_lead), column: column_for_token_at(idx_lead) },
+            make_token(:newline, val[2], index_for_value(val[2]))
+          ]
 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 63)
+module_eval(<<'.,.,', 'demotape.y', 51)
   def _reduce_11(val, _values, result)
-                 @command_start_index = @token_index - 6
-             speed_index = @token_index - 3  # TIME_UNIT position
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 6),
-               make_token(:operator, "@", @token_index - 5),
-               make_token(:duration, val[2], @token_index - 4),
-               make_token(:string, val[4], @token_index - 1)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], val[4], speed: val[2])
-             cmd.tokens = tokens
-             result = attach_location(cmd, speed_index: speed_index).prepare!
+               tokens, start_idx = val[0]
+           idx_do = index_for_value(val[1])
+           idx_nl = index_for_value(val[2])
+           idx_end = index_for_value(val[4])
+           result = {
+             type: :group,
+             tokens: tokens + [make_token(:keyword, val[1], idx_do), make_token(:keyword, val[4], idx_end)],
+             children: [make_token(:newline, val[2], idx_nl)] + val[3],
+             line: line_for_token_at(start_idx),
+             column: column_for_token_at(start_idx)
+           }
 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 78)
+module_eval(<<'.,.,', 'demotape.y', 64)
   def _reduce_12(val, _values, result)
-                 @command_start_index = @token_index - 6
-             speed_index = @token_index - 3  # TIME_UNIT position
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 6),
-               make_token(:operator, "@", @token_index - 5),
-               make_token(:duration, val[2], @token_index - 4),
-               make_token(:number, val[4], @token_index - 1)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], "", speed: val[2], count: val[4])
-             cmd.tokens = tokens
-             result = attach_location(cmd, speed_index: speed_index).prepare!
+               tokens, start_idx = val[0]
+           idx_do = index_for_value(val[1])
+           idx_nl = index_for_value(val[2])
+           idx_end = index_for_value(val[5])
+           # Discard the LEADING_SPACE before END (val[4])
+           result = {
+             type: :group,
+             tokens: tokens + [make_token(:keyword, val[1], idx_do), make_token(:keyword, val[5], idx_end)],
+             children: [make_token(:newline, val[2], idx_nl)] + val[3],
+             line: line_for_token_at(start_idx),
+             column: column_for_token_at(start_idx)
+           }
 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 93)
+module_eval(<<'.,.,', 'demotape.y', 78)
   def _reduce_13(val, _values, result)
-                 @command_start_index = @token_index - 5
-             speed_index = @token_index - 3
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 5),
-               make_token(:operator, "@", @token_index - 4),
-               make_token(:number, val[2], @token_index - 3),
-               make_token(:number, val[4], @token_index - 1)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], "", speed: "#{val[2]}s", count: val[4])
-             cmd.tokens = tokens
-             result = attach_location(cmd, speed_index: speed_index).prepare!
+               idx_lead = index_for_value(val[0])
+           tokens, _ = val[1]
+           idx_do = index_for_value(val[2])
+           idx_nl = index_for_value(val[3])
+           idx_end = index_for_value(val[5])
+           result = {
+             type: :group,
+             tokens: [make_token(:leading_space, val[0], idx_lead)] + tokens + [make_token(:keyword, val[2], idx_do), make_token(:keyword, val[5], idx_end)],
+             children: [make_token(:newline, val[3], idx_nl)] + val[4],
+             line: line_for_token_at(idx_lead),
+             column: column_for_token_at(idx_lead)
+           }
 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 108)
+module_eval(<<'.,.,', 'demotape.y', 92)
   def _reduce_14(val, _values, result)
-                 @command_start_index = @token_index - 5
-             speed_index = @token_index - 3
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 5),
-               make_token(:operator, "@", @token_index - 4),
-               make_token(:number, val[2], @token_index - 3),
-               make_token(:string, val[4], @token_index - 1)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], val[4], speed: "#{val[2]}s")
-             cmd.tokens = tokens
-             result = attach_location(cmd, speed_index: speed_index).prepare!
+               idx_lead = index_for_value(val[0])
+           tokens, _ = val[1]
+           idx_do = index_for_value(val[2])
+           idx_nl = index_for_value(val[3])
+           idx_end = index_for_value(val[6])
+           # Discard the LEADING_SPACE before END (val[5])
+           result = {
+             type: :group,
+             tokens: [make_token(:leading_space, val[0], idx_lead)] + tokens + [make_token(:keyword, val[2], idx_do), make_token(:keyword, val[6], idx_end)],
+             children: [make_token(:newline, val[3], idx_nl)] + val[4],
+             line: line_for_token_at(idx_lead),
+             column: column_for_token_at(idx_lead)
+           }
 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 123)
+module_eval(<<'.,.,', 'demotape.y', 107)
   def _reduce_15(val, _values, result)
-                 @command_start_index = @token_index - 3
-             speed_index = @token_index - 1
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 3),
-               make_token(:operator, "@", @token_index - 2),
-               make_token(:number, val[2], @token_index - 1)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], "", speed: "#{val[2]}s")
-             cmd.tokens = tokens
-             result = attach_location(cmd, speed_index: speed_index).prepare!
+               tokens, start_idx = val[0]
+           idx_do = index_for_value(val[1])
+           idx_trail = index_for_value(val[2])
+           idx_nl = index_for_value(val[3])
+           idx_end = index_for_value(val[5])
+           result = {
+             type: :group,
+             tokens: tokens + [make_token(:keyword, val[1], idx_do), make_token(:trailing_space, val[2], idx_trail), make_token(:keyword, val[5], idx_end)],
+             children: [make_token(:newline, val[3], idx_nl)] + val[4],
+             line: line_for_token_at(start_idx),
+             column: column_for_token_at(start_idx)
+           }
 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 137)
+module_eval(<<'.,.,', 'demotape.y', 121)
   def _reduce_16(val, _values, result)
-                 @command_start_index = @token_index - 4
-             speed_index = @token_index - 1  # TIME_UNIT position
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 4),
-               make_token(:operator, "@", @token_index - 3),
-               make_token(:duration, val[2], @token_index - 2)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], "", speed: val[2])
-             cmd.tokens = tokens
-             result = attach_location(cmd, speed_index: speed_index).prepare!
+               tokens, start_idx = val[0]
+           idx_do = index_for_value(val[1])
+           idx_trail = index_for_value(val[2])
+           idx_nl = index_for_value(val[3])
+           idx_end = index_for_value(val[6])
+           # Discard the LEADING_SPACE before END (val[5])
+           result = {
+             type: :group,
+             tokens: tokens + [make_token(:keyword, val[1], idx_do), make_token(:trailing_space, val[2], idx_trail), make_token(:keyword, val[6], idx_end)],
+             children: [make_token(:newline, val[3], idx_nl)] + val[4],
+             line: line_for_token_at(start_idx),
+             column: column_for_token_at(start_idx)
+           }
 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 151)
+module_eval(<<'.,.,', 'demotape.y', 136)
   def _reduce_17(val, _values, result)
-                 @command_start_index = @token_index - 6
-             timeout_index = @token_index - 3  # TIME_UNIT position
-             regex_index = @token_index - 1
-
-             begin
-               Regexp.new(val[4])
-             rescue RegexpError => e
-               line_info = @lexer.line_map[regex_index] || {}
-               error_msg = "Invalid regex: #{e.message} at #{@file}:#{line_info[:line]}:#{line_info[:column]}:\n"
-               error_msg += "  #{line_info[:content]}\n"
-               error_msg += "  #{' ' * (line_info[:column] - 1)}^"
-               raise DemoTape::ParseError, error_msg
-             end
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 6),
-               make_token(:operator, "@", @token_index - 5),
-               make_token(:duration, val[2], @token_index - 4),
-               make_token(:regex, val[4], regex_index)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], val[4], timeout: val[2])
-             cmd.tokens = tokens
-             result = attach_location(cmd, timeout_index: timeout_index).prepare!
+               idx_lead = index_for_value(val[0])
+           tokens, _ = val[1]
+           idx_do = index_for_value(val[2])
+           idx_trail = index_for_value(val[3])
+           idx_nl = index_for_value(val[4])
+           idx_end = index_for_value(val[7])
+           # Discard the LEADING_SPACE before END (val[6])
+           result = {
+             type: :group,
+             tokens: [make_token(:leading_space, val[0], idx_lead)] + tokens + [make_token(:keyword, val[2], idx_do), make_token(:trailing_space, val[3], idx_trail), make_token(:keyword, val[7], idx_end)],
+             children: [make_token(:newline, val[4], idx_nl)] + val[5],
+             line: line_for_token_at(idx_lead),
+             column: column_for_token_at(idx_lead)
+           }
 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 177)
+module_eval(<<'.,.,', 'demotape.y', 152)
   def _reduce_18(val, _values, result)
-                 @command_start_index = @token_index - 3
-             regex_index = @token_index - 1
-
-             begin
-               Regexp.new(val[2])
-             rescue RegexpError => e
-               line_info = @lexer.line_map[regex_index] || {}
-               error_msg = "Invalid regex: #{e.message} at #{@file}:#{line_info[:line]}:#{line_info[:column]}:\n"
-               error_msg += "  #{line_info[:content]}\n"
-               error_msg += "  #{' ' * (line_info[:column] - 1)}^"
-               raise DemoTape::ParseError, error_msg
-             end
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 3),
-               make_token(:regex, val[2], regex_index)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], val[2])
-             cmd.tokens = tokens
-             result = attach_location(cmd).prepare!
+               idx_lead = index_for_value(val[0])
+           tokens, _ = val[1]
+           idx_do = index_for_value(val[2])
+           idx_trail = index_for_value(val[3])
+           idx_nl = index_for_value(val[4])
+           idx_end = index_for_value(val[6])
+           result = {
+             type: :group,
+             tokens: [make_token(:leading_space, val[0], idx_lead)] + tokens + [make_token(:keyword, val[2], idx_do), make_token(:trailing_space, val[3], idx_trail), make_token(:keyword, val[6], idx_end)],
+             children: [make_token(:newline, val[4], idx_nl)] + val[5],
+             line: line_for_token_at(idx_lead),
+             column: column_for_token_at(idx_lead)
+           }
 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 200)
+module_eval(<<'.,.,', 'demotape.y', 169)
   def _reduce_19(val, _values, result)
-                 @command_start_index = @token_index - 9
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 9),
-               make_token(:identifier, val[2], @token_index - 7),
-               make_token(:number, val[4], @token_index - 5),
-               make_token(:operator, ",", @token_index - 4),
-               make_token(:number, val[6], @token_index - 3),
-               make_token(:operator, ",", @token_index - 2),
-               make_token(:number, val[8], @token_index - 1),
-               make_token(:operator, ",", @token_index - 1),
-               make_token(:number, val[10], @token_index - 1)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], [val[4], val[6], val[8], val[10]], option: val[2])
-             cmd.tokens = tokens
-             result = attach_location(cmd).prepare!
-
+     result = []
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 219)
+module_eval(<<'.,.,', 'demotape.y', 170)
   def _reduce_20(val, _values, result)
-                 @command_start_index = @token_index - 13
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 13),
-               make_token(:identifier, val[2], @token_index - 11),
-               make_token(:number, val[4], @token_index - 9),
-               make_token(:operator, ",", @token_index - 8),
-               make_token(:number, val[7], @token_index - 6),
-               make_token(:operator, ",", @token_index - 5),
-               make_token(:number, val[10], @token_index - 3),
-               make_token(:operator, ",", @token_index - 2),
-               make_token(:number, val[13], @token_index - 1)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], [val[4], val[7], val[10], val[13]], option: val[2])
-             cmd.tokens = tokens
-             result = attach_location(cmd).prepare!
-
+     result = val[0].flatten.compact
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 238)
+module_eval(<<'.,.,', 'demotape.y', 172)
   def _reduce_21(val, _values, result)
-                 @command_start_index = @token_index - 7
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 7),
-               make_token(:identifier, val[2], @token_index - 5),
-               make_token(:number, val[4], @token_index - 3),
-               make_token(:operator, ",", @token_index - 2),
-               make_token(:number, val[6], @token_index - 1),
-               make_token(:operator, ",", @token_index - 1),
-               make_token(:number, val[8], @token_index - 1)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], [val[4], val[6], val[8]], option: val[2])
-             cmd.tokens = tokens
-             result = attach_location(cmd).prepare!
-
+     result = val[0]
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 255)
+module_eval(<<'.,.,', 'demotape.y', 173)
   def _reduce_22(val, _values, result)
-                 @command_start_index = @token_index - 11
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 11),
-               make_token(:identifier, val[2], @token_index - 9),
-               make_token(:number, val[4], @token_index - 7),
-               make_token(:operator, ",", @token_index - 6),
-               make_token(:number, val[7], @token_index - 4),
-               make_token(:operator, ",", @token_index - 3),
-               make_token(:number, val[10], @token_index - 1)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], [val[4], val[7], val[10]], option: val[2])
-             cmd.tokens = tokens
-             result = attach_location(cmd).prepare!
-
+     result = val[0] + val[1]
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 272)
+module_eval(<<'.,.,', 'demotape.y', 177)
   def _reduce_23(val, _values, result)
-                 @command_start_index = @token_index - 5
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 5),
-               make_token(:identifier, val[2], @token_index - 3),
-               make_token(:number, val[4], @token_index - 1),
-               make_token(:operator, ",", @token_index - 1),
-               make_token(:number, val[6], @token_index - 1)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], [val[4], val[6]], option: val[2])
-             cmd.tokens = tokens
-             result = attach_location(cmd).prepare!
+                    result = [
+                  make_token(:comment, val[0], index_for_value(val[0])),
+                  make_token(:newline, val[1], index_for_value(val[1]))
+                ]
 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 287)
+module_eval(<<'.,.,', 'demotape.y', 182)
   def _reduce_24(val, _values, result)
-                 @command_start_index = @token_index - 8
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 8),
-               make_token(:identifier, val[2], @token_index - 6),
-               make_token(:number, val[4], @token_index - 4),
-               make_token(:operator, ",", @token_index - 3),
-               make_token(:number, val[7], @token_index - 1)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], [val[4], val[7]], option: val[2])
-             cmd.tokens = tokens
-             result = attach_location(cmd).prepare!
-
+     result = [make_token(:newline, val[0], index_for_value(val[0]))]
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 302)
+module_eval(<<'.,.,', 'demotape.y', 184)
   def _reduce_25(val, _values, result)
-                 @command_start_index = @token_index - 6
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 6),
-               make_token(:identifier, val[2], @token_index - 4),
-               make_token(:duration, val[4], @token_index - 2)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], val[4], option: val[2])
-             cmd.tokens = tokens
-             result = attach_location(cmd).prepare!
+                    tokens, start_idx = val[0]
+                result = [
+                  { type: :command, tokens: tokens, line: line_for_token_at(start_idx), column: column_for_token_at(start_idx) },
+                  make_token(:newline, val[1], index_for_value(val[1]))
+                ]
 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 315)
+module_eval(<<'.,.,', 'demotape.y', 191)
   def _reduce_26(val, _values, result)
-                 @command_start_index = @token_index - 5
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 5),
-               make_token(:identifier, val[2], @token_index - 3),
-               make_token(:number, val[4], @token_index - 1)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], val[4].to_s, option: val[2])
-             cmd.tokens = tokens
-             result = attach_location(cmd).prepare!
+                    idx_lead = index_for_value(val[0])
+                tokens, _ = val[1]
+                result = [
+                  { type: :command, tokens: [make_token(:leading_space, val[0], idx_lead)] + tokens, line: line_for_token_at(idx_lead), column: column_for_token_at(idx_lead) },
+                  make_token(:newline, val[2], index_for_value(val[2]))
+                ]
 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 328)
+module_eval(<<'.,.,', 'demotape.y', 199)
   def _reduce_27(val, _values, result)
-                 @command_start_index = @token_index - 5
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 5),
-               make_token(:identifier, val[2], @token_index - 3),
-               make_token(:identifier, val[4], @token_index - 1)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], val[4], option: val[2])
-             cmd.tokens = tokens
-             result = attach_location(cmd).prepare!
+                    result = [
+                  make_token(:leading_space, val[0], index_for_value(val[0])),
+                  make_token(:comment, val[1], index_for_value(val[1])),
+                  make_token(:newline, val[2], index_for_value(val[2]))
+                ]
 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 341)
+module_eval(<<'.,.,', 'demotape.y', 208)
   def _reduce_28(val, _values, result)
-                 @command_start_index = @token_index - 5
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 5),
-               make_token(:identifier, val[2], @token_index - 3),
-               make_token(:string, val[4], @token_index - 1)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], val[4], option: val[2])
-             cmd.tokens = tokens
-             result = attach_location(cmd).prepare!
-
+     result = [[val[0][:token]], val[0][:index]]
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 354)
+module_eval(<<'.,.,', 'demotape.y', 209)
   def _reduce_29(val, _values, result)
-                 @command_start_index = @token_index - 3
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 3),
-               make_token(:string, val[2], @token_index - 1)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], val[2])
-             cmd.tokens = tokens
-             result = attach_location(cmd).prepare!
-
+     result = [val[0][0] << val[1][:token], val[0][1]]
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 366)
+module_eval(<<'.,.,', 'demotape.y', 211)
   def _reduce_30(val, _values, result)
-                 @command_start_index = @token_index - 4
-             duration_index = @token_index - 1  # TIME_UNIT position
-
-             tokens = [
-               make_token(:identifier, val[0], @token_index - 4),
-               make_token(:duration, val[2], @token_index - 2)
-             ]
-
-             cmd = DemoTape::Command.new(val[0], val[2])
-             cmd.tokens = tokens
-             result = attach_location(cmd, duration_index: duration_index).prepare!
-
+     idx = index_for_value(val[0]); result = { token: make_token(:identifier, val[0], idx), index: idx }
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 379)
+module_eval(<<'.,.,', 'demotape.y', 212)
   def _reduce_31(val, _values, result)
-                 @command_start_index = @token_index - 2
-
-             tokens = [make_token(:identifier, val[0], @token_index - 2)]
-
-             cmd = DemoTape::Command.new(val[0], "")
-             cmd.tokens = tokens
-             result = attach_location(cmd).prepare!
-
+     idx = index_for_value(val[0]); result = { token: make_token(:string, val[0], idx), index: idx }
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 389)
+module_eval(<<'.,.,', 'demotape.y', 213)
   def _reduce_32(val, _values, result)
-                   @last_key_index = @token_index - 1
-
-               tokens = [
-                 make_token(:identifier, val[0], @token_index - 3),
-                 make_token(:operator, "+", @token_index - 2),
-                 make_token(:identifier, val[2], @token_index - 1)
-               ]
-
-               result = { keys: [val[0], val[2]], tokens: tokens }
-
+     idx = index_for_value(val[0]); result = { token: make_token(:multiline_string, val[0], idx), index: idx }
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 400)
+module_eval(<<'.,.,', 'demotape.y', 214)
   def _reduce_33(val, _values, result)
-                   @last_key_index = @token_index - 1
-
-               tokens = [
-                 make_token(:identifier, val[0], @token_index - 3),
-                 make_token(:operator, "+", @token_index - 2),
-                 make_token(:number, val[2], @token_index - 1)
-               ]
-
-               result = { keys: [val[0], val[2].to_s], tokens: tokens }
-
+     idx = index_for_value(val[0]); result = { token: make_token(:number, val[0], idx), index: idx }
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 411)
+module_eval(<<'.,.,', 'demotape.y', 215)
   def _reduce_34(val, _values, result)
-                   @last_key_index = @token_index - 1
-
-               tokens = val[0][:tokens] + [
-                 make_token(:operator, "+", @token_index - 2),
-                 make_token(:identifier, val[2], @token_index - 1)
-               ]
-
-               result = { keys: val[0][:keys] << val[2], tokens: tokens }
-
+     idx = index_for_value(val[0]); result = { token: make_token(:duration, val[0], idx), index: idx }
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 421)
+module_eval(<<'.,.,', 'demotape.y', 216)
   def _reduce_35(val, _values, result)
-                   @last_key_index = @token_index - 1
-
-               tokens = val[0][:tokens] + [
-                 make_token(:operator, "+", @token_index - 2),
-                 make_token(:number, val[2], @token_index - 1)
-               ]
-
-               result = { keys: val[0][:keys] << val[2].to_s, tokens: tokens }
-
+     idx = index_for_value(val[0]); result = { token: make_token(:regex, val[0], idx), index: idx }
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 431)
+module_eval(<<'.,.,', 'demotape.y', 217)
   def _reduce_36(val, _values, result)
-     result = val[0]
+     idx = index_for_value(val[0]); result = { token: make_token(:space, val[0], idx), index: idx }
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 432)
+module_eval(<<'.,.,', 'demotape.y', 218)
   def _reduce_37(val, _values, result)
-     result = val[0]
+     idx = index_for_value(val[0]); result = { token: make_token(:trailing_space, val[0], idx), index: idx }
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'demotape.y', 434)
+module_eval(<<'.,.,', 'demotape.y', 219)
   def _reduce_38(val, _values, result)
-     result = "#{val[0]}#{val[1]}"
+     idx = index_for_value(val[0]); result = { token: make_token(:operator, ",", idx), index: idx }
+    result
+  end
+.,.,
+
+module_eval(<<'.,.,', 'demotape.y', 220)
+  def _reduce_39(val, _values, result)
+     idx = index_for_value(val[0]); result = { token: make_token(:operator, "+", idx), index: idx }
+    result
+  end
+.,.,
+
+module_eval(<<'.,.,', 'demotape.y', 221)
+  def _reduce_40(val, _values, result)
+     idx = index_for_value(val[0]); result = { token: make_token(:operator, "@", idx), index: idx }
+    result
+  end
+.,.,
+
+module_eval(<<'.,.,', 'demotape.y', 222)
+  def _reduce_41(val, _values, result)
+     idx = index_for_value(val[0]); result = { token: make_token(:time_unit, val[0], idx), index: idx }
     result
   end
 .,.,