automotive-ecu 0.3.6 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ab586bfaab8ab1d647a8f0f76cd247798baba7e10e207a9e03173a854fb13817
-  data.tar.gz: 39cf49dca991aa8966dd679914d9d88f05585e7b0a34fdff29a8783e2eae19d9
+  metadata.gz: 9e5446929b1a5bedccd68a5e669c148a0386663feda51949c8ebaee2fe143db3
+  data.tar.gz: ede842831f425f87c3092be6cd72185b82f5eb8b2a7d50ccc635ba23ec85b661
 SHA512:
-  metadata.gz: 21a979a28ff42b637574725b54a0dc46ff5a4659f0a26bdb50ec0e6a81c0262a46d14891af2f09f8846a545f1decb6e2994c4de92dbf05b8f0171c6f946b27f5
-  data.tar.gz: 2f2026c2d122d4ecd2295ff941c4fb397fe5b20383ab927e731a80178733d69e86a23343429abf751c8beab39b8483bd8c35ce62049a3af0302436668a986d6d
+  metadata.gz: 4b408ab9515e6909db8d4f9efa029ffd9cfa498afb9f5d5ecbff14a6283338741b6d8cc931db4b2b5eec52096ecb77b20e8072890d7e229b8b0f1fb1f176e02a
+  data.tar.gz: 7a9bed9c8d5568bccd4423f0578c28062e8e769b4a0c5bc44770d2356307bc1f8e33dcbebac0d430e48a6f8578021032d2c79c7072c42194828b269e4a52b445

data/lib/ecu/interfaces/dcm/dcm_parser.rb

@@ -1,238 +1,159 @@
 # frozen_string_literal: true
 
-require_relative "dcm_lexer"
-require_relative "dcm_parser_error"
-require "set"
+require_relative "dcm_scanner"
 
 class Ecu
   class LabelList
+    # Recursive-descent parser for DCM 2.0:
+    #
+    #   file      := comment* HEADER (comment | functions | label)*
+    #   functions := FUNKTIONEN (FKT ...)* END
+    #   label     := KIND name dimensions property* END
+    #   property  := KEYWORD value* EOL
+    #
+    # Comments before the header are #headers, those after it #subheaders.
     class DcmParser
+      LABEL_KINDS = {
+        "FESTWERT"                => Festwert,
+        "FESTWERTEBLOCK"          => Festwerteblock,
+        "KENNLINIE"               => Kennlinie,
+        "GRUPPENKENNLINIE"        => Gruppenkennlinie,
+        "FESTKENNLINIE"           => Festkennlinie,
+        "KENNFELD"                => Kennfeld,
+        "GRUPPENKENNFELD"         => Gruppenkennfeld,
+        "FESTKENNFELD"            => Festkennfeld,
+        "STUETZSTELLENVERTEILUNG" => Stuetzstellenverteilung,
+      }.freeze
+
+      FUNCTIONS = "FUNKTIONEN"
+      FUNCTION  = "FKT"
+      END_LABEL = "END"
+
+      # keyword => [label attribute, value reader of DcmScanner]
+      PROPERTIES = {
+        "LANGNAME"    => [:description, :text],
+        "FUNKTION"    => [:function,    :identifier],
+        "DISPLAYNAME" => [nil,          :identifier], # repeats the label name
+        "EINHEIT_X"   => [:xunit,       :text],
+        "EINHEIT_Y"   => [:yunit,       :text],
+        "EINHEIT_W"   => [:unit,        :text],
+        "ST/X"        => [:xvalue,      :numbers],
+        "ST/Y"        => [:yvalue,      :numbers],
+        "WERT"        => [:value,       :numbers],
+        "ST_TX/X"     => [:xvalue,      :texts],
+        "ST_TX/Y"     => [:yvalue,      :texts],
+        "TEXT"        => [:value,       :texts],
+      }.freeze
+
+      # Readers of values that may continue on following lines.
+      LIST_READERS = %i[numbers texts].freeze
+
+      attr_reader :headers, :subheaders, :labels
 
-      ARY_PROPERTIES = Set.new([:xvalue, :yvalue, :value]).freeze
-      KEY_MAPPING = {
-        "ST/X":         :xvalue,
-        "ST/Y":         :yvalue,
-        "WERT":         :value,
-        "ST_TX/X":      :xvalue,
-        "ST_TX/Y":      :yvalue,
-        "TEXT":         :value,
-        "FUNKTION":     :function,
-        "DISPLAYNAME":  :displayname,
-        "EINHEIT_X":    :xunit,
-        "EINHEIT_Y":    :yunit,
-        "EINHEIT_W":    :unit,
-        "LANGNAME":     :description,
-      }
-
-      attr_reader :lexer, :state, :labels, :headers, :subheaders
       def initialize(doc)
-        @lexer      = DcmLexer.new(doc)
-        @state      = :PRE_HEADER
-        @labels     = []
+        @scanner    = DcmScanner.new(doc)
         @headers    = []
         @subheaders = []
-        @properties = {}
-        reset_label!
+        @labels     = []
       end
 
       def call
-        while tok = lexer.next_token
-          begin
-            case @state
-            when :PRE_HEADER
-              case tok
-              when :HEADER  then next_state(:POST_HEADER)
-              when :COMMENT then add_header(lexer.token_value)
-              when :NEWLINE then # noop
-              else raise DcmParserError.new("Unexpected token #{debug_token(tok)} (state: #{@state})", lexer)
-              end
-            when :POST_HEADER
-              case tok
-              when :FUNKTIONEN              then next_state(:FUNCTIONS_CONTENT)
-              when :FESTWERT                then start_label(Festwert)
-              when :FESTWERTEBLOCK          then start_label(Festwerteblock)
-              when :KENNLINIE               then start_label(Kennlinie)
-              when :GRUPPENKENNLINIE        then start_label(Gruppenkennlinie)
-              when :FESTKENNLINIE           then start_label(Festkennlinie)
-              when :KENNFELD                then start_label(Kennfeld)
-              when :GRUPPENKENNFELD         then start_label(Gruppenkennfeld)
-              when :FESTKENNFELD            then start_label(Festkennfeld)
-              when :STUETZSTELLENVERTEILUNG then start_label(Stuetzstellenverteilung)
-              when :COMMENT                 then add_subheader(lexer.token_value)
-              when :NEWLINE                 then # noop
-              else raise DcmParserError.new("Unexpected token #{debug_token(tok)} (state: #{@state})", lexer)
-              end
-            when :FUNCTIONS_CONTENT
-              case tok
-              when :FKT     then next_state(:FUNCTION_CONTENT)
-              when :NEWLINE then # noop
-              when :END     then next_state(:POST_HEADER)
-              else raise DcmParserError.new("Unexpected token #{debug_token(tok)} (state: #{@state})", lexer)
-              end
-            when :FUNCTION_CONTENT
-              case tok
-              when :IDENTIFIER  then # append that shit?
-              when :QUOTED_TEXT then # append that shit as well?
-              when :NEWLINE     then next_state(:FUNCTIONS_CONTENT)
-              else raise DcmParserError.new("Unexpected token #{debug_token(tok)} (state: #{@state})", lexer)
-              end
-            when :LABEL_HEADLINE
-              case tok
-              when :IDENTIFIER     then add_name(lexer.token_value)
-              when :DIMENSIONS_SEP then # noop, order handled by #add_dimension
-              when :INT            then add_dimension(lexer.token_value.to_i)
-              when :NEWLINE        then next_state(:LABEL_CONTENT)
-              else raise DcmParserError.new("Unexpected token #{debug_token(tok)} (state: #{@state})", lexer)
-              end
-            when :LABEL_CONTENT
-              case tok
-              when :WERT        then start_property(:WERT,        :NUMERIC_PROPERTY)
-              when :TEXT        then start_property(:TEXT,        :TEXT_PROPERTY)
-              when :EINHEIT_X   then start_property(:EINHEIT_X,   :TEXT_PROPERTY)
-              when :EINHEIT_Y   then start_property(:EINHEIT_Y,   :TEXT_PROPERTY)
-              when :EINHEIT_W   then start_property(:EINHEIT_W,   :TEXT_PROPERTY)
-              when :LANGNAME    then start_property(:LANGNAME,    :TEXT_PROPERTY)
-              when :FUNKTION    then start_property(:FUNKTION,    :ID_PROPERTY)
-              when :DISPLAYNAME then start_property(:DISPLAYNAME, :ID_PROPERTY)
-              when :END         then labels << finish_label
-              when :"ST/X"      then start_property(:"ST/X",      :NUMERIC_PROPERTY)
-              when :"ST/Y"      then start_property(:"ST/Y",      :NUMERIC_PROPERTY)
-              when :"ST_TX/X"   then start_property(:"ST_TX/X",   :TEXT_PROPERTY)
-              when :"ST_TX/Y"   then start_property(:"ST_TX/Y",   :TEXT_PROPERTY)
-              when :NEWLINE     then # noop, DCM badly formatted
-              when :COMMENT     then # noop, some programs add SST in comments
-              else raise DcmParserError.new("Unexpected token #{debug_token(tok)} (state: #{@state})", lexer)
-              end
-            when :NUMERIC_PROPERTY
-              case tok
-              when :FLOAT   then append_property(lexer.token_value.to_f)
-              when :INT     then append_property(lexer.token_value.to_i)
-              when :NEWLINE then finish_property
-              else raise DcmParserError.new("Unexpected token #{debug_token(tok)} (state: #{@state})", lexer)
-              end
-            when :TEXT_PROPERTY
-              case tok
-              when :QUOTED_TEXT then append_property(lexer.quoted_value)
-              when :NEWLINE     then finish_property
-              else raise DcmParserError.new("Unexpected token #{debug_token(tok)} (state: #{@state})", lexer)
-              end
-            when :ID_PROPERTY
-              case tok
-              when :IDENTIFIER then append_property(lexer.token_value)
-              when :NEWLINE    then finish_property
-              else raise DcmParserError.new("Unexpected token #{debug_token(tok)} (state: #{@state})", lexer)
-              end
-            end
-          rescue DcmParserError
-            raise
-          rescue StandardError => e
-            raise DcmParserError.new("#{e.message} on #{e.backtrace.first} (state: #{@state})", lexer)
-          end
-        end
-        fail_on_truncation!
+        read_comments(headers)
+        @scanner.header or raise @scanner.error("Expected #{DcmScanner::HEADER}")
+
+        read_blocks
         [labels, headers, subheaders]
       end
 
-      # A well-formed file ends between labels, not inside one.
-      def fail_on_truncation!
-        return if %i[PRE_HEADER POST_HEADER].include?(@state)
+      private
 
-        raise DcmParserError.new("Unexpected end of file (state: #{@state})", lexer)
-      end
+      def read_comments(into)
+        loop do
+          @scanner.skip_blank_lines
+          comment = @scanner.comment or return
 
-      def debug_token(tok)
-        if tok == :UNKNOWN_CHAR
-          "UNKNOWN_CHAR: #{lexer.token_value}"
-        else
-          tok
+          into << comment
         end
       end
 
-      def next_state(newstate)
-        yield if block_given?
-      ensure
-        @state = newstate
-      end
+      def read_blocks
+        loop do
+          read_comments(subheaders)
+          return if @scanner.at_end?
 
-      def add_header(str)
-        @headers << str[1..].strip
-      end
-
-      def add_subheader(str)
-        @subheaders << str[1..].strip
-      end
+          start   = @scanner.pos
+          keyword = @scanner.keyword
+          next skip_functions if keyword == FUNCTIONS
 
-      def start_label(constructor)
-        next_state(:LABEL_HEADLINE) do
-          @constructor = constructor
+          kind = LABEL_KINDS[keyword] or raise @scanner.error("Unexpected #{keyword || "text"}", start)
+          labels << read_label(kind, start)
         end
       end
 
-      def finish_label
-        next_state(:POST_HEADER) do
-          @properties.delete(:displayname)
-          @constructor.new(**@properties).tap { reset_label! }
-        end
-      end
+      # The function list is not used, but must be well-formed.
+      def skip_functions
+        @scanner.end_of_line
 
-      def add_name(str)
-        fail "Duplicate name" if @properties.key?(:name)
+        loop do
+          @scanner.skip_noise
+          at      = @scanner.pos
+          keyword = @scanner.keyword
 
-        @properties[:name] = str
+          case keyword
+          when END_LABEL then return @scanner.end_of_line
+          when FUNCTION  then @scanner.skip_line
+          else raise @scanner.error("Unexpected #{keyword || "end of file"} in #{FUNCTIONS}", at)
+          end
+        end
       end
 
-      def add_dimension(n)
-        fail "Wrong order name/dimensions" unless @properties.key?(:name)
-        fail "Too many dimensions" if @properties.key?(:xdim) && @properties.key?(:ydim)
+      def read_label(kind, start)
+        props = { name: @scanner.name }
+        props.merge!(@scanner.dimensions)
 
-        if @properties.key?(:xdim)
-          @properties[:ydim] = n
-        else
-          @properties[:xdim] = n
-        end
-      end
+        loop do
+          @scanner.skip_noise
+          at      = @scanner.pos
+          keyword = @scanner.keyword or raise @scanner.error(missing_property_message, at)
+          break @scanner.end_of_line if keyword == END_LABEL
 
-      def start_property(key, newstate)
-        next_state(newstate) do
-          @key    = KEY_MAPPING[key]
-          @is_ary = ARY_PROPERTIES.include?(@key)
-          if @is_ary
-            @properties[@key] ||= []
-            @value = @properties[@key]   # alias: append_property writes directly to target
-          else
-            @value = nil
-          end
+          read_property(keyword, props, at)
         end
+
+        build(kind, props, start)
       end
 
-      def append_property(value)
-        if @is_ary
-          @value << value
-        else
-          fail "Multiple definitions for #{@key}" if @value
+      def read_property(keyword, props, at)
+        attribute, reader = PROPERTIES[keyword]
+        raise @scanner.error("Unknown property #{keyword}", at) unless reader
 
-          @value = value
-        end
-      end
+        value = @scanner.public_send(reader)
+        return unless attribute
 
-      def finish_property
-        next_state(:LABEL_CONTENT) do
-          next if @is_ary  # values already in @properties[@key] via @value alias
+        store(props, attribute, reader, value, at)
+      end
 
-          fail "Multiple property lines not allowed for #{@key}" if @properties.key?(@key)
-          @properties[@key] = @value
+      # Lists such as ST/X may span several lines, single values may not.
+      def store(props, attribute, reader, value, at)
+        if LIST_READERS.include?(reader)
+          (props[attribute] ||= []).concat(value)
+          return
         end
+
+        raise @scanner.error("Duplicate #{attribute}", at) if props.key?(attribute)
+
+        props[attribute] = value
       end
 
-      def reset_label!
-        @constructor = nil
-        @properties.clear
-        @key         = nil
-        @is_ary      = false
-        @value       = nil
+      def build(kind, props, start)
+        kind.new(**props)
+      rescue ArgumentError => e
+        raise @scanner.error("#{props[:name]}: #{e.message.lines.first.strip}", start)
       end
 
+      def missing_property_message
+        @scanner.at_end? ? "Unexpected end of file" : "Expected a property or #{END_LABEL}"
+      end
     end
-
   end
 end

data/lib/ecu/interfaces/dcm/dcm_parser_error.rb

@@ -4,10 +4,12 @@ class Ecu
     CTXLENGHT = 5
 
     attr_reader :doc, :lineno
-    def initialize(msg, lexer)
+
+    # offset: byte position in doc where parsing failed
+    def initialize(msg, doc, offset)
       @msg    = msg
-      @doc    = lexer.doc
-      @lineno = lexer.lineno
+      @doc    = doc
+      @lineno = doc.byteslice(0, offset).count("\n") + 1
     end
 
     def message = @msg

data/lib/ecu/interfaces/dcm/dcm_scanner.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require "strscan"
+require_relative "dcm_parser_error"
+
+class Ecu
+  class LabelList
+    # Reads DCM text one construct at a time. Each reader consumes what it
+    # returns, including the line break, and raises DcmParserError when the
+    # input is malformed.
+    #
+    # Values must be separated by blanks, also from their keyword:
+    #
+    #   WERT 1 2.5      ok
+    #   WERT-1          rejected
+    #   ST/X 1.5.2      rejected
+    class DcmScanner
+      DIMENSION_NAMES = %i[xdim ydim].freeze
+      DIMENSION_SEP   = "@"
+      MAX_DIMENSIONS  = DIMENSION_NAMES.size
+
+      HEADER       = "KONSERVIERUNG_FORMAT 2.0"
+      LINE_END     = /(?:\r\n|\n|\r)/
+      EOL          = /[ \t]*(?:\r\n|\n|\r|\z)/
+      BLANK_LINES  = /(?:[ \t]*#{LINE_END})*(?:[ \t]+\z)?/
+      NOISE        = /(?:[ \t]*(?:\*[^\r\n]*)?#{LINE_END})*/
+      COMMENT      = /[ \t]*\*([^\r\n]*)(?:\r\n|\n|\r|\z)/
+      KEYWORD      = %r{[ \t]*([A-Z][A-Z_/]*)}
+      IDENTIFIER   = /[ \t]+([A-Za-z][A-Za-z0-9_.\[\]]*)/
+      QUOTED       = /[ \t]+"([^"]*)"/
+      REST_OF_LINE = /[^\r\n]*/
+      NUMBER_CHARS = /\A[-+.eE0-9 \t]*\z/
+      TRAILING_DOT = /\A[-+]?[0-9]+\.\z/ # "9." is a valid DCM number, Ruby disagrees
+
+      attr_reader :doc
+
+      def initialize(doc)
+        @doc = doc
+        @s   = StringScanner.new(doc)
+      end
+
+      def pos     = @s.pos
+      def at_end? = @s.eos?
+
+      def error(message, at = pos)
+        DcmParserError.new(message, doc, at)
+      end
+
+      def header
+        @s.skip(HEADER) or return false
+
+        end_of_line
+        true
+      end
+
+      def skip_blank_lines
+        @s.skip(BLANK_LINES)
+      end
+
+      # Blank and comment lines, which carry no data inside a label.
+      def skip_noise
+        @s.skip(NOISE)
+      end
+
+      # Text of the next comment line without the leading "*", or nil.
+      def comment
+        @s.skip(COMMENT) ? @s[1].strip : nil
+      end
+
+      # Upper-case word such as WERT, ST_TX/X or END, or nil.
+      def keyword
+        @s.skip(KEYWORD) ? @s[1] : nil
+      end
+
+      def end_of_line
+        return if @s.skip(EOL)
+
+        raise error("Unexpected #{@s.rest.lines.first.strip.inspect}")
+      end
+
+      def skip_line
+        @s.skip(REST_OF_LINE)
+        end_of_line
+      end
+
+      # Label name, which is followed by the dimensions on the same line.
+      def name
+        @s.skip(IDENTIFIER) or raise error("Expected an identifier")
+        @s[1]
+      end
+
+      def identifier
+        value = name
+        end_of_line
+        value
+      end
+
+      # Exactly one quoted text, e.g. ` "kg"`.
+      def text
+        at     = pos
+        values = texts
+        raise error("Expected one quoted text", at) unless values.size == 1
+
+        values.first
+      end
+
+      # Any number of quoted texts, e.g. ` "a" "b"`.
+      def texts
+        values = []
+        values << @s[1] while @s.skip(QUOTED)
+        end_of_line
+        values
+      end
+
+      # Any number of integers and floats, e.g. ` 1 2.5 -3e2`.
+      def numbers
+        at   = pos
+        line = @s.scan(REST_OF_LINE)
+
+        unless line.empty? || line.start_with?(" ", "\t")
+          raise error("Whitespace required before value", at)
+        end
+        raise error("Invalid number in #{line.strip.inspect}", at) unless line.match?(NUMBER_CHARS)
+
+        values = line.split
+        values.map! { number(_1) or raise error("Invalid number #{_1.inspect}", at) }
+        end_of_line
+        values
+      end
+
+      # Label dimensions as { xdim:, ydim: }, e.g. ` 3 @ 4` or ` 3`.
+      def dimensions
+        at     = pos
+        tokens = @s.scan(REST_OF_LINE).split
+        tokens.delete(DIMENSION_SEP)
+        raise error("Too many dimensions", at) if tokens.size > MAX_DIMENSIONS
+
+        ints = tokens.map { Integer(_1, 10, exception: false) or raise error("Invalid dimension #{_1.inspect}", at) }
+        end_of_line
+        DIMENSION_NAMES.first(ints.size).zip(ints).to_h
+      end
+
+      private
+
+      def number(token)
+        Integer(token, 10, exception: false) ||
+          Float(token, exception: false) ||
+          (token.to_f if TRAILING_DOT.match?(token))
+      end
+    end
+  end
+end

data/lib/ecu/labels/label_list_comparison.rb

@@ -63,6 +63,9 @@ class Ecu
       LabelList.new(labels, @unified_headers, @unified_subheaders)
     end
 
+    def headers = @unified_headers
+    def subheaders = @unified_subheaders
+
     private
 
     attr_reader :left, :right

data/lib/ecu/signals/signal_list_comparison.rb

@@ -44,6 +44,9 @@ class Ecu
         @unified_subheaders
     end
 
+    def headers = @unified_headers
+    def subheaders = @unified_subheaders
+
     private
 
     attr_reader :left, :right

data/lib/ecu/version.rb

@@ -1,3 +1,3 @@
 class Ecu
-  VERSION = "0.3.6"
+  VERSION = "0.4.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: automotive-ecu
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.4.1
 platform: ruby
 authors:
 - Jonas Mueller
@@ -99,9 +99,9 @@ files:
 - lib/ecu/interfaces/a2l/label_list.rb
 - lib/ecu/interfaces/a2l/signal_list.rb
 - lib/ecu/interfaces/dbc/signal_list.rb
-- lib/ecu/interfaces/dcm/dcm_lexer.rb
 - lib/ecu/interfaces/dcm/dcm_parser.rb
 - lib/ecu/interfaces/dcm/dcm_parser_error.rb
+- lib/ecu/interfaces/dcm/dcm_scanner.rb
 - lib/ecu/interfaces/dcm/festkennfeld.rb
 - lib/ecu/interfaces/dcm/festkennlinie.rb
 - lib/ecu/interfaces/dcm/festwert.rb
@@ -164,7 +164,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.23
 specification_version: 4
 summary: A simple API for managing automotive ecu signals and labels
 test_files: []

data/lib/ecu/interfaces/dcm/dcm_lexer.rb

@@ -1,90 +0,0 @@
-# frozen_string_literal: true
-
-require "strscan"
-
-class Ecu
-  class LabelList
-    class DcmLexer
-      attr_reader :doc
-      def initialize(doc)
-        @doc  = doc
-        @scan = StringScanner.new(doc)
-      end
-
-      KEYWORDS = [
-        "FUNKTIONEN", "FESTWERT", "FESTWERTEBLOCK", "KENNLINIE", "GRUPPENKENNLINIE", "FESTKENNLINIE",
-        "KENNFELD", "GRUPPENKENNFELD", "FESTKENNFELD", "STUETZSTELLENVERTEILUNG",
-        "FKT",
-        "ST/X", "ST/Y", "WERT", "ST_TX/X", "ST_TX/Y", "TEXT",
-        "FUNKTION",
-        "EINHEIT_X", "EINHEIT_Y", "EINHEIT_W",
-        "LANGNAME", "DISPLAYNAME",
-        "END",
-      ].freeze
-
-      HEADER         = "KONSERVIERUNG_FORMAT 2.0"
-      WHITESPACE     = %r{ [ \t]+ }x
-      NEWLINE        = %r{ \r\n|\n|\r }x
-      COMMENT        = %r{ ^\*.*$ }x
-      DIMENSIONS_SEP = %r{ @ }x
-      QUOTED_TEXT    = %r{ "[^"]*" }x
-      IDENTIFIER     = %r{ [A-Za-z][A-Za-z0-9_\.\[\]]* }x
-      UNSIGNED_INT   = %r{ [0]|[1-9][0-9]* }x
-      INT            = %r{ [-]?#{UNSIGNED_INT} }x
-      FLOAT_EXP      = %r{ [eE][+-]?[0-9]+ }x
-      FLOAT          = %r{
-                            [-+]?
-                            (?:
-                               #{UNSIGNED_INT}?[.][0-9]+#{FLOAT_EXP} |      # 1.23e10 or .45e3
-                               #{UNSIGNED_INT}          #{FLOAT_EXP} |      # 3e4
-                               #{UNSIGNED_INT}?[.][0-9]+             |      # 7.3 or .50
-                               #{UNSIGNED_INT} [.][0-9]*                    # 3.02 or 9.
-                            )
-                       }x
-      KW_RE          = /#{Regexp.union(KEYWORDS.sort)}\b/
-      KW_TABLE       = Hash[KEYWORDS.map { [_1, _1.upcase.to_sym] }]
-
-
-      def next_token
-        # This is a hot path during DCM parsing. Make sure to optimize
-        # here are far as possible
-
-        @scan.skip(WHITESPACE)
-
-        return if @scan.eos?
-
-        case @doc.getbyte(@scan.pos)
-        when 10, 13          then @scan.skip(NEWLINE)        && :NEWLINE        # \n \r
-        when 34              then @scan.skip(QUOTED_TEXT)    && :QUOTED_TEXT    # "
-        when 42              then @scan.skip(COMMENT)        && :COMMENT        # *
-        when 64              then @scan.skip(DIMENSIONS_SEP) && :DIMENSIONS_SEP # @
-        when 43, 45, 46, 48..57                                                 # + - . 0-9
-          if    @scan.skip(FLOAT) then :FLOAT
-          elsif @scan.skip(INT)   then :INT
-          else  @scan.getch;           :UNKNOWN_CHAR
-          end
-        else
-          if    s = @scan.scan(KW_RE)    then KW_TABLE[s]
-          elsif @scan.skip(HEADER)       then :HEADER
-          elsif @scan.skip(IDENTIFIER)   then :IDENTIFIER
-          else  @scan.getch;                  :UNKNOWN_CHAR
-          end
-        end
-      end
-
-      def token_value
-        @scan.matched
-      end
-
-      # Extracts quoted-text content without surrounding quotes in a single allocation.
-      def quoted_value
-        @scan.string.byteslice(@scan.pos - @scan.matched_size + 1, @scan.matched_size - 2)
-      end
-
-      def lineno
-        @doc.byteslice(0, @scan.pos).count("\n") +
-          (@scan.beginning_of_line? ? 0 : 1)
-      end
-    end
-  end
-end