oedipus_lex 2.4.1 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ab5673d633e7abae710deb96e4b764b2518bed0e
-  data.tar.gz: 9e16b31d4ffeaad609fe27812e69f8dfa3b0b8ff
+  metadata.gz: 61879597d87d6ef9e59b495a38e10617e03f3277
+  data.tar.gz: e5e415035c15bc8c02007a3479f0bbd2fd0d8621
 SHA512:
-  metadata.gz: c3948a898e603ccb19494bdcb753f1ae5f799d21660c42a8ba80611884e18a94f615ef1511c79d1ac5f592e6ea7288ab673b6729988e8f8a07d37c881e9f6c6f
-  data.tar.gz: 7876d02faa16c81ef199ceec5b342cf9bd69e9e0d6c4970328e8e92c49abc3904dd06e1b1a40734fe5cc80314030e2b5cc21d7a2ec60afb60ff66f532a93572c
+  metadata.gz: dd4aea54ccd7eb1731bdc5639d4932fe1eadb5b188cf9a79a75971599adeadbfbc867b5790fcb09cf8bdad68f09f715154852a251953b0aa37a66f142c6bc423
+  data.tar.gz: 0beb1c01b05a3e567bc0c0702c2c675096a96e0801604e89d28ef7197e7e02fd154a5c1facff60447d068211d2013e04a67e8be260bdcfb2420222a072ff4e97

data/History.rdoc

@@ -1,3 +1,18 @@
+=== 2.5.0 / 2016-11-30
+
+* 5 minor enhancements:
+
+  * Added #location to generated template, provides file:line:column per options.
+  * Added LexerError and made ScanError subclass it.
+  * Added column option.
+  * Errors try to provide location now.
+  * Re-bootstrapped.
+
+* 2 bug fixes:
+
+  * Fixed some whitespace generation when using :column.
+  * Fixed wiring on column. (steakknife)
+
 === 2.4.1 / 2016-01-21
 
 * 1 minor enhancement:

data/README.rdoc

@@ -49,6 +49,7 @@ resource for CS learning. Books... books are good. I like books.
                   | /debug/i
                   | /do_parse/i
                   | /lineno/i
+                  | /column/i
 
   inner_section   = /inner/ NL (misc_line)*
 
@@ -129,6 +130,8 @@ Specify `lineno` to generate automatic line number handling at the
 beginning of `next_token`. This was the default in 1.0.0 and you must
 now activate it.
 
+Specify `column` to generate automatic column number handling.
+
 ==== Inner
 
 The inner section is just code, like header or footer, but inner gets

data/lib/oedipus_lex.rb

@@ -4,7 +4,7 @@ require "erb"
 require "oedipus_lex.rex"
 
 class OedipusLex
-  VERSION = "2.4.1"
+  VERSION = "2.5.0"
 
   attr_accessor :class_name
   attr_accessor :header
@@ -270,7 +270,8 @@ class OedipusLex
 %   end
 
 % end
-        class ScanError < StandardError ; end
+        class LexerError < StandardError ; end
+        class ScanError < LexerError ; end
 
 % if option[:lineno] then
         attr_accessor :lineno
@@ -298,8 +299,8 @@ class OedipusLex
           idx = ss.string.rindex("\n", old_pos) || -1
           old_pos - idx - 1
         end
-% end
 
+% end
 % if option[:do_parse] then
         def do_parse
           while token = next_token do
@@ -331,6 +332,20 @@ class OedipusLex
           end
         end
 
+        def location
+          [
+            (filename || "<input>"),
+% if option[:lineno] then
+            lineno,
+% elsif option[:column] then
+            "?",
+% end
+% if option[:column] then
+            column,
+% end
+          ].compact.join(":")
+        end
+
         def next_token
 % starts.each do |s|
           <%= s %>
@@ -358,17 +373,17 @@ class OedipusLex
 %   end # the_states.each
                 else
                   text = ss.string[ss.pos .. -1]
-                  raise ScanError, "can not match (#{state.inspect}): '#{text}'"
+                  raise ScanError, "can not match (#{state.inspect}) at #{location}: '#{text}'"
                 end
 % end # all_states
               else
-                raise ScanError, "undefined state: '#{state}'"
+                raise ScanError, "undefined state at #{location}: '#{state}'"
               end # token = case state
 
             next unless token # allow functions to trigger redo w/ nil
           end # while
 
-          raise "bad lexical result: #{token.inspect}" unless
+          raise LexerError, "bad lexical result at #{location}: #{token.inspect}" unless
             token.nil? || (Array === token && token.size >= 2)
 
           # auto-switch state

data/lib/oedipus_lex.rex

@@ -4,6 +4,7 @@ option
 
   do_parse
   lineno
+  column
 
 macro
   ST    /(?:(:\S+|\w+\??))/
@@ -31,6 +32,7 @@ rule
   :option       /debug/i                { [:option, text] }
   :option       /do_parse/i             { [:option, text] }
   :option       /lineno/i               { [:option, text] }
+  :option       /column/i               { [:option, text] }
 
   :inner        /.*/                    { [:inner, text] }
 

data/lib/oedipus_lex.rex.rb

@@ -1,7 +1,7 @@
 # encoding: UTF-8
 #--
 # This file is automatically generated. Do not modify it.
-# Generated by: oedipus_lex version 2.3.0.
+# Generated by: oedipus_lex version 2.4.1.
 # Source: lib/oedipus_lex.rex
 #++
 
@@ -12,7 +12,8 @@ class OedipusLex
   RE  = /(\/(?:\\.|[^\/])+\/[ion]?)/
   ACT = /(\{.*|:?\w+)/
 
-  class ScanError < StandardError ; end
+  class LexerError < StandardError ; end
+  class ScanError < LexerError ; end
 
   attr_accessor :lineno
   attr_accessor :filename
@@ -31,6 +32,13 @@ class OedipusLex
     yield
   end
 
+  attr_accessor :old_pos
+
+  def column
+    idx = ss.string.rindex("\n", old_pos) || -1
+    old_pos - idx - 1
+  end
+
   def do_parse
     while token = next_token do
       type, *vals = token
@@ -58,35 +66,44 @@ class OedipusLex
     end
   end
 
+  def location
+    [
+      (filename || "<input>"),
+      lineno,
+      column,
+    ].compact.join(":")
+  end
+
   def next_token
-    self.lineno += 1 if ss.peek(1) == "\n"
 
     token = nil
 
     until ss.eos? or token do
+      self.lineno += 1 if ss.peek(1) == "\n"
+      self.old_pos = ss.pos
       token =
         case state
         when nil, :option, :inner, :start, :macro, :rule, :group then
           case
-          when text = ss.scan(/options?.*/) then
+          when ss.skip(/options?.*/) then
             [:state, :option]
-          when text = ss.scan(/inner.*/) then
+          when ss.skip(/inner.*/) then
             [:state, :inner]
-          when text = ss.scan(/macros?.*/) then
+          when ss.skip(/macros?.*/) then
             [:state, :macro]
-          when text = ss.scan(/rules?.*/) then
+          when ss.skip(/rules?.*/) then
             [:state, :rule]
-          when text = ss.scan(/start.*/) then
+          when ss.skip(/start.*/) then
             [:state, :start]
-          when text = ss.scan(/end/) then
+          when ss.skip(/end/) then
             [:state, :END]
-          when text = ss.scan(/\A((?:.|\n)*)class ([\w:]+.*)/) then
+          when ss.skip(/\A((?:.|\n)*)class ([\w:]+.*)/) then
             action { [:class, *matches] }
-          when text = ss.scan(/\n+/) then
+          when ss.skip(/\n+/) then
             # do nothing
           when text = ss.scan(/\s*(\#.*)/) then
             action { [:comment, text] }
-          when (state == :option) && (text = ss.scan(/\s+/)) then
+          when (state == :option) && (ss.skip(/\s+/)) then
             # do nothing
           when (state == :option) && (text = ss.scan(/stub/i)) then
             action { [:option, text] }
@@ -96,44 +113,46 @@ class OedipusLex
             action { [:option, text] }
           when (state == :option) && (text = ss.scan(/lineno/i)) then
             action { [:option, text] }
+          when (state == :option) && (text = ss.scan(/column/i)) then
+            action { [:option, text] }
           when (state == :inner) && (text = ss.scan(/.*/)) then
             action { [:inner, text] }
           when (state == :start) && (text = ss.scan(/.*/)) then
             action { [:start, text] }
-          when (state == :macro) && (text = ss.scan(/\s+(\w+)\s+#{RE}/o)) then
+          when (state == :macro) && (ss.skip(/\s+(\w+)\s+#{RE}/o)) then
             action { [:macro, *matches] }
-          when (state == :rule) && (text = ss.scan(/\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o)) then
+          when (state == :rule) && (ss.skip(/\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o)) then
             action { [:rule, *matches] }
-          when (state == :rule) && (text = ss.scan(/\s*:[\ \t]*#{RE}/o)) then
+          when (state == :rule) && (ss.skip(/\s*:[\ \t]*#{RE}/o)) then
             action { [:grouphead, *matches] }
-          when (state == :group) && (text = ss.scan(/\s*:[\ \t]*#{RE}/o)) then
+          when (state == :group) && (ss.skip(/\s*:[\ \t]*#{RE}/o)) then
             action { [:grouphead, *matches] }
-          when (state == :group) && (text = ss.scan(/\s*\|\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o)) then
+          when (state == :group) && (ss.skip(/\s*\|\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o)) then
             action { [:group, *matches] }
-          when (state == :group) && (text = ss.scan(/\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o)) then
+          when (state == :group) && (ss.skip(/\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o)) then
             action { [:groupend, *matches] }
           else
             text = ss.string[ss.pos .. -1]
-            raise ScanError, "can not match (#{state.inspect}): '#{text}'"
+            raise ScanError, "can not match (#{state.inspect}) at #{location}: '#{text}'"
           end
         when :END then
           case
-          when text = ss.scan(/\n+/) then
+          when ss.skip(/\n+/) then
             # do nothing
           when text = ss.scan(/.*/) then
             action { [:end, text] }
           else
             text = ss.string[ss.pos .. -1]
-            raise ScanError, "can not match (#{state.inspect}): '#{text}'"
+            raise ScanError, "can not match (#{state.inspect}) at #{location}: '#{text}'"
           end
         else
-          raise ScanError, "undefined state: '#{state}'"
+          raise ScanError, "undefined state at #{location}: '#{state}'"
         end # token = case state
 
       next unless token # allow functions to trigger redo w/ nil
     end # while
 
-    raise "bad lexical result: #{token.inspect}" unless
+    raise LexerError, "bad lexical result at #{location}: #{token.inspect}" unless
       token.nil? || (Array === token && token.size >= 2)
 
     # auto-switch state

data/test/test_oedipus_lex.rb

@@ -82,6 +82,26 @@ class TestOedipusLex < Minitest::Test
     assert_equal expected_msg, e.message
   end
 
+  def assert_token_error grammar, input, expected_msg
+    _, mod = eval_lexer grammar
+
+    calc = mod::Calculator.new
+
+    def calc.do_parse
+      tokens = []
+      while token = next_token
+        tokens << token
+      end
+      tokens
+    end
+
+    e = assert_raises mod::Calculator::LexerError do
+      calc.parse input
+    end
+
+    assert_equal expected_msg, e.message
+  end
+
   def test_simple_scanner
     src = <<-'REX'
       class Calculator
@@ -136,7 +156,7 @@ class TestOedipusLex < Minitest::Test
       end
     REX
 
-    assert_generate_error src, "can not match (:rule): '"
+    assert_generate_error src, "can not match (:rule) at <input>:4:0: '"
   end
 
   def test_simple_scanner_macro
@@ -760,9 +780,50 @@ class TestOedipusLex < Minitest::Test
     assert_lexer src, txt, exp
 
     txt = "aa"
-    exp = [[:A, 'a'], [:B, 'b'], [:A, 'a'], [:B, 'b'], [:A, 'a']]
 
-    assert_lexer_error src, txt, "can not match (:B): 'a'"
+    assert_lexer_error src, txt, "can not match (:B) at <input>: 'a'"
+  end
+
+  def test_error_undefined_state
+    src = <<-'REX'
+      class Calculator
+      rule
+             /a/       { self.state = :C  ; [:A, text] }
+        :B   /b/       { self.state = nil ; [:B, text] }
+      end
+    REX
+
+    txt = "aa"
+
+    assert_lexer_error src, txt, "undefined state at <input>: 'C'"
+  end
+
+  def test_error_bad_token
+    src = <<-'REX'
+      class Calculator
+      rule
+             /a/       { self.state = :B  ; :A }
+        :B   /b/       { self.state = nil ; [:B, text] }
+      end
+    REX
+
+    txt = "aa"
+
+    assert_token_error src, txt, "bad lexical result at <input>: :A"
+  end
+
+  def test_error_bad_token_size
+    src = <<-'REX'
+      class Calculator
+      rule
+             /a/       { self.state = :B  ; [:A] }
+        :B   /b/       { self.state = nil ; [:B, text] }
+      end
+    REX
+
+    txt = "aa"
+
+    assert_token_error src, txt, "bad lexical result at <input>: [:A]"
   end
 
   def test_incrementing_lineno_on_nil_token
@@ -781,4 +842,36 @@ class TestOedipusLex < Minitest::Test
 
     assert_lexer src, txt, exp
   end
+
+  def assert_location exp, option = {}
+    self.option = option
+
+    src = "class Calculator\nrule\n  /\\d+/ { [:number, text.to_i] }\nend\n"
+
+    _, mod = eval_lexer src
+
+    calc = mod::Calculator.new
+    def calc.do_parse
+      [next_token]
+    end
+
+    calc.filename = option[:filename] if option[:filename]
+    calc.parse "42"
+
+    assert_equal exp, calc.location
+  end
+
+  def test_location
+    t = true
+
+    assert_location "<input>"
+    assert_location "<input>:1",   :lineno => t
+    assert_location "<input>:?:0", :column => t
+    assert_location "<input>:1:0", :lineno => t, :column => t
+
+    assert_location "blah",     :filename => "blah"
+    assert_location "blah:1",   :filename => "blah", :lineno => t
+    assert_location "blah:?:0", :filename => "blah", :column => t
+    assert_location "blah:1:0", :filename => "blah", :lineno => t, :column => t
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: oedipus_lex
 version: !ruby/object:Gem::Version
-  version: 2.4.1
+  version: 2.5.0
 platform: ruby
 authors:
 - Ryan Davis
@@ -10,9 +10,9 @@ bindir: bin
 cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
-  MIIDPjCCAiagAwIBAgIBAzANBgkqhkiG9w0BAQUFADBFMRMwEQYDVQQDDApyeWFu
+  MIIDijCCAnKgAwIBAgIBATANBgkqhkiG9w0BAQUFADBFMRMwEQYDVQQDDApyeWFu
   ZC1ydWJ5MRkwFwYKCZImiZPyLGQBGRYJemVuc3BpZGVyMRMwEQYKCZImiZPyLGQB
-  GRYDY29tMB4XDTE1MDkxOTIwNTEyMloXDTE2MDkxODIwNTEyMlowRTETMBEGA1UE
+  GRYDY29tMB4XDTE2MDkyNjAxNTczNVoXDTE3MDkyNjAxNTczNVowRTETMBEGA1UE
   AwwKcnlhbmQtcnVieTEZMBcGCgmSJomT8ixkARkWCXplbnNwaWRlcjETMBEGCgmS
   JomT8ixkARkWA2NvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALda
   b9DCgK+627gPJkB6XfjZ1itoOQvpqH1EXScSaba9/S2VF22VYQbXU1xQXL/WzCkx
@@ -20,31 +20,18 @@ cert_chain:
   oOvjtt5P8+GSK9zLzxQP0gVLS/D0FmoE44XuDr3iQkVS2ujU5zZL84mMNqNB1znh
   GiadM9GHRaDiaxuX0cIUBj19T01mVE2iymf9I6bEsiayK/n6QujtyCbTWsAS9Rqt
   qhtV7HJxNKuPj/JFH0D2cswvzznE/a5FOYO68g+YCuFi5L8wZuuM8zzdwjrWHqSV
-  gBEfoTEGr7Zii72cx+sCAwEAAaM5MDcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAw
-  HQYDVR0OBBYEFEfFe9md/r/tj/Wmwpy+MI8d9k/hMA0GCSqGSIb3DQEBBQUAA4IB
-  AQB+Hx8xUgrpZa4P8H8gR8zme5kISwQrG80MbpqJV6/G3/ZicRFhN5sjwu0uHGue
-  bd9Cymf6oIRwHVarJux2M32T6bL07Hmi07w2QaPc3MnMKB/D46SRZ2JSSGPFRBTc
-  SilobMRoGs/7B15uGFUEnNrCB/ltMqhwwSx1r++UQPfeySHEV9uqu03E5Vb7J37O
-  2Er6PLXHRiYsIycD1LkMi6YnixdITRHmrqJYE2rsjaIfpIehiusVAPHkNf7qbpHq
-  qx3h45R1CAsObX0SQDIT+rRbQrtKz1GHIZTOFYvEJjUY1XmRTZupD3CJ8Q7sDqSy
-  NLq5jm1fq6Y9Uolu3RJbmycf
+  gBEfoTEGr7Zii72cx+sCAwEAAaOBhDCBgTAJBgNVHRMEAjAAMAsGA1UdDwQEAwIE
+  sDAdBgNVHQ4EFgQUR8V72Z3+v+2P9abCnL4wjx32T+EwIwYDVR0RBBwwGoEYcnlh
+  bmQtcnVieUB6ZW5zcGlkZXIuY29tMCMGA1UdEgQcMBqBGHJ5YW5kLXJ1YnlAemVu
+  c3BpZGVyLmNvbTANBgkqhkiG9w0BAQUFAAOCAQEAIGzgp0aZ2W9+v96ujmBcQHoC
+  buy0iU68MVj2VlxMyfr1KPZIh1OyhU4UO4zrkREcH8ML70v9cYHNvOd9oynRHnvC
+  l2tj/fD3YJ0AEkJxGrYwRWQmvMfC4bJ02bC1+rVOUIXXKp3+cUmiN4sTniof8VFo
+  bo/YYP4c7erpERa+9hrqygg6WQbJlk2YRlH3JXPFjmu869i2dcbR5ZLOAeEy+axH
+  E4oJcnPkJAr0rw504JGtlZtONZQblwmRJOIdXzolaE3NRGUzGVOUSptZppAKiavY
+  fO6tdKQc/5RfA8oQEkg8hrxA5PQSz4TOFJGLpFvIapEk6tMruQ0bHgkhr9auXg==
   -----END CERTIFICATE-----
-date: 2016-01-21 00:00:00.000000000 Z
+date: 2016-11-30 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '5.8'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '5.8'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -65,14 +52,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '3.14'
+        version: '3.15'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '3.14'
+        version: '3.15'
 description: |-
   Oedipus Lex is a lexer generator in the same family as Rexical and
   Rex. Oedipus Lex is my independent lexer fork of Rexical. Rexical was