oedipus_lex 2.5.0 → 2.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 61879597d87d6ef9e59b495a38e10617e03f3277
-  data.tar.gz: e5e415035c15bc8c02007a3479f0bbd2fd0d8621
+SHA256:
+  metadata.gz: 8cb423b537f9675d2c16f2dea13fe319f1a9201164e8eeefa9a4275640220607
+  data.tar.gz: b92269a7b11988954a9457a36df57c7199f1f95f74d862fbc20e7d6aec770ad6
 SHA512:
-  metadata.gz: dd4aea54ccd7eb1731bdc5639d4932fe1eadb5b188cf9a79a75971599adeadbfbc867b5790fcb09cf8bdad68f09f715154852a251953b0aa37a66f142c6bc423
-  data.tar.gz: 0beb1c01b05a3e567bc0c0702c2c675096a96e0801604e89d28ef7197e7e02fd154a5c1facff60447d068211d2013e04a67e8be260bdcfb2420222a072ff4e97
+  metadata.gz: 9bd0f74cb429fd8f92b202a1dbaa38f4a69eab05da150801f59f36c2e7476b2d494304d17e8993ecbd563692dce0e7c15f20dfee5f4b0d6c92c5abc72d96b4b8
+  data.tar.gz: ba7831924e35607b5db556c42e7d0469c8f71d8c87feb67322b05599cfe83d8884f4559e5f068570dc7e36a1b8dcc92c17b2e148685742a45a199b04caf50bdc

data.tar.gz.sig

@@ -1,2 +1,2 @@
-�+��|@�+�7q�N�GG����l���	`x�j1\���փ/"|�&v�����U���p�ч��i�ue�S�Z���k"�I�����k�"\�ǝ)����
-v/����NQ�ʑhQ<�<��q�z�ڣV���q"�t��ǰ�f:��#wK��x�����xl��;#IN-�koy�
+�Lr�o��s�����_�d_ɢ��x��L�P��#�_�:�*���F�6W�bV�yi��N�=|Wzd)U��ϫ�s[b��3�]��4���U�7�=�;��%�����Lױ��t�KO�_m`hqa{�0vG]1Q %Uї�*N	�Z��hk'	~wy�M�/m�8\��e@ �D�W�T(�V�O\R����Eq�b8�T0�g�0m��@���`z�'/���sGg�^M\��q�)�����E���
����>�
+Q�

data/History.rdoc

@@ -1,3 +1,13 @@
+=== 2.5.1 / 2019-06-03
+
+* 1 minor enhancement:
+
+  * Added full rdoc an re-bootstrapped.
+
+* 1 bug fix:
+
+  * Fixed a deprecation warning in ruby 2.6+.
+
 === 2.5.0 / 2016-11-30
 
 * 5 minor enhancements:

data/lib/oedipus_lex.rb

@@ -3,20 +3,85 @@ require 'strscan'
 require "erb"
 require "oedipus_lex.rex"
 
+##
+# Oedipus Lex is a lexer generator in the same family as Rexical and
+# Rex. Oedipus Lex is my independent lexer fork of Rexical. Rexical
+# was in turn a fork of Rex. We've been unable to contact the author
+# of rex in order to take it over, fix it up, extend it, and relicense
+# it to MIT. So, Oedipus was written clean-room in order to bypass
+# licensing constraints (and because bootstrapping is fun).
+#
+# Oedipus brings a lot of extras to the table and at this point is
+# only historically related to rexical. The syntax has changed enough
+# that any rexical lexer will have to be tweaked to work inside of
+# oedipus. At the very least, you need to add slashes to all your
+# regexps.
+#
+# Oedipus, like rexical, is based primarily on generating code much
+# like you would a hand-written lexer. It is _not_ a table or hash
+# driven lexer. It uses StrScanner within a multi-level case
+# statement. As such, Oedipus matches on the _first_ match, not the
+# longest (like lex and its ilk).
+#
+# This documentation is not meant to bypass any prerequisite knowledge
+# on lexing or parsing. If you'd like to study the subject in further
+# detail, please try [TIN321] or the [LLVM Tutorial] or some other
+# good resource for CS learning. Books... books are good. I like
+# books.
+
 class OedipusLex
-  VERSION = "2.5.0"
+  VERSION = "2.5.1" # :nodoc:
+
+  ##
+  # The class name to generate.
 
   attr_accessor :class_name
+
+  ##
+  # An array of header lines to have before the lexer class.
+
   attr_accessor :header
+
+  ##
+  # An array of lines to have after the lexer class.
+
   attr_accessor :ends
+
+  ##
+  # An array of lines to have inside (but at the bottom of) the lexer
+  # class.
+
   attr_accessor :inners
+
+  ##
+  # An array of name/regexp pairs to generate constants inside the
+  # lexer class.
+
   attr_accessor :macros
+
+  ##
+  # A hash of options for the code generator. See README.rdoc for
+  # supported options.
+
   attr_accessor :option
+
+  ##
+  # The rules for the lexer.
+
   attr_accessor :rules
+
+  ##
+  # An array of lines of code to generate into the top of the lexer
+  # (next_token) loop.
+
   attr_accessor :starts
+
+  ##
+  # An array of all the groups within the lexer rules.
+
   attr_accessor :group
 
-  DEFAULTS = {
+  DEFAULTS = { # :nodoc:
     :debug    => false,
     :do_parse => false,
     :lineno   => false,
@@ -24,21 +89,39 @@ class OedipusLex
     :stub     => false,
   }
 
+  ##
+  # A Rule represents the main component of Oedipus Lex. These are the
+  # things that "get stuff done" at the lexical level. They consist of:
+  #
+  # + an optional required start state symbol or predicate method name
+  # + a regexp to match on
+  # + an optional action method or block
+
   class Rule < Struct.new :start_state, :regexp, :action
+    ##
+    # What group this rule is in, if any.
+
     attr_accessor :group
-    alias :group? :group
+
+    alias :group? :group # :nodoc:
+
+    ##
+    # A simple constructor
 
     def self.[] start, regexp, action
       new start, regexp.inspect, action
     end
 
-    def initialize start_state, regexp, action
+    def initialize start_state, regexp, action # :nodoc:
       super
       self.group = nil
     end
 
     undef_method :to_a
 
+    ##
+    # Generate equivalent ruby code for the rule.
+
     def to_ruby state, predicates, exclusive
       return unless group? or
         start_state == state or
@@ -73,7 +156,7 @@ class OedipusLex
       ["when #{cond} then", body]
     end
 
-    def pretty_print pp
+    def pretty_print pp # :nodoc:
       pp.text "Rule"
       pp.group 2, "[", "]" do
         pp.pp start_state
@@ -85,25 +168,37 @@ class OedipusLex
     end
   end
 
+  ##
+  # A group allows you to group up multiple rules under a single
+  # regular prefix expression, allowing optimized code to be generated
+  # that skips over all actions if the prefix isn't matched.
+
   class Group < Struct.new :regex, :rules
     alias :start_state :regex
 
+    ##
+    # A convenience method to create a new group with a +start+ and
+    # given +subrules+.
+
     def self.[] start, *subrules
       r = new start.inspect
       r.rules.concat subrules
       r
     end
 
-    def initialize start
+    def initialize start # :nodoc:
       super(start, [])
     end
 
+    ##
+    # Add a rule to this group.
+
     def << rule
       rules << rule
       nil
     end
 
-    def to_ruby state, predicates, exclusive
+    def to_ruby state, predicates, exclusive # :nodoc:
       [
        "when ss.match?(#{regex}) then",
        "  case",
@@ -115,7 +210,7 @@ class OedipusLex
       ]
     end
 
-    def pretty_print pp
+    def pretty_print pp # :nodoc:
       pp.text "Group"
       pp.group 2, "[", "]" do
         pp.seplist([regex] + rules, lambda { pp.comma_breakable }, :each) { |v|
@@ -125,6 +220,10 @@ class OedipusLex
     end
   end
 
+  ##
+  # A convenience method to create a new lexer with a +name+ and given
+  # +rules+.
+
   def self.[](name, *rules)
     r = new
     r.class_name = name
@@ -132,7 +231,7 @@ class OedipusLex
     r
   end
 
-  def initialize opts = {}
+  def initialize opts = {} # :nodoc:
     self.option     = DEFAULTS.merge opts
     self.class_name = nil
 
@@ -145,7 +244,7 @@ class OedipusLex
     self.group   = nil
   end
 
-  def == o
+  def == o # :nodoc:
     (o.class      == self.class      and
      o.class_name == self.class_name and
      o.header     == self.header     and
@@ -156,7 +255,7 @@ class OedipusLex
      o.starts     == self.starts)
   end
 
-  def pretty_print pp
+  def pretty_print pp # :nodoc:
     commas = lambda { pp.comma_breakable }
 
     pp.text "Lexer"
@@ -165,67 +264,109 @@ class OedipusLex
     end
   end
 
+  ##
+  # Process a +class+ lexeme.
+
   def lex_class prefix, name
     header.concat prefix.split(/\n/)
     self.class_name = name
   end
 
+  ##
+  # Process a +comment+ lexeme.
+
   def lex_comment line
     # do nothing
   end
 
+  ##
+  # Process an +end+ lexeme.
+
   def lex_end line
     ends << line
   end
 
+  ##
+  # Process an +inner+ lexeme.
+
   def lex_inner line
     inners << line
   end
 
+  ##
+  # Process a +start+ lexeme.
+
   def lex_start line
     starts << line.strip
   end
 
+  ##
+  # Process a +macro+ lexeme.
+
   def lex_macro name, value
     macros << [name, value]
   end
 
+  ##
+  # Process an +option+ lexeme.
+
   def lex_option option
     self.option[option.to_sym] = true
   end
 
+  ##
+  # Process a +X+ lexeme.
+
   def lex_rule start_state, regexp, action = nil
     rules << Rule.new(start_state, regexp, action)
   end
 
+  ##
+  # Process a +group head+ lexeme.
+
   def lex_grouphead re
     end_group if group
     self.state = :group
     self.group = Group.new re
   end
 
+  ##
+  # Process a +group+ lexeme.
+
   def lex_group start_state, regexp, action = nil
     rule = Rule.new(start_state, regexp, action)
     rule.group = group
     self.group << rule
   end
 
+  ##
+  # End a group.
+
   def end_group
     rules << group
     self.group = nil
     self.state = :rule
   end
 
+  ##
+  # Process the end of a +group+ lexeme.
+
   def lex_groupend start_state, regexp, action = nil
     end_group
     lex_rule start_state, regexp, action
   end
 
+  ##
+  # Process a +state+ lexeme.
+
   def lex_state new_state
     end_group if group
     # do nothing -- lexer switches state for us
   end
 
+  ##
+  # Generate the lexer.
+
   def generate
     filter = lambda { |r| Rule === r && r.start_state || nil }
     _mystates = rules.map(&filter).flatten.compact.uniq
@@ -241,9 +382,17 @@ class OedipusLex
     encoding = header.shift if header.first =~ /encoding:/
     encoding ||= "# encoding: UTF-8"
 
-    ERB.new(TEMPLATE, nil, "%").result binding
+    erb = if RUBY_VERSION >= "2.6.0" then
+            ERB.new(TEMPLATE, trim_mode:"%")
+          else
+            ERB.new(TEMPLATE, nil, "%")
+          end
+
+    erb.result binding
   end
 
+  # :stopdoc:
+
   TEMPLATE = <<-'REX'.gsub(/^ {6}/, '')
       <%= encoding %>
       #--
@@ -260,41 +409,74 @@ class OedipusLex
 %   end
 
 % end
+
+      ##
+      # The generated lexer <%= class_name %>
+
       class <%= class_name %>
         require 'strscan'
 
 % unless macros.empty? then
+        # :stopdoc:
 %   max = macros.map { |(k,_)| k.size }.max
 %   macros.each do |(k,v)|
         <%= "%-#{max}s = %s" % [k, v] %>
 %   end
-
+        # :startdoc:
 % end
+        # :stopdoc:
         class LexerError < StandardError ; end
         class ScanError < LexerError ; end
+        # :startdoc:
 
 % if option[:lineno] then
+        ##
+        # The current line number.
+
         attr_accessor :lineno
 % end
+        ##
+        # The file name / path
+
         attr_accessor :filename
+
+        ##
+        # The StringScanner for this lexer.
+
         attr_accessor :ss
+
+        ##
+        # The current lexical state.
+
         attr_accessor :state
 
         alias :match :ss
 
+        ##
+        # The match groups for the current scan.
+
         def matches
           m = (1..9).map { |i| ss[i] }
           m.pop until m[-1] or m.empty?
           m
         end
 
+        ##
+        # Yields on the current action.
+
         def action
           yield
         end
 
 % if option[:column] then
+        ##
+        # The previous position. Only available if the :column option is on.
+
         attr_accessor :old_pos
 
+        ##
+        # The current column. Only available if the :column option is on.
+
         def column
           idx = ss.string.rindex("\n", old_pos) || -1
           old_pos - idx - 1
@@ -302,6 +484,9 @@ class OedipusLex
 
 % end
 % if option[:do_parse] then
+        ##
+        # Parse the file by getting all tokens and calling lex_+type+ on them.
+
         def do_parse
           while token = next_token do
             type, *vals = token
@@ -311,10 +496,17 @@ class OedipusLex
         end
 
 % end
+
+        ##
+        # The current scanner class. Must be overridden in subclasses.
+
         def scanner_class
           StringScanner
         end unless instance_methods(false).map(&:to_s).include?("scanner_class")
 
+        ##
+        # Parse the given string.
+
         def parse str
           self.ss     = scanner_class.new str
 % if option[:lineno] then
@@ -325,6 +517,9 @@ class OedipusLex
           do_parse
         end
 
+        ##
+        # Read in and parse the file at +path+.
+
         def parse_file path
           self.filename = path
           open path do |f|
@@ -332,6 +527,9 @@ class OedipusLex
           end
         end
 
+        ##
+        # The current location in the parse.
+
         def location
           [
             (filename || "<input>"),
@@ -346,6 +544,9 @@ class OedipusLex
           ].compact.join(":")
         end
 
+        ##
+        # Lex the next token.
+
         def next_token
 % starts.each do |s|
           <%= s %>
@@ -427,6 +628,8 @@ class OedipusLex
       end
 % end
   REX
+
+  # :startdoc:
 end
 
 if $0 == __FILE__ then

data/lib/oedipus_lex.rex.rb

@@ -1,44 +1,80 @@
 # encoding: UTF-8
 #--
 # This file is automatically generated. Do not modify it.
-# Generated by: oedipus_lex version 2.4.1.
+# Generated by: oedipus_lex version 2.5.0.
 # Source: lib/oedipus_lex.rex
 #++
 
+
+##
+# The generated lexer OedipusLex
+
 class OedipusLex
   require 'strscan'
 
+  # :stopdoc:
   ST  = /(?:(:\S+|\w+\??))/
   RE  = /(\/(?:\\.|[^\/])+\/[ion]?)/
   ACT = /(\{.*|:?\w+)/
-
+  # :startdoc:
+  # :stopdoc:
   class LexerError < StandardError ; end
   class ScanError < LexerError ; end
+  # :startdoc:
+
+  ##
+  # The current line number.
 
   attr_accessor :lineno
+  ##
+  # The file name / path
+
   attr_accessor :filename
+
+  ##
+  # The StringScanner for this lexer.
+
   attr_accessor :ss
+
+  ##
+  # The current lexical state.
+
   attr_accessor :state
 
   alias :match :ss
 
+  ##
+  # The match groups for the current scan.
+
   def matches
     m = (1..9).map { |i| ss[i] }
     m.pop until m[-1] or m.empty?
     m
   end
 
+  ##
+  # Yields on the current action.
+
   def action
     yield
   end
 
+  ##
+  # The previous position. Only available if the :column option is on.
+
   attr_accessor :old_pos
 
+  ##
+  # The current column. Only available if the :column option is on.
+
   def column
     idx = ss.string.rindex("\n", old_pos) || -1
     old_pos - idx - 1
   end
 
+  ##
+  # Parse the file by getting all tokens and calling lex_+type+ on them.
+
   def do_parse
     while token = next_token do
       type, *vals = token
@@ -47,10 +83,17 @@ class OedipusLex
     end
   end
 
+
+  ##
+  # The current scanner class. Must be overridden in subclasses.
+
   def scanner_class
     StringScanner
   end unless instance_methods(false).map(&:to_s).include?("scanner_class")
 
+  ##
+  # Parse the given string.
+
   def parse str
     self.ss     = scanner_class.new str
     self.lineno = 1
@@ -59,6 +102,9 @@ class OedipusLex
     do_parse
   end
 
+  ##
+  # Read in and parse the file at +path+.
+
   def parse_file path
     self.filename = path
     open path do |f|
@@ -66,6 +112,9 @@ class OedipusLex
     end
   end
 
+  ##
+  # The current location in the parse.
+
   def location
     [
       (filename || "<input>"),
@@ -74,6 +123,9 @@ class OedipusLex
     ].compact.join(":")
   end
 
+  ##
+  # Lex the next token.
+
   def next_token
 
     token = nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: oedipus_lex
 version: !ruby/object:Gem::Version
-  version: 2.5.0
+  version: 2.5.1
 platform: ruby
 authors:
 - Ryan Davis
@@ -10,9 +10,9 @@ bindir: bin
 cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
-  MIIDijCCAnKgAwIBAgIBATANBgkqhkiG9w0BAQUFADBFMRMwEQYDVQQDDApyeWFu
+  MIIDPjCCAiagAwIBAgIBAzANBgkqhkiG9w0BAQsFADBFMRMwEQYDVQQDDApyeWFu
   ZC1ydWJ5MRkwFwYKCZImiZPyLGQBGRYJemVuc3BpZGVyMRMwEQYKCZImiZPyLGQB
-  GRYDY29tMB4XDTE2MDkyNjAxNTczNVoXDTE3MDkyNjAxNTczNVowRTETMBEGA1UE
+  GRYDY29tMB4XDTE4MTIwNDIxMzAxNFoXDTE5MTIwNDIxMzAxNFowRTETMBEGA1UE
   AwwKcnlhbmQtcnVieTEZMBcGCgmSJomT8ixkARkWCXplbnNwaWRlcjETMBEGCgmS
   JomT8ixkARkWA2NvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALda
   b9DCgK+627gPJkB6XfjZ1itoOQvpqH1EXScSaba9/S2VF22VYQbXU1xQXL/WzCkx
@@ -20,46 +20,51 @@ cert_chain:
   oOvjtt5P8+GSK9zLzxQP0gVLS/D0FmoE44XuDr3iQkVS2ujU5zZL84mMNqNB1znh
   GiadM9GHRaDiaxuX0cIUBj19T01mVE2iymf9I6bEsiayK/n6QujtyCbTWsAS9Rqt
   qhtV7HJxNKuPj/JFH0D2cswvzznE/a5FOYO68g+YCuFi5L8wZuuM8zzdwjrWHqSV
-  gBEfoTEGr7Zii72cx+sCAwEAAaOBhDCBgTAJBgNVHRMEAjAAMAsGA1UdDwQEAwIE
-  sDAdBgNVHQ4EFgQUR8V72Z3+v+2P9abCnL4wjx32T+EwIwYDVR0RBBwwGoEYcnlh
-  bmQtcnVieUB6ZW5zcGlkZXIuY29tMCMGA1UdEgQcMBqBGHJ5YW5kLXJ1YnlAemVu
-  c3BpZGVyLmNvbTANBgkqhkiG9w0BAQUFAAOCAQEAIGzgp0aZ2W9+v96ujmBcQHoC
-  buy0iU68MVj2VlxMyfr1KPZIh1OyhU4UO4zrkREcH8ML70v9cYHNvOd9oynRHnvC
-  l2tj/fD3YJ0AEkJxGrYwRWQmvMfC4bJ02bC1+rVOUIXXKp3+cUmiN4sTniof8VFo
-  bo/YYP4c7erpERa+9hrqygg6WQbJlk2YRlH3JXPFjmu869i2dcbR5ZLOAeEy+axH
-  E4oJcnPkJAr0rw504JGtlZtONZQblwmRJOIdXzolaE3NRGUzGVOUSptZppAKiavY
-  fO6tdKQc/5RfA8oQEkg8hrxA5PQSz4TOFJGLpFvIapEk6tMruQ0bHgkhr9auXg==
+  gBEfoTEGr7Zii72cx+sCAwEAAaM5MDcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAw
+  HQYDVR0OBBYEFEfFe9md/r/tj/Wmwpy+MI8d9k/hMA0GCSqGSIb3DQEBCwUAA4IB
+  AQCbJwLmpJR2PomLU+Zzw3KRzH/hbyUWc/ftru71AopZ1fy4iY9J/BW5QYKVYwbP
+  V0FSBWtvfI/RdwfKGtuGhPKECZgmLieGuZ3XCc09qPu1bdg7i/tu1p0t0c6163ku
+  nDMDIC/t/DAFK0TY9I3HswuyZGbLW7rgF0DmiuZdN/RPhHq2pOLMLXJmFclCb/im
+  9yToml/06TJdUJ5p64mkBs0TzaK66DIB1Smd3PdtfZqoRV+EwaXMdx0Hb3zdR1JR
+  Em82dBUFsipwMLCYj39kcyHWAxyl6Ae1Cn9r/ItVBCxoeFdrHjfavnrIEoXUt4bU
+  UfBugfLD19bu3nvL+zTAGx/U
   -----END CERTIFICATE-----
-date: 2016-11-30 00:00:00.000000000 Z
+date: 2019-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
 - !ruby/object:Gem::Dependency
   name: hoe
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.15'
+        version: '3.17'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.15'
+        version: '3.17'
 description: |-
   Oedipus Lex is a lexer generator in the same family as Rexical and
   Rex. Oedipus Lex is my independent lexer fork of Rexical. Rexical was
@@ -93,7 +98,7 @@ extra_rdoc_files:
 - README.rdoc
 - sample/error1.txt
 files:
-- .autotest
+- ".autotest"
 - History.rdoc
 - Manifest.txt
 - README.rdoc
@@ -126,23 +131,22 @@ licenses:
 metadata: {}
 post_install_message: 
 rdoc_options:
-- --main
+- "--main"
 - README.rdoc
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 3.0.2
 signing_key: 
 specification_version: 4
 summary: Oedipus Lex is a lexer generator in the same family as Rexical and Rex