oedipus_lex 2.1.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 16d5bf4e2e41af8957fcfee63012c44ee33f65df
-  data.tar.gz: 55ec9b8e51834e8bee5f1ae27d6d4a1276c53094
+  metadata.gz: d0039bfa4502e60d46539650a3671e8efb4e6131
+  data.tar.gz: 5ce6636d84be656a55937f9c14dfc09ab1435082
 SHA512:
-  metadata.gz: d78c2765d033065c1be87a0a18860bec9d0f7ce359218ad88db34cd5381e70ff34435e489cdcb6fa2a6091b115f16524333dda547fcb712fac36c1192d8b8553
-  data.tar.gz: 62cc88bdb0ee37be1d38aaca8a7ad7979839d6a9602d1797f25361873a40994d394c8d44e0d0fe15d255916cb5e716d829a410161783a8e4283b90a16bb8153b
+  metadata.gz: 62803c0e04df04609964b62541026fd385dcfd8b8a77093b3abdd85c8f227f7bf49629709d62010a5e42c06bdbd50add8133004c7a548af073b0a03fa10cdb87
+  data.tar.gz: 01bcb903f4722cd8e6e77d3a119ce52982083fdcec796338afe6a7ca93b807b5ff6829c2884e49bd104ed397e9e5244cfb75911abcaff026d342ddbd50a7705c

checksums.yaml.gz.sig

@@ -1,2 +1,2 @@
-���.�n�7hZ.�A�CK��/����^�<H�-,m���HU!�����1>��Zc���uֈ�P��d/��a��!m	�dO�~Z�H�d%�G��2�&.���-�w��M�"j�

-ˢ��cq��پ�>���"���O�A�W�]p��t���{e��p�q����Iq�EK�6���S#o/6:C*A�/�ћ�ʈ�����<�$O�qן��|U�u�^���E�{���[<
+k�Q_]~�j���$�
+
�eX��z�Y�3�_������A:��PO���z)\163A=�+����k}�ON�'=��&��S����t;3]��������n� �uON1�aT-w�I�a>�>�.W7�{��|R�01��JD���P�l�~ʤ�w�ƾa�:�G	�87��ޔ���|��o�;��6��s���D��΁��x����R\Z����������2|;����|'�I�yDŽ��Դ�"�h��2m��

data/History.rdoc

@@ -1,3 +1,11 @@
+=== 2.2.0 / 2014-03-14
+
+* 3 minor enhancements:
+
+  * Added debug rake task.
+  * Added rule grouping. Naive benchmarking seems to show ~15% improvement in ruby_parser.
+  * Refactored rule handling part of template to its own variable.
+
 === 2.1.0 / 2014-01-22
 
 * 3 minor enhancements:

data/README.rdoc

@@ -59,8 +59,9 @@ resource for CS learning. Books... books are good. I like books.
   name            = /\w+/
   regexp          = /(\/(?:\\.|[^\/])+\/[io]?)/
 
-  rule_section    = /rules?/ NL (rule)*
+  rule_section    = /rules?/ NL (rule|group)*
   rule            = (state)? regexp (action)?
+  group           = /:/ regexp NL (rule)+
   state           = label
                   | predicate
   label           = /:\w+/

data/Rakefile

@@ -69,4 +69,13 @@ task :clean do
   rm Dir["sample/*.rb"]
 end
 
+task :debug do
+  require "oedipus_lex"
+  f = ENV["F"]
+  rex = OedipusLex.new $rex_option
+  rex.parse_file f
+
+  puts rex.generate
+end
+
 # vim: syntax=ruby

data/lib/oedipus_lex.rb

@@ -4,7 +4,7 @@ require "erb"
 require "oedipus_lex.rex"
 
 class OedipusLex
-  VERSION = "2.1.0"
+  VERSION = "2.2.0"
 
   attr_accessor :class_name
   attr_accessor :header
@@ -14,6 +14,7 @@ class OedipusLex
   attr_accessor :option
   attr_accessor :rules
   attr_accessor :starts
+  attr_accessor :group
 
   DEFAULTS = {
     :debug    => false,
@@ -32,6 +33,7 @@ class OedipusLex
     self.macros  = []
     self.rules   = []
     self.starts  = []
+    self.group   = nil
   end
 
   def lex_class prefix, name
@@ -67,30 +69,79 @@ class OedipusLex
     rules << [start_state, regexp, action]
   end
 
-  def lex_rule2(*vals)
-    raise vals.inspect
+  def lex_grouphead(*vals)
+    end_group if group
+    self.state = :group
+    self.group = vals
+  end
+
+  def lex_group start_state, regexp, action = nil
+    self.group << [start_state, regexp, action]
+  end
+
+  def end_group
+    rules << group
+    self.group = nil
+    self.state = :rule
+  end
+
+  def lex_groupend start_state, regexp, action = nil
+    end_group
+    lex_rule start_state, regexp, action
   end
 
   def lex_state new_state
+    end_group if group
     # do nothing -- lexer switches state for us
   end
 
   def generate
-    states                 = rules.map(&:first).compact.uniq
-    exclusives, inclusives = states.partition { |s| s =~ /^:[A-Z]/ }
+    filter = lambda { |r| Array === r.last ? nil : r.first }
+    _mystates = rules.map(&filter).flatten.compact.uniq
+    exclusives, inclusives = _mystates.partition { |s| s =~ /^:[A-Z]/ }
 
     # NOTE: doubling up assignment to remove unused var warnings in
     # ERB binding.
 
     all_states =
-      all_states = [[nil,                        # non-state # eg [[nil,
-                     *inclusives],               # incls     #      :a, :b],
+      all_states = [[nil, *inclusives],          # nil+incls # eg [[nil, :a],
                     *exclusives.map { |s| [s] }] # [excls]   #     [:A], [:B]]
 
     ERB.new(TEMPLATE, nil, "%").result binding
   end
 
-  TEMPLATE = <<-'REX'.gsub(/^ {6}/, '\1')
+  rule = <<-'END_RULE'.chomp
+%         start_state, rule_expr, rule_action = *rule
+%         if start_state == state or (state.nil? and predicates.include? start_state) then
+%           if start_state and not exclusive then
+%             if start_state =~ /^:/ then
+                when (state == <%= start_state %>) && (text = ss.scan(<%= rule_expr %>)) then
+%             else
+                when <%= start_state %> && (text = ss.scan(<%= rule_expr %>)) then
+%             end
+%           else
+                when text = ss.scan(<%= rule_expr %>) then
+%           end
+%           if rule_action then
+%             case rule_action
+%             when /^\{/ then
+                  action <%= rule_action %>
+%             when /^:/, "nil" then
+                  [:state, <%= rule_action %>]
+%             else
+                  <%= rule_action %> text
+%             end
+%           else
+                  # do nothing
+%           end
+%         end # start_state == state
+  END_RULE
+
+  subrule = rule.gsub(/^ /, "   ").
+    sub(/\*rule/, "*subrule"). # use subrules, not top level rules
+    sub(/start_state == state or/, "true or") # hack to prevent state checks
+
+  TEMPLATE = <<-'REX'.sub(/SUBRULE/, subrule).sub(/RULE/, rule).gsub(/^ {6}/, '\1')
       #--
       # This file is automatically generated. Do not modify it.
       # Generated by: oedipus_lex version <%= VERSION %>.
@@ -178,36 +229,22 @@ class OedipusLex
               case state
 % all_states.each do |the_states|
 %   exclusive = the_states.first != nil
-%   all_states, predicates = the_states.partition { |s| s.nil? or s.start_with? ":" }
-%   filtered_states = the_states.select { |s| s.nil? or s.start_with? ":" }
-              when <%= all_states.map { |s| s || "nil" }.join ", " %> then
+%   the_states, predicates = the_states.partition { |s| s.nil? or s.start_with? ":" }
+              when <%= the_states.map { |s| s || "nil" }.join ", " %> then
                 case
-%   all_states.each do |state|
+%   the_states.each do |state|
 %     rules.each do |rule|
-%       start_state, rule_expr, rule_action = *rule
-%       if start_state == state or (state.nil? and predicates.include? start_state) then
-%         if start_state and not exclusive then
-%           if start_state =~ /^:/ then
-                when (state == <%= start_state %>) && (text = ss.scan(<%= rule_expr %>)) then
-%           else
-                when <%= start_state %> && (text = ss.scan(<%= rule_expr %>)) then
-%           end
-%         else
-                when text = ss.scan(<%= rule_expr %>) then
-%         end
-%         if rule_action then
-%           case rule_action
-%           when /^\{/ then
-                  action <%= rule_action %>
-%           when /^:/, "nil" then
-                  [:state, <%= rule_action %>]
-%           else
-                  <%= rule_action %> text
-%           end
-%         else
-                  # do nothing
+%       if Array === rule.last then
+%         group_re, *subrules = rule
+                when ss.check(<%= group_re %>) then
+                  case
+%         subrules.each do |subrule|
+SUBRULE
 %         end
-%       end # start_state == state
+                  end # group <%= group_re %>
+%       else # rule or group
+RULE
+%       end # rule or group
 %     end # rules.each
 %   end # the_states.each
                 else

data/lib/oedipus_lex.rex

@@ -1,16 +1,3 @@
-# [Header Part]
-# "class" Foo
-# ["option"
-#   [options] ]
-# ["inner"
-#   [methods] ]
-# ["macro"
-#   [macro-name  /pattern/[flags]] ]
-# "rule"
-#   [:state | method_name]  /pattern/[flags]  [{ code } | method_name | :state]
-# "end"
-# [Footer Part]
-
 class OedipusLex
 
 option
@@ -51,7 +38,11 @@ rule
 
   :macro        /\s+(\w+)\s+#{RE}/o     { [:macro, *matches] }
 
-  :rule         /\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o { [:rule, *matches] }
+  :rule         /\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o      { [:rule, *matches] }
+  :rule         /\s*:[\ \t]*#{RE}/o                         { [:grouphead, *matches] }
+  :group        /\s*:[\ \t]*#{RE}/o                         { [:grouphead, *matches] }
+  :group        /\s*\|\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o { [:group, *matches] }
+  :group        /\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o      { [:groupend, *matches] }
 
   :END          /\n+/                   # do nothing
   :END          /.*/                    { [:end, text] }

data/lib/oedipus_lex.rex.rb

@@ -1,22 +1,9 @@
 #--
 # This file is automatically generated. Do not modify it.
-# Generated by: oedipus_lex version 2.0.0.
+# Generated by: oedipus_lex version 2.1.0.
 # Source: lib/oedipus_lex.rex
 #++
 
-# [Header Part]
-# "class" Foo
-# ["option"
-#   [options] ]
-# ["inner"
-#   [methods] ]
-# ["macro"
-#   [macro-name  /pattern/[flags]] ]
-# "rule"
-#   [:state | method_name]  /pattern/[flags]  [{ code } | method_name | :state]
-# "end"
-# [Footer Part]
-
 class OedipusLex
   require 'strscan'
 
@@ -78,7 +65,7 @@ class OedipusLex
     until ss.eos? or token do
       token =
         case state
-        when nil, :option, :inner, :start, :macro, :rule then
+        when nil, :option, :inner, :start, :macro, :rule, :group then
           case
           when text = ss.scan(/options?.*/) then
             [:state, :option]
@@ -116,6 +103,14 @@ class OedipusLex
             action { [:macro, *matches] }
           when (state == :rule) && (text = ss.scan(/\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o)) then
             action { [:rule, *matches] }
+          when (state == :rule) && (text = ss.scan(/\s*:[\ \t]*#{RE}/o)) then
+            action { [:grouphead, *matches] }
+          when (state == :group) && (text = ss.scan(/\s*:[\ \t]*#{RE}/o)) then
+            action { [:grouphead, *matches] }
+          when (state == :group) && (text = ss.scan(/\s*\|\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o)) then
+            action { [:group, *matches] }
+          when (state == :group) && (text = ss.scan(/\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o)) then
+            action { [:groupend, *matches] }
           else
             text = ss.string[ss.pos .. -1]
             raise ScanError, "can not match (#{state.inspect}): '#{text}'"

data/test/test_oedipus_lex.rb

@@ -420,6 +420,54 @@ class TestOedipusLex < Minitest::Test
     assert_match "when :ARG then", ruby
   end
 
+  def test_simple_scanner_group
+    src = <<-'REX'
+      class Calculator
+      rules
+
+        : /\d/
+        |     /\d+\.\d+/  { [:float, text.to_f] }
+        |     /\d+/       { [:int, text.to_i] }
+              /\s+/
+      end
+    REX
+
+    ruby = generate_lexer src
+
+    assert_match "when ss.check(/\\d/) then", ruby
+    assert_match "when text = ss.scan(/\\d+\\.\\d+/) then", ruby
+    assert_match "when text = ss.scan(/\\d+/) then", ruby
+    assert_match "end # group /\\d/", ruby
+  end
+
+  def test_simple_scanner_group_I_am_dumb
+    src = <<-'REX'
+      class Calculator
+      rules
+
+        : /\d/
+        |     /\d+\.\d+/  { [:float, text.to_f] }
+        |     /\d+/       { [:int, text.to_i] }
+        : /\+/
+        | xx? /\+whatever/  { [:x, text] }
+        | :x  /\+\d+/       { [:y, text] }
+              /\s+/
+      end
+    REX
+
+    ruby = generate_lexer src
+
+    assert_match "when ss.check(/\\d/) then", ruby
+    assert_match "when text = ss.scan(/\\d+\\.\\d+/) then", ruby
+    assert_match "when text = ss.scan(/\\d+/) then", ruby
+    assert_match "end # group /\\d/", ruby
+
+    assert_match "when ss.check(/\\+/) then", ruby
+    assert_match "when xx? && (text = ss.scan(/\\+whatever/)) then", ruby
+    assert_match "when (state == :x) && (text = ss.scan(/\\+\\d+/)) then", ruby
+    assert_match "end # group /\\d/", ruby
+  end
+
   def test_generator_start
     src = <<-'REX'
       class Calculator

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: oedipus_lex
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Ryan Davis
@@ -29,7 +29,7 @@ cert_chain:
   Y4evBVezr3SjXz08vPqRO5YRdO3zfeMT8gBjRqZjWJGMZ2lD4XNfrs7eky74CyZw
   xx3n58i0lQkBE1EpKE0lFu/y
   -----END CERTIFICATE-----
-date: 2014-01-22 00:00:00.000000000 Z
+date: 2014-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -37,14 +37,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '5.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '5.3'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -65,14 +65,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.10'
 description: |-
   Oedipus Lex is a lexer generator in the same family as Rexical and
   Rex. Oedipus Lex is my independent lexer fork of Rexical. Rexical was
@@ -107,7 +107,6 @@ extra_rdoc_files:
 - sample/error1.txt
 files:
 - .autotest
-- .gemtest
 - History.rdoc
 - Manifest.txt
 - README.rdoc
@@ -134,6 +133,7 @@ files:
 - sample/xhtmlparser.rex
 - sample/xhtmlparser.xhtml
 - test/test_oedipus_lex.rb
+- .gemtest
 homepage: http://github.com/seattlerb/oedipus_lex
 licenses:
 - MIT
@@ -155,8 +155,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: oedipus_lex
-rubygems_version: 2.2.1
+rubyforge_project: 
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: Oedipus Lex is a lexer generator in the same family as Rexical and Rex