oedipus_lex 2.2.1 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f7fe436e528415763cca7067c30307dbff6e9793
-  data.tar.gz: e5ebf8486d367d3a5d98741eb5a06a6ac9db6cac
+  metadata.gz: e4d847d43454e2e7497b7737efea33c3ac3639d2
+  data.tar.gz: 94c532557bd692ed547f901b40667befe44fe39a
 SHA512:
-  metadata.gz: a0b1887372dbb69a6e74de3b5343f314583ff5c18b1c5fb338d6dccc5dda19ada9de8fb1b85d4750e919385b055f61e1e6cab75a0f903e1e4f1f230f26c40e1f
-  data.tar.gz: b4859b2af69385f7c78cb6f2fd3906d2cc4cd6d7dec65f0dbd1d48153b61240a97bbfd304fc144650685c718a83cf0584fecabb4ba8fdf26b862bf34e7852bbd
+  metadata.gz: 7d8ed1a67c28ca5ffafcf7d48abac036d6d3ccb88254bfd11e953d646f35b08a27e3c246f40317e13127961c6ca77f98a71b0eaa9aded873ae5d4f207176efe2
+  data.tar.gz: 473a2296045308dec98b4a9355d4f1dd89168f2fae68cdea7f70efa6c4249f630fccf4b3be02d96478f343e995cdd61f96ce766f523ccaa9d7f59123b93ab4e9

data/History.rdoc

@@ -1,3 +1,12 @@
+=== 2.3.0 / 2014-05-16
+
+* 4 minor enhancements:
+
+  * Added == method to lexer.
+  * Added pretty_print methods to lexer+rule+group.
+  * Added structural test so I can later make optimization changes cleanly.
+  * Refactored to (mostly) use an AST for code generation. Nukes 2 ERB templates.
+
 === 2.2.1 / 2014-04-02
 
 * 1 bug fix:

data/lib/oedipus_lex.rb

@@ -4,7 +4,7 @@ require "erb"
 require "oedipus_lex.rex"
 
 class OedipusLex
-  VERSION = "2.2.1"
+  VERSION = "2.3.0"
 
   attr_accessor :class_name
   attr_accessor :header
@@ -23,6 +23,109 @@ class OedipusLex
     :stub     => false,
   }
 
+  class Rule < Struct.new :start_state, :regexp, :action
+    attr_accessor :group
+    alias :group? :group
+
+    def self.[] start, regexp, action
+      new start, regexp.inspect, action
+    end
+
+    def initialize start_state, regexp, action
+      super
+      self.group = nil
+    end
+
+    undef_method :to_a
+
+    def to_ruby state, predicates, exclusive
+      return unless group? or
+        start_state == state or
+        (state.nil? and predicates.include? start_state)
+
+      cond =
+        if exclusive or not start_state then
+          "when text = ss.scan(#{regexp}) then"
+        elsif start_state =~ /^:/ then
+          "when (state == #{start_state}) && (text = ss.scan(#{regexp})) then"
+        else
+          "when #{start_state} && (text = ss.scan(#{regexp})) then"
+        end
+
+      body =
+        case action
+        when nil, false then
+          "  # do nothing"
+        when /^\{/ then
+          "  action #{action}"
+        when /^:/, "nil" then
+          "  [:state, #{action}]"
+        else
+          "  #{action} text"
+        end
+
+      [cond, body]
+    end
+
+    def pretty_print pp
+      pp.text "Rule"
+      pp.group 2, "[", "]" do
+        pp.pp start_state
+        pp.text ", "
+        pp.text regexp
+        pp.text ", "
+        pp.send(action ? :text : :pp, action)
+      end
+    end
+  end
+
+  class Group < Struct.new :regex, :rules
+    alias :start_state :regex
+
+    def self.[] start, *subrules
+      r = new start.inspect
+      r.rules.concat subrules
+      r
+    end
+
+    def initialize start
+      super(start, [])
+    end
+
+    def << rule
+      rules << rule
+      nil
+    end
+
+    def to_ruby state, predicates, exclusive
+      [
+       "when ss.check(#{regex}) then",
+       "  case",
+       rules.map { |subrule|
+         s = subrule.to_ruby(state, predicates, exclusive)
+         s && s.join("\n").gsub(/^/, "  ")
+       }.compact,
+       "  end # group #{regex}"
+      ]
+    end
+
+    def pretty_print pp
+      pp.text "Group"
+      pp.group 2, "[", "]" do
+        pp.seplist([regex] + rules, lambda { pp.comma_breakable }, :each) { |v|
+          pp.send(String === v ? :text : :pp, v)
+        }
+      end
+    end
+  end
+
+  def self.[](name, *rules)
+    r = new
+    r.class_name = name
+    r.rules.concat rules
+    r
+  end
+
   def initialize opts = {}
     self.option     = DEFAULTS.merge opts
     self.class_name = nil
@@ -36,6 +139,26 @@ class OedipusLex
     self.group   = nil
   end
 
+  def == o
+    (o.class      == self.class      and
+     o.class_name == self.class_name and
+     o.header     == self.header     and
+     o.ends       == self.ends       and
+     o.inners     == self.inners     and
+     o.macros     == self.macros     and
+     o.rules      == self.rules      and
+     o.starts     == self.starts)
+  end
+
+  def pretty_print pp
+    commas = lambda { pp.comma_breakable }
+
+    pp.text "Lexer"
+    pp.group 2, "[", "]" do
+      pp.seplist([class_name] + rules, commas, :each) { |v| pp.pp v }
+    end
+  end
+
   def lex_class prefix, name
     header.concat prefix.split(/\n/)
     self.class_name = name
@@ -66,17 +189,19 @@ class OedipusLex
   end
 
   def lex_rule start_state, regexp, action = nil
-    rules << [start_state, regexp, action]
+    rules << Rule.new(start_state, regexp, action)
   end
 
-  def lex_grouphead(*vals)
+  def lex_grouphead re
     end_group if group
     self.state = :group
-    self.group = vals
+    self.group = Group.new re
   end
 
   def lex_group start_state, regexp, action = nil
-    self.group << [start_state, regexp, action]
+    rule = Rule.new(start_state, regexp, action)
+    rule.group = group
+    self.group << rule
   end
 
   def end_group
@@ -96,7 +221,7 @@ class OedipusLex
   end
 
   def generate
-    filter = lambda { |r| Array === r.last ? nil : r.first }
+    filter = lambda { |r| Rule === r && r.start_state || nil }
     _mystates = rules.map(&filter).flatten.compact.uniq
     exclusives, inclusives = _mystates.partition { |s| s =~ /^:[A-Z]/ }
 
@@ -110,38 +235,7 @@ class OedipusLex
     ERB.new(TEMPLATE, nil, "%").result binding
   end
 
-  rule = <<-'END_RULE'.chomp
-%         start_state, rule_expr, rule_action = *rule
-%         if start_state == state or (state.nil? and predicates.include? start_state) then
-%           if start_state and not exclusive then
-%             if start_state =~ /^:/ then
-                when (state == <%= start_state %>) && (text = ss.scan(<%= rule_expr %>)) then
-%             else
-                when <%= start_state %> && (text = ss.scan(<%= rule_expr %>)) then
-%             end
-%           else
-                when text = ss.scan(<%= rule_expr %>) then
-%           end
-%           if rule_action then
-%             case rule_action
-%             when /^\{/ then
-                  action <%= rule_action %>
-%             when /^:/, "nil" then
-                  [:state, <%= rule_action %>]
-%             else
-                  <%= rule_action %> text
-%             end
-%           else
-                  # do nothing
-%           end
-%         end # start_state == state
-  END_RULE
-
-  subrule = rule.gsub(/^ /, "   ").
-    sub(/\*rule/, "*subrule"). # use subrules, not top level rules
-    sub(/start_state == state or/, "true or") # hack to prevent state checks
-
-  TEMPLATE = <<-'REX'.sub(/SUBRULE/, subrule).sub(/RULE/, rule).gsub(/^ {6}/, '\1')
+  TEMPLATE = <<-'REX'.gsub(/^ {6}/, '')
       #--
       # This file is automatically generated. Do not modify it.
       # Generated by: oedipus_lex version <%= VERSION %>.
@@ -233,19 +327,8 @@ class OedipusLex
               when <%= the_states.map { |s| s || "nil" }.join ", " %> then
                 case
 %   the_states.each do |state|
-%     rules.each do |rule|
-%       if Array === rule.last then
-%         group_re, *subrules = rule
-                when ss.check(<%= group_re %>) then
-                  case
-%         subrules.each do |subrule|
-SUBRULE
-%         end
-                  end # group <%= group_re %>
-%       else # rule or group
-RULE
-%       end # rule or group
-%     end # rules.each
+%     lines = rules.map { |r| r.to_ruby state, predicates, exclusive }.compact
+<%= lines.join("\n").gsub(/^/, " " * 10) %>
 %   end # the_states.each
                 else
                   text = ss.string[ss.pos .. -1]

data/test/test_oedipus_lex.rb

@@ -468,6 +468,42 @@ class TestOedipusLex < Minitest::Test
     assert_match "end # group /\\d/", ruby
   end
 
+  def test_scanner_inspect_slash_structure
+    src = <<-'REX'
+      class Calculator
+      rules
+
+        : /\d/
+        |     /\d+\.\d+/  { [:float, text.to_f] }
+        |     /\d+/       { [:int, text.to_i] }
+        : /\+/
+        | xx? /\+whatever/  { [:x, text] }
+        | :x  /\+\d+/       { [:y, text] }
+        | :x  /\+\w+/       { [:z, text] }
+              /\s+/
+      end
+    REX
+
+    rex = OedipusLex.new option
+    rex.parse cleanup src
+
+    lex = OedipusLex
+    group, rule = lex::Group, lex::Rule
+    expected = lex["Calculator",
+                   group[/\d/,
+                         rule[nil, /\d+\.\d+/, "{ [:float, text.to_f] }"],
+                         rule[nil, /\d+/, "{ [:int, text.to_i] }"]],
+                   group[/\+/,
+                         rule["xx?", /\+whatever/, "{ [:x, text] }"],
+                         rule[":x", /\+\d+/, "{ [:y, text] }"],
+                         rule[":x", /\+\w+/, "{ [:z, text] }"]],
+                   rule[nil, /\s+/, nil]]
+
+    assert_equal expected, rex
+  end
+
+  make_my_diffs_pretty!
+
   def test_generator_start
     src = <<-'REX'
       class Calculator

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: oedipus_lex
 version: !ruby/object:Gem::Version
-  version: 2.2.1
+  version: 2.3.0
 platform: ruby
 authors:
 - Ryan Davis
@@ -29,7 +29,7 @@ cert_chain:
   Y4evBVezr3SjXz08vPqRO5YRdO3zfeMT8gBjRqZjWJGMZ2lD4XNfrs7eky74CyZw
   xx3n58i0lQkBE1EpKE0lFu/y
   -----END CERTIFICATE-----
-date: 2014-04-02 00:00:00.000000000 Z
+date: 2014-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -65,14 +65,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '3.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '3.12'
 description: |-
   Oedipus Lex is a lexer generator in the same family as Rexical and
   Rex. Oedipus Lex is my independent lexer fork of Rexical. Rexical was
@@ -155,7 +155,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: oedipus_lex
+rubyforge_project: 
 rubygems_version: 2.2.1
 signing_key: 
 specification_version: 4