tdparser 1.5.0

data/manifest.scm

@@ -0,0 +1 @@
+(specifications->manifest (list "ruby@3.1" "ruby-rubocop"))

data/samples/sample1.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# -*- ruby -*-
+# adder-substractor
+
+require 'tdparser'
+
+class MyParser
+  include TDParser
+
+  def expr
+    (token(/\d+/) - (((token('+') | token('-')) - token(/\d+/)) * 0)) >> proc { |x|
+      n = x[0].to_i
+      x[1].inject(n) do |acc, i|
+        case i[0]
+        when '-'
+          acc - i[1].to_i
+        when '+'
+          acc + i[1].to_i
+        end
+      end
+    }
+  end
+
+  def parse(str)
+    tokens = str.split(%r{(?:\s+)|([+\-*/])}).reject { |x| x == '' }
+    expr.parse(tokens)
+  end
+end
+
+ENV.fetch('TEST', nil) and return
+
+parser = MyParser.new
+puts("1+10 = #{parser.parse('1+10')}")
+puts("2-1-20 = #{parser.parse('2 - 1 - 20')}")
+puts("1+2-3 = #{parser.parse('1 + 2 - 3')}")

data/samples/sample2.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# -*- ruby -*-
+# parsing four arithmetic expressions with tdputils.
+
+require 'tdparser'
+require 'tdparser/utils'
+
+class Sample2Parser
+  include TDParser
+
+  def expr1
+    (rule(:expr2) - (((token('+') | token('-')) - rule(:expr2)) * 0)) >> proc { |x|
+      x[1].inject(x[0]) do |n, y|
+        case y[0]
+        when '+'
+          n + y[1]
+        when '-'
+          n - y[1]
+        end
+      end
+    }
+  end
+
+  def expr2
+    (rule(:prim) - (((token('*') | token('/')) - rule(:prim)) * 0)) >> proc { |x|
+      x[1].inject(x[0]) do |n, y|
+        case y[0]
+        when '*'
+          n * y[1]
+        when '/'
+          n / y[1]
+        end
+      end
+    }
+  end
+
+  def prim
+    (token(:int) >> proc { |x| x[0].value.to_i }) |
+      ((token('(') - rule(:expr1) - token(')')) >> proc { |x| x[1] })
+  end
+
+  def parse(str)
+    tokenizer = StringTokenizer[
+      /\d+(?!\.\d)/ => :int,
+      /\d+\.\d+/ => :real,
+    ]
+    expr1.parse(tokenizer.generate(str))
+  end
+end
+
+ENV.fetch('TEST', nil) and return
+
+parser = Sample2Parser.new
+puts("1+10 = #{parser.parse('1+10')}")
+puts("2-1*20+18 = #{parser.parse('2 - 1 * 20 + 18')}")
+puts("2-(1-20) = #{parser.parse('2 - (1 - 20)')}")
+puts("1+2-3 = #{parser.parse('1 + 2 - 3')}")

data/samples/sample3.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# -*- ruby -*-
+# parsing four arithmetic expressions with tdputils.
+
+require 'tdparser'
+require 'tdparser/utils'
+
+class Sample3Parser
+  include TDParser
+
+  def expr1
+    n = nil
+    ((rule(:expr2) >> proc { |x| n = x[0] }) -
+      ((((token('+') | token('-')) - rule(:expr2)) >> proc { |x|
+        case x[0]
+        when '+'
+          n += x[1]
+        when '-'
+          n -= x[1]
+        end
+        n
+      }) * 0)) >> proc { n }
+  end
+
+  def expr2
+    n = nil
+    ((rule(:prim) >> proc { |x| n = x[0] }) -
+      ((((token('*') | token('/')) - rule(:prim)) >> proc { |x|
+        case x[0]
+        when '*'
+          n *= x[1]
+        when '/'
+          n /= x[1]
+        end
+        n
+      }) * 0)) >> proc { n }
+  end
+
+  def prim
+    (token(:int) >> proc { |x| x[0].value.to_i }) |
+      ((token('(') - rule(:expr1) - token(')')) >> proc { |x| x[1] })
+  end
+
+  def parse(str)
+    tokenizer = StringTokenizer[
+      /\d+(?!\.\d)/ => :int,
+      /\d+\.\d+/ => :real,
+    ]
+    expr1.parse(tokenizer.generate(str))
+  end
+end
+
+ENV.fetch('TEST', nil) and return
+
+parser = Sample3Parser.new
+puts("1+10 = #{parser.parse('1+10')}")
+puts("2-1*20+18 = #{parser.parse('2 - 1 * 20 + 18')}")
+puts("2-(1-20) = #{parser.parse('2 - (1 - 20)')}")
+puts("1+2-3 = #{parser.parse('1 + 2 - 3')}")

data/samples/sample4.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+# -*- ruby -*-
+# caching constructed grammars
+
+require 'tdparser'
+require 'tdparser/utils'
+
+class Sample4Parser
+  include TDParser
+
+  def expr1
+    (rule(:expr2) - (((token('+') | token('-')) - rule(:expr2)) * 0)) >> proc { |x|
+      x[1].inject(x[0]) do |n, y|
+        case y[0]
+        when '+'
+          n + y[1]
+        when '-'
+          n - y[1]
+        end
+      end
+    }
+  end
+
+  def expr2
+    (rule(:prim) - (((token('*') | token('/')) - rule(:prim)) * 0)) >> proc { |x|
+      n = x[0]
+      x[1].inject(x[0]) do |n, y|
+        case y[0]
+        when '*'
+          n * y[1]
+        when '/'
+          n / y[1]
+        end
+      end
+    }
+  end
+
+  def prim
+    (token(:int) >> proc { |x| x[0].value.to_i }) |
+      ((token('(') - rule(:expr1) - token(')')) >> proc { |x| x[1] })
+  end
+
+  def parse(str)
+    tokenizer = StringTokenizer[
+      /\d+(?!\.\d)/ => :int,
+      /\d+\.\d+/ => :real,
+    ]
+    expr1.parse(tokenizer.generate(str))
+  end
+end
+
+class FastParser < Sample4Parser
+  def expr1
+    @expr1 ||= super()
+  end
+
+  def expr2
+    @expr2 ||= super()
+  end
+
+  def prim
+    @prim  ||= super()
+  end
+
+  def parse(str)
+    tokens = str.split(%r{(?:\s+)|([()+\-*/])}).reject { |x| x == '' }
+    expr1.parse(tokens)
+  end
+end
+
+ENV.fetch('TEST', nil) and return
+
+parser = Sample4Parser.new
+puts("1+10 = #{parser.parse('1+10')}")
+puts("2-1*20+18 = #{parser.parse('2 - 1 * 20 + 18')}")
+puts("2-(1-20) = #{parser.parse('2 - (1 - 20)')}")
+puts("1+2-3 = #{parser.parse('1 + 2 - 3')}")

data/samples/sample5.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# -*- ruby -*-
+# writing grammars in the substitution style.
+
+require 'tdparser'
+require 'tdparser/utils'
+
+parser = TDParser.define do |g|
+  g.plus = '+'
+  g.minus = '-'
+  g.mult = '*'
+  g.div = '/'
+
+  g.expr1 =
+    (g.expr2 - (((g.plus | g.minus) - g.expr2) * 0)) >> proc { |x|
+      x[1].inject(x[0])  do |n, y|
+        case y[0]
+        when '+'
+          n + y[1]
+        when '-'
+          n - y[1]
+        end
+      end
+    }
+
+  g.expr2 =
+    (g.prim - (((g.mult | g.div) - g.prim) * 0)) >> proc { |x|
+      x[1].inject(x[0]) do |n, y|
+        case y[0]
+        when '*'
+          n * y[1]
+        when '/'
+          n / y[1]
+        end
+      end
+    }
+
+  g.prim =
+    (g.token(:int) >> proc { |x| x[0].value.to_i }) |
+    ((g.token('(') - g.expr1 - g.token(')')) >> proc { |x| x[1] })
+
+  def parse(str)
+    tokenizer = TDParser::StringTokenizer[
+      /\d+(?!\.\d)/ => :int,
+      /\d+\.\d+/ => :real,
+    ]
+    expr1.parse(tokenizer.generate(str))
+  end
+end
+
+if ENV['TEST']
+  Sample5Parser = parser
+  return
+end
+
+puts("1+10 = #{parser.parse('1+10')}")
+puts("2-1*20+18 = #{parser.parse('2 - 1 * 20 + 18')}")
+puts("2-(1-20) = #{parser.parse('2 - (1 - 20)')}")
+puts("1+2-3 = #{parser.parse('1 + 2 - 3')}")

data/samples/sample_expr.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# -*- ruby -*-
+# writing grammars using chainl().
+
+require 'tdparser'
+require 'tdparser/utils'
+
+parser = TDParser.define do |g|
+  g.plus = '+'
+  g.minus = '-'
+  g.mult = '*'
+  g.div = '/'
+
+  g.expr1 =
+    chainl(prim, mult | div, plus | minus) do |x|
+      case x[1]
+      when '+'
+        x[0] + x[2]
+      when '-'
+        x[0] - x[2]
+      when '*'
+        x[0] * x[2]
+      when '/'
+        x[0] / x[2]
+      end
+    end
+
+  g.prim =
+    (token(:int) >> proc { |x| x[0].value.to_i }) |
+    ((token('(') - expr1 - token(')')) >> proc { |x| x[1] })
+
+  def parse(str)
+    tokenizer = TDParser::StringTokenizer[
+      /\d+(?!\.\d)/ => :int,
+      /\d+\.\d+/ => :real,
+    ]
+    expr1.parse(tokenizer.generate(str))
+  end
+end
+
+if ENV['TEST']
+  SampleExprParser = parser
+  return
+end
+
+puts("1 = #{parser.parse('1')}")
+puts("1+10 = #{parser.parse('1+10')}")
+puts("2-1*20+18 = #{parser.parse('2 - 1 * 20 + 18')}")
+puts("2-(1-20) = #{parser.parse('2 - (1 - 20)')}")
+puts("1+2-3 = #{parser.parse('1 + 2 - 3')}")

data/samples/sample_list.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'tdparser'
+
+parser = TDParser.define do |g|
+  g.lp = '('
+  g.rp = ')'
+  g.str = /\w+/
+
+  # Note that "g.elem*1" is a iteration of a sequence that consists
+  # of only "g.elem", but it is not a iteration of "g.elem".
+  g.list = (g.lp - (g.elem * 1) - g.rp) >> proc { |x| x[1].collect { |y| y[0] } }
+  g.elem = (g.str | g.list) >> proc { |x| x[0] }
+
+  def parse(str)
+    buff = str.split(/\s+|([()])/).select { |s| s.size.positive? }
+    list.parse(buff)
+  end
+end
+
+if ENV['TEST']
+  SampleListParser = parser
+  return
+end
+
+list = '(a (b c d) (e f g))'
+r = parser.parse(list)
+p r

data/samples/sample_optimize.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'tdparser'
+require 'benchmark'
+
+# disable auto optimization
+module TDParser
+  class Parser
+    def |(other)
+      ChoiceParser.new(self, other)
+    end
+  end
+end
+
+parser = TDParser.define do |g|
+  f = proc { |x| x.flatten }
+  g.rule1 =
+    ((token('1') - token('2') - rule1 - token('a')) >> f) |
+    ((token('1') - token('2') - rule1 - token('b')) >> f) |
+    empty
+
+  g.rule2 =
+    (((token('1') - token('2') - rule2 - token('a')) >> f) |
+     ((token('1') - token('2') - rule2 - token('b')) >> f) |
+     empty).optimize
+
+  g.rule3 =
+    (((token('1') - token('2') - rule3 - (token('a') | token('b'))) >> f) |
+     empty)
+end
+
+puts(parser.rule1.to_s)
+puts(parser.rule2.to_s)
+puts(parser.rule3.to_s)
+
+N = 10
+Benchmark.bm do |x|
+  buff = %w[1 2]
+  b = ['b']
+  [5, 10, 15].each do |i|
+    puts('--')
+    x.report { N.times { $r1 = parser.rule1.parse((buff * i) + (b * i)) } }
+    x.report { N.times { $r2 = parser.rule2.parse((buff * i) + (b * i)) } }
+    x.report { N.times { $r3 = parser.rule3.parse((buff * i) + (b * i)) } }
+  end
+end

data/samples/sample_xml.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'tdparser'
+require 'tdparser/utils'
+require 'tdparser/xml'
+
+translator = TDParser.define do |g|
+  extend TDParser::XMLParser
+
+  g.xml =
+    (element('a') do
+      element('b') do
+        g.xmlseq
+      end >> dom_constructor(&:children)
+    end >> dom_constructor do |node|
+             node.name = 'AB'
+             node
+           end) |
+    (element(String) do
+      g.xmlseq
+    end >> dom_constructor do |node|
+             node.name = node.name.upcase
+             node
+           end) |
+    (doctype do
+      g.xmlseq
+    end >> dom_constructor { |node| node }) |
+    (text >> proc { |x| x[0] }) |
+    (elementdecl >> proc { |x| x[0] }) |
+    (xmldecl >> proc { |x| x[0] }) |
+    (comment >> proc { |x| x[0] }) |
+    (any_node >> proc { |x| x[0] })
+
+  g.xmlseq =
+    ((g.xml * 0) >> proc { |x| x[0].collect { |y| y[0] } }) |
+    def translate(src)
+      gen = TDParser::XMLParser::XMLTokenGenerator.new(src)
+      xmlseq.parse(gen)
+    end
+end
+
+if ENV['TEST']
+  XMLTranslator = translator
+  return
+end
+
+seq = translator.translate(<<~EOS)
+  <?xml version="1.0" ?>
+  <!DOCTYPE body [
+   <!ELEMENT body (#PCDATA, strong*)>
+   <!ELEMENT strong (#PCDATA)>
+   ]>
+  <list>
+    <!-- comment -->
+    <a><b><c>hoge</c></b></a>
+    <b>b?</b>
+  </list>
+EOS
+doc = REXML::Document.new
+seq.each { |x| doc.add(x) }
+puts(doc)

metadata

@@ -0,0 +1,70 @@
+--- !ruby/object:Gem::Specification
+name: tdparser
+version: !ruby/object:Gem::Version
+  version: 1.5.0
+platform: ruby
+authors:
+- Takaaki Tateishi
+- gemmaro
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-11-08 00:00:00.000000000 Z
+dependencies: []
+description: TDParser (formerly TDP4R) is a top-down parser library that consists
+  of parser combinators and utility functions.
+email:
+- ttate@ttsky.net
+- gemmaro.dev@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".dir-locals.el"
+- ".envrc"
+- ".rubocop.yml"
+- CHANGELOG.md
+- COPYING
+- README
+- Rakefile
+- doc/faq.rdoc
+- doc/guide.rdoc
+- lib/tdparser.rb
+- lib/tdparser/utils.rb
+- lib/tdparser/version.rb
+- lib/tdparser/xml.rb
+- manifest.scm
+- samples/sample1.rb
+- samples/sample2.rb
+- samples/sample3.rb
+- samples/sample4.rb
+- samples/sample5.rb
+- samples/sample_expr.rb
+- samples/sample_list.rb
+- samples/sample_optimize.rb
+- samples/sample_xml.rb
+homepage:
+licenses:
+- BSD-3-Clause
+metadata:
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.26
+signing_key:
+specification_version: 4
+summary: Top down parser library
+test_files: []