tdparser 1.5.1 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 25d5372580072f4703947c061df7b6d2cb5f85c3c073ef130747f211671a04e4
-  data.tar.gz: 94d335eb747dfe643cc5e488c15b22cc0bd3936bf2a76ed2bbec9008e30e2c6f
+  metadata.gz: 9df17ece1521def452aa92e124da446abd512000073b36a989e53fb0f5032574
+  data.tar.gz: 1f1766675086fcba1870127d05419eaa5aa6de6f436347fad23f7b5d37afc8c6
 SHA512:
-  metadata.gz: 48e4f42bb49c592021fd4fb4455a5120a8087f639ab9f21cf2986d90a2d0f87b1f108bd6315197934769f738064828e372b10ea1dba2495266486c58a5f56dff
-  data.tar.gz: 34cf78b68bf980d9814f1b02ba6bfbe3b9a8a9167bb6b84d8d0a598b060f496d9805a5eca42314fb3c66fcd70b0ceb0cbfd409273f445ffcf3750e99a8c88e71
+  metadata.gz: 3821c0d37e95f49d3db3fe8230ba9e662cb9bb648497698dac95c34c0507396d78c02720943ab1cd4028f457c930e90f37a93024b68a2bf06664498ef7290357
+  data.tar.gz: c8ac3e9fca74e15a1a888f376ff9058fdb797f6944876a3f77f224dd4de4cf20f7ed88891b115d7138c624cb91a07bed0144f1cc8d0ee1a96b97a1275ed02295

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 ## [Unreleased]
 
+## 1.6.0 - 2024-11-17
+
+* Add more readable method aliases.
+* Make token buffer's state accessible attribute.
+* Split files such as parsers.  You might need to require individual files.
+* Change token buffer class not to inherit from array class.
+* Move XML token generator class to upper namespace.
+
 ## 1.5.1 - 2024-11-10
 
 This is a maintenance release, no user facing changes.

data/README

@@ -23,6 +23,11 @@ the feature of (2), we need not consider how to prevent conflicts
 among production rules.  In addition, TDParser can be viewed as an
 internal DSL for writing LL(k) grammars because of (3).
 
+Repository is located at:
+
+* Disroot[https://git.disroot.org/gemmaro/tdparser]
+* GitHub[https://github.com/gemmaro/tdparser] (for GitHub Pages)
+
 == License
 
   Copyright(C) 2003, 2004, 2005, 2006 Takaaki Tateishi <ttate@ttsky.net>

data/Rakefile

@@ -16,3 +16,7 @@ Rake::TestTask.new do |t|
   t.libs << 'samples' << 'test'
   t.test_files = FileList['test/*_test.rb']
 end
+
+task :gensig do
+  sh 'typeprof', '-r', 'forwardable', '-r', 'strscan', '-r', 'enumerable', '-o', 'sig/tdparser.gen.rbs', 'sig/tdparser.rbs', *Dir['lib/**/*.rb']
+end

data/lib/tdparser/action_parser.rb

@@ -0,0 +1,29 @@
+module TDParser
+  class ActionParser < CompositeParser # :nodoc:
+    attr_reader :action
+
+    def initialize(parser, act)
+      @action = act
+      super(parser)
+    end
+
+    def call(tokens, buff)
+      if (x = @parsers[0].call(tokens, buff)).nil?
+        nil
+      else
+        x = TokenBuffer[*x]
+        x.map = buff.map
+        Sequence[@action[x]]
+      end
+    end
+
+    def ==(other)
+      super(other) &&
+        (@action == other.action)
+    end
+
+    def to_s
+      "(#{@parsers[0]} <action>)"
+    end
+  end
+end

data/lib/tdparser/any_parser.rb

@@ -0,0 +1,20 @@
+module TDParser
+  class AnyParser < Parser # :nodoc:
+    def call(tokens, _buff)
+      t = tokens.shift
+      if t.nil?
+        nil
+      else
+        Sequence[t]
+      end
+    end
+
+    def to_s
+      '<any>'
+    end
+
+    def ==(_other)
+      true
+    end
+  end
+end

data/lib/tdparser/backref_parser.rb

@@ -0,0 +1,29 @@
+module TDParser
+  class BackrefParser < ReferenceParser # :nodoc:
+    attr_reader :label, :equality
+
+    def initialize(label, eqsym)
+      @label = label
+      @equality = eqsym
+    end
+
+    def call(tokens, buff)
+      ys = buff.map[@label]
+      if ys.nil? || ys.empty?
+        nil
+      else
+        back_ref(ys.dup, @equality).call(tokens, buff)
+      end
+    end
+
+    def to_s
+      "<backref:#{@label}>"
+    end
+
+    def ==(other)
+      super(other) &&
+        (@label == other.label) &&
+        (@equality == other.equality)
+    end
+  end
+end

data/lib/tdparser/buffer_utils.rb

@@ -0,0 +1,13 @@
+module TDParser
+  module BufferUtils # :nodoc:
+    def prepare(buff)
+      b = TokenBuffer.new
+      b.map = buff.map
+      b
+    end
+
+    def recover(buff, ts)
+      buff.each { |b| ts.unshift(b) }
+    end
+  end
+end

data/lib/tdparser/choice_parser.rb

@@ -0,0 +1,80 @@
+module TDParser
+  class ChoiceParser < CompositeParser # :nodoc:
+    def call(tokens, buff)
+      b = prepare(buff)
+      if (x = @parsers[0].call(tokens, b)).nil?
+        recover(b, tokens)
+        @parsers[1].call(tokens, buff)
+      else
+        buff.insert(0, *b)
+        x
+      end
+    end
+
+    def to_s
+      "(#{@parsers[0]} | #{@parsers[1]})"
+    end
+
+    def shared_sequence(r1, r2)
+      if r1.is_a?(ConcatParser) && r2.is_a?(ConcatParser)
+        r11 = r1.parsers[0]
+        r12 = r1.parsers[1]
+        r21 = r2.parsers[0]
+        r22 = r2.parsers[1]
+        if r11.same?(r21)
+          share, r12, r22, = shared_sequence(r12, r22)
+          return [r11 - share, r12, r22] if share
+
+          return [r11, r12, r22]
+
+        end
+      end
+      [nil, r1, r2]
+    end
+
+    def optimize(default = false)
+      r1 = @parsers[0]
+      r2 = @parsers[1]
+      if r1.is_a?(ActionParser)
+        act1 = r1.action
+        r1 = r1.parsers[0]
+      end
+      if r2.is_a?(ActionParser)
+        act2 = r2.action
+        r2 = r2.parsers[0]
+      end
+      share, r12, r22, = shared_sequence(r1, r2)
+      if share
+        r = share - (r12 + r22)
+        if act1
+          r = if act2
+                r >> proc do |x|
+                  y0, y1, *_ = x.pop
+                  if y0
+                    act1.call(x.push(*y0))
+                  else
+                    act2.call(x.push(*y1))
+                  end
+                end
+              else
+                r >> proc do |x|
+                  y0, = x.pop
+                  act1.call(x.push(*y0)) if y0
+                end
+              end
+        elsif act2
+          r = r >> proc do |x|
+                     _, y1, = x.pop
+                     act2.call(x.push(*y1)) if y1
+                   end
+        end
+        return r
+      end
+      if default
+        dup
+      else
+        super(default)
+      end
+    end
+  end
+end

data/lib/tdparser/composite_parser.rb

@@ -0,0 +1,29 @@
+module TDParser
+  class CompositeParser < Parser # :nodoc:
+    attr_accessor :parsers
+
+    def initialize(*parsers)
+      @parsers = parsers
+    end
+
+    def optimize(default = false)
+      parser = dup
+      parser.parsers = @parsers.collect { |x| x.optimize(default) }
+      parser
+    end
+
+    def ==(other)
+      (self.class == other.class) &&
+        (@parsers == other.parsers)
+    end
+
+    def same?(r)
+      super(r) &&
+        @parsers.zip(r.parsers).all? { |x, y| x.same?(y) }
+    end
+
+    def to_s
+      "<composite: #{@parsers.collect(&:to_s)}>"
+    end
+  end
+end

data/lib/tdparser/concat_parser.rb

@@ -0,0 +1,21 @@
+module TDParser
+  class ConcatParser < CompositeParser # :nodoc:
+    def call(tokens, buff)
+      if (x = @parsers[0].call(tokens, buff)).nil?
+        nil
+      elsif (y = @parsers[1].call(tokens, buff)).nil?
+        nil
+      else
+        x + y
+      end
+    end
+
+    def -(other)
+      @parsers[0] - (@parsers[1] - other)
+    end
+
+    def to_s
+      "(#{@parsers[0]} #{@parsers[1]})"
+    end
+  end
+end

data/lib/tdparser/condition_parser.rb

@@ -0,0 +1,28 @@
+module TDParser
+  class ConditionParser < Parser # :nodoc:
+    attr_reader :condition
+
+    def initialize(&condition)
+      @condition = condition
+    end
+
+    def call(_tokens, buff)
+      return unless (res = @condition.call(buff.map))
+
+      Sequence[res]
+    end
+
+    def to_s
+      "<condition:#{@condition}>"
+    end
+
+    def ==(other)
+      super(other) &&
+        (@condition == other.condition)
+    end
+
+    def same?(_r)
+      false
+    end
+  end
+end

data/lib/tdparser/empty_parser.rb

@@ -0,0 +1,15 @@
+module TDParser
+  class EmptyParser < Parser # :nodoc:
+    def call(_tokens, _buff)
+      Sequence[nil]
+    end
+
+    def to_s
+      '<empty>'
+    end
+
+    def ==(_other)
+      true
+    end
+  end
+end

data/lib/tdparser/fail_parser.rb

@@ -0,0 +1,15 @@
+module TDParser
+  class FailParser < Parser # :nodoc:
+    def call(_tokens, _buff)
+      nil
+    end
+
+    def to_s
+      '<fail>'
+    end
+
+    def ==
+      (self.class == r.class)
+    end
+  end
+end

data/lib/tdparser/grammar.rb

@@ -0,0 +1,23 @@
+module TDParser
+  class Grammar # :nodoc:
+    include TDParser
+
+    alias define instance_eval
+
+    def method_missing(sym, *args)
+      if sym[-1, 1] == '='
+        parser, = args
+        name = sym[0..-2]
+        parser.is_a?(Parser) or parser = token(parser)
+        self.class.instance_eval do
+          instance_methods.include?(name.intern) or
+            define_method(name) { parser }
+        end
+      elsif args.empty?
+        rule(sym)
+      else
+        raise(NoMethodError, "undefined method `#{sym}' for #{inspect}")
+      end
+    end
+  end
+end

data/lib/tdparser/iteration_parser.rb

@@ -0,0 +1,73 @@
+module TDParser
+  class IterationParser < CompositeParser # :nodoc:
+    attr_reader :min, :range
+
+    def initialize(parser, n, range)
+      @min = n
+      @range = range
+      super(parser)
+    end
+
+    def call(ts, buff)
+      r = @parsers[0]
+      n = @min
+      x  = true
+      xs = []
+      while n.positive?
+        n -= 1
+        b = prepare(buff)
+        if (x = r.call(ts, b)).nil?
+          recover(b, ts)
+          break
+        else
+          buff.insert(0, *b)
+          xs.push(x)
+        end
+      end
+      if x.nil?
+        nil
+      else
+        if range
+          range.each do
+            loop do
+              y = x
+              b = prepare(buff)
+              if (x = r.call(ts, b)).nil?
+                recover(b, ts)
+                x = y
+                break
+              else
+                buff.insert(0, *b)
+                xs.push(x)
+              end
+            end
+          end
+        else
+          loop do
+            y = x
+            b = prepare(buff)
+            if (x = r.call(ts, b)).nil?
+              recover(b, ts)
+              x = y
+              break
+            else
+              buff.insert(0, *b)
+              xs.push(x)
+            end
+          end
+        end
+        Sequence[xs]
+      end
+    end
+
+    def to_s
+      "(#{@parsers[0]})*#{@range ? @range.to_s : @min.to_s}"
+    end
+
+    def ==(other)
+      super(other) &&
+        (@min == other.min) &&
+        (@range == other.range)
+    end
+  end
+end

data/lib/tdparser/label_parser.rb

@@ -0,0 +1,25 @@
+module TDParser
+  class LabelParser < CompositeParser # :nodoc:
+    attr_reader :label
+
+    def initialize(parser, label)
+      @label = label
+      super(parser)
+    end
+
+    def call(tokens, buff)
+      x = @parsers[0].call(tokens, buff)
+      buff.map[@label] = x
+      x
+    end
+
+    def ==(other)
+      super(other) &&
+        (@label == other.label)
+    end
+
+    def to_s
+      "(#{@parsers[0]}/#{@label})"
+    end
+  end
+end

data/lib/tdparser/negative_parser.rb

@@ -0,0 +1,17 @@
+module TDParser
+  class NegativeParser < CompositeParser # :nodoc:
+    def call(tokens, buff)
+      b = prepare(buff)
+      r = @parsers[0].call(tokens, b)
+      rev = b.reverse
+      recover(b, tokens)
+      return unless r.nil?
+
+      Sequence[Sequence[*rev]]
+    end
+
+    def to_s
+      "~#{@parsers[0]}"
+    end
+  end
+end

data/lib/tdparser/non_terminal_parser.rb

@@ -0,0 +1,33 @@
+module TDParser
+  class NonTerminalParser < Parser # :nodoc:
+    attr_reader :context, :symbol, :options
+
+    def initialize(context, sym, *options)
+      @context = context
+      @symbol = sym
+      @options = options
+    end
+
+    def call(tokens, buff)
+      res = nil
+      case @symbol
+      when Symbol, String
+        res = @context.__send__(@symbol, *@options).call(tokens, buff)
+      when Parser
+        res = @symbol.call(tokens, buff)
+      end
+      res
+    end
+
+    def ==(other)
+      (self.class == other.class) &&
+        (@context == other.context) &&
+        (@symbol == other.symbol) &&
+        (@options == other.options)
+    end
+
+    def to_s
+      @symbol.to_s
+    end
+  end
+end

data/lib/tdparser/none_parser.rb

@@ -0,0 +1,18 @@
+module TDParser
+  class NoneParser < Parser # :nodoc:
+    def call(tokens, _buff)
+      t = tokens.shift
+      return unless t.nil?
+
+      Sequence[nil]
+    end
+
+    def to_s
+      '<none>'
+    end
+
+    def ==(_other)
+      true
+    end
+  end
+end

data/lib/tdparser/parallel_parser.rb

@@ -0,0 +1,18 @@
+module TDParser
+  class ParallelParser < CompositeParser # :nodoc:
+    def call(tokens, buff)
+      b = prepare(buff)
+      if (x = @parsers[0].call(tokens, b)).nil?
+        recover(b, tokens)
+        Sequence[Sequence[nil, @parsers[1].call(tokens, buff)]]
+      else
+        buff.insert(0, *b)
+        Sequence[Sequence[x, nil]]
+      end
+    end
+
+    def to_s
+      "(#{@parsers[0]} + #{@parsers[1]})"
+    end
+  end
+end

data/lib/tdparser/parser.rb

@@ -0,0 +1,104 @@
+module TDParser
+  class Parser
+    include BufferUtils
+    include TDParser
+
+    def to_proc
+      proc { |*x| call(*x) }
+    end
+
+    def to_s
+      '??'
+    end
+
+    def call(*args); end
+
+    def optimize(_default = false)
+      dup
+    end
+
+    def ==(_other)
+      false
+    end
+
+    def same?(r)
+      self == r
+    end
+
+    def -(other)
+      ConcatParser.new(self, other)
+    end
+
+    def +(other)
+      ParallelParser.new(self, other)
+    end
+
+    def |(other)
+      ChoiceParser.new(self, other).optimize(true)
+    end
+
+    def *(other)
+      if other.is_a?(Range)
+        n = other.min
+      else
+        n = other
+        other = nil
+      end
+      IterationParser.new(self, n, other)
+    end
+
+    def >>(other)
+      ActionParser.new(self, other)
+    end
+
+    def /(other)
+      LabelParser.new(self, other)
+    end
+
+    def %(other)
+      StackParser.new(self, other)
+    end
+
+    def >(other)
+      Parser.new do |tokens, buff|
+        buff[other] = buff.dup
+        self[tokens, buff]
+      end
+    end
+
+    def ~@
+      NegativeParser.new(self)
+    end
+
+    def parse(tokens = nil, buff = nil, &blk)
+      buff ||= TokenBuffer.new
+      @tokens = if blk.nil?
+                  if tokens.respond_to?(:shift) && tokens.respond_to?(:unshift)
+                    tokens
+                  elsif tokens.respond_to?(:each)
+                    TokenGenerator.new(tokens)
+                  else
+                    tokens
+                  end
+                else
+                  TokenGenerator.new(&blk)
+                end
+      r = call(@tokens, buff)
+      if r.nil?
+        nil
+      else
+        r[0]
+      end
+    end
+
+    def peek
+      t = @tokens.shift
+      @tokens.unshift(t) unless t.nil?
+      t
+    end
+
+    def do(&block)
+      self >> block
+    end
+  end
+end

data/lib/tdparser/reference_parser.rb

@@ -0,0 +1,23 @@
+module TDParser
+  class ReferenceParser < Parser # :nodoc:
+    private
+
+    def back_ref(xs, eqsym)
+      x = xs.shift
+      xs.inject(token(x, eqsym)) do |acc, x|
+        case x
+        when Sequence
+          acc - back_ref(x, eqsym)
+        else
+          acc - token(x, eqsym)
+        end
+      end
+    end
+
+    alias __backref__ back_ref
+
+    def same?(_r)
+      false
+    end
+  end
+end

data/lib/tdparser/sequence.rb

@@ -0,0 +1,7 @@
+module TDParser
+  class Sequence < Array # :nodoc:
+    def +(other)
+      dup.concat(other)
+    end
+  end
+end