pils 0.1.4

data/lib/pils/de/small.rb

@@ -0,0 +1,128 @@
+# This file is part of Pils.
+#
+#     Pils is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU Lesser General Public License as
+#     published by the Free Software Foundation, either version 3 of
+#     the License, or (at your option) any later version.
+#
+#     Pils is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU Lesser General Public License for more details.
+#
+#     You should have received a copy of the
+#     GNU Lesser General Public License along with Pils.
+#     If not, see <http://www.gnu.org/licenses/>.
+
+# This is a language config file.
+
+module Pils
+  module De
+    module Small
+
+      CAS = %w(nom gen dat acc) #[:nom, :gen, :dat, :acc]
+      NUM = %w(sg pl) #[:sg, :pl]
+      GEN = %w(m f n) # [:m, :f, :n]
+      PER = %w(1 2 3) # [:er, :zw, :dr]
+      MOD = %w(imp ind) # [:imp, :fin, :inf]
+      TMP = %w(pres)
+
+      def self.add_german_verb(lex, grundform, zweitform, praetform, partform, semantic_component)
+        lex.add_wordform Wordform.new(normalize_forms("#{grundform}e"),   "V_fin_sg_1_pres".to_sym,   {}, semantic_component.clone )
+        lex.add_wordform Wordform.new(normalize_forms("#{zweitform}st"),  "V_fin_sg_2_pres".to_sym,   {}, semantic_component.clone )
+        lex.add_wordform Wordform.new(normalize_forms("#{zweitform}t"),   "V_fin_sg_3_pres".to_sym,   {}, semantic_component.clone )
+        lex.add_wordform Wordform.new(normalize_forms("#{grundform}en"),  "V_fin_pl_1_pres".to_sym,   {}, semantic_component.clone )
+        lex.add_wordform Wordform.new(normalize_forms("#{grundform}t"),   "V_fin_pl_2_pres".to_sym,   {}, semantic_component.clone )
+        lex.add_wordform Wordform.new(normalize_forms("#{grundform}en"),  "V_fin_pl_3_pres".to_sym,   {}, semantic_component.clone )
+      end
+
+      def self.add_german_noun(lex, semantic_component, gender, forms)
+        lex.add_wordform(Wordform.new(forms[0], "N_nom_sg_#{gender}_3".to_sym, {}, semantic_component.clone ))
+        lex.add_wordform(Wordform.new(forms[1], "N_gen_sg_#{gender}_3".to_sym, {}, semantic_component.clone ))
+        lex.add_wordform(Wordform.new(forms[2], "N_dat_sg_#{gender}_3".to_sym, {}, semantic_component.clone ))
+        lex.add_wordform(Wordform.new(forms[3], "N_acc_sg_#{gender}_3".to_sym, {}, semantic_component.clone ))
+        lex.add_wordform(Wordform.new(forms[4], "N_nom_pl_#{gender}_3".to_sym, {}, semantic_component.clone ))
+        lex.add_wordform(Wordform.new(forms[5], "N_gen_pl_#{gender}_3".to_sym, {}, semantic_component.clone ))
+        lex.add_wordform(Wordform.new(forms[6], "N_dat_pl_#{gender}_3".to_sym, {}, semantic_component.clone ))
+        lex.add_wordform(Wordform.new(forms[7], "N_acc_pl_#{gender}_3".to_sym, {}, semantic_component.clone ))
+      end
+
+      def self.normalize_forms(form)
+        form = form.gsub(/ee/, 'e')
+        form
+      end
+
+      def self.define_grammar
+        include Pils::Parsing
+        g = Grammar.new()
+        sentence = Cat.new(:S)
+        g.starting_cats = [ sentence ]
+
+        # S -> NP VP, both numerus
+        NUM.each do |num|
+          PER.each do |per|
+            TMP.each do |tmp|
+              GEN.each do |gen|
+                r = Rule.new(Cat.new(:S), [ Cat.new("NP_nom_#{num}_#{gen}_#{per}".to_sym, :agens), Cat.new("VP_fin_#{num}_#{per}_#{tmp}".to_sym, :agens)], {})
+                g.rules << r
+              end
+              r = Rule.new(Cat.new("VP_fin_#{num}_#{per}_#{tmp}".to_sym, :agens), [ Cat.new("V_fin_#{num}_#{per}_#{tmp}".to_sym, :agens)], {})
+              g.rules << r
+            end
+          end
+        end
+
+        # NP rules
+        CAS.each do |cas|
+          NUM.each do |num|
+            GEN.each do |gen|
+              #PER.each do |per|
+                r = Rule.new(Cat.new("NP_#{cas}_#{num}_#{gen}_3".to_sym), [ Cat.new("DETD_#{cas}_#{num}_#{gen}_3".to_sym), Cat.new("NPX_#{cas}_#{num}_#{gen}_3".to_sym)], {})
+                g.rules << r
+                r = Rule.new(Cat.new("NPX_#{cas}_#{num}_#{gen}_3".to_sym), [ Cat.new("N_#{cas}_#{num}_#{gen}_3".to_sym, :agens)], {})
+                g.rules << r
+                #r = FetaRule.new(FetaCat.new("NPX_#{cas}_#{num}_#{gen}_#{per}".to_sym), [ FetaCat.new("ADJA_#{cas}_#{num}_#{gen}_#{per}".to_sym), FetaCat.new("NPX_#{cas}_#{num}_#{gen}_#{per}".to_sym)], {})
+                #g.rules << r
+                #end
+            end
+          end
+        end
+
+        return g
+      end
+
+      def self.define_lexicon
+        lexicon = Lexicon.new()
+
+        # Definite Artikel
+
+        defin = %w(der des dem den die der den die die der der die die der den die das des dem das die der den die)
+
+        id = 0
+        GEN.each do |gen|
+          NUM.each do |num|
+            CAS.each do |cas|
+              form = defin[id]
+              id = id + 1
+              # DETD_nom_sg_f_3
+              lexicon.add_wordform Wordform.new(form, "DETD_#{cas}_#{num}_#{gen}_3".to_sym, {}, {det: :yes})
+            end
+          end
+        end
+
+        # some nouns
+
+        # add_german_noun(lexicon, :car, :n, %w(Auto Autos Auto Auto Autos Autos Autos Autos))
+        add_german_noun(lexicon, {species: :dog}, :m, %w(Hund Hundes Hund Hund Hunde Hunde Hunden Hunde))
+        add_german_noun(lexicon, {species: :cat}, :f, %w(Katze Katze Katze Katze Katzen Katzen Katzen Katzen))
+        add_german_noun(lexicon, {species: :pig}, :n, %w(Schwein Schweins Schwein Schwein Schweine Schweine Schweinen Schweine))
+        add_german_verb(lexicon, "fauch", "fauch", "faucht", "faucht", {sound: :hiss} )
+        add_german_verb(lexicon, "miau",  "miau",  "miaut",  "miaut",  {sound: :meow} )
+        add_german_verb(lexicon, "bell",  "bell",  "bellt",  "bellt",  {sound: :bark} )
+        add_german_verb(lexicon, "grunz", "grunz", "grunzt", "grunzt", {sound: :oink} )
+
+        return lexicon
+      end
+    end
+  end
+end

data/lib/pils/parsing.rb

@@ -0,0 +1,31 @@
+# This file is part of Pils.
+#
+#     Pils is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU Lesser General Public License as
+#     published by the Free Software Foundation, either version 3 of
+#     the License, or (at your option) any later version.
+#
+#     Pils is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU Lesser General Public License for more details.
+#
+#     You should have received a copy of the
+#     GNU Lesser General Public License along with Pils.
+#     If not, see <http://www.gnu.org/licenses/>.
+
+# This module encapsulates all classes related to parsing natural
+# language.
+module Pils
+  module Parsing
+
+  end
+end
+
+require "pils/parsing/cat"
+require "pils/parsing/rule"
+require "pils/parsing/grammar"
+require "pils/parsing/wordform"
+require "pils/parsing/lexicon"
+require "pils/parsing/tree"
+require "pils/parsing/parser"

data/lib/pils/parsing/cat.rb

@@ -0,0 +1,62 @@
+# This file is part of Pils.
+#
+#     Pils is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU Lesser General Public License as
+#     published by the Free Software Foundation, either version 3 of
+#     the License, or (at your option) any later version.
+#
+#     Pils is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU Lesser General Public License for more details.
+#
+#     You should have received a copy of the
+#     GNU Lesser General Public License along with Pils.
+#     If not, see <http://www.gnu.org/licenses/>.
+
+
+module Pils
+  module Parsing
+    class Cat
+      attr_accessor :grammar
+      attr_accessor :rule
+      attr_accessor :cat
+      attr_accessor :feat
+      attr_accessor :descriptor
+
+
+      def initialize(new_cat, new_desc = nil, new_feat={})
+        @cat = new_cat
+        @descriptor = new_desc
+        @feat = Pils::Structures::Avm.new()
+        new_feat.each do |k,v|
+          @feat[k] = v
+        end
+      end
+
+      # true if the grammar lets you expand this symbol
+      # def expandible?()
+      #   grammar.rules.each do |r|
+      #     return true if self < r.left
+      #   end#
+      # end
+
+      # true if this cat matches other one.
+      def <(other)
+        return false if self.cat != other.cat
+        return self.feat < other.feat
+      end
+
+      def display
+        "%s" % [cat.to_s, feat.values.collect{|v| v.to_s}.join(',')]
+      end
+
+      def to_s
+        self.display
+      end
+
+    end
+  end
+end
+
+

data/lib/pils/parsing/grammar.rb

@@ -0,0 +1,47 @@
+# This file is part of Pils.
+#
+#     Pils is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU Lesser General Public License as
+#     published by the Free Software Foundation, either version 3 of
+#     the License, or (at your option) any later version.
+#
+#     Pils is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU Lesser General Public License for more details.
+#
+#     You should have received a copy of the
+#     GNU Lesser General Public License along with Pils.
+#     If not, see <http://www.gnu.org/licenses/>.
+
+
+module Pils
+  module Parsing
+    class Grammar
+
+      attr_accessor :starting_cats
+      attr_accessor :rules
+
+      def initialize()
+        @rules = []
+        @starting_cats = []
+      end
+
+      def expandible?(cat)
+        !@rules.find{|r| r.expandible?(cat)}.nil?
+      end
+
+      def expand(cat)
+        @rules.select{|r| r.expandible?(cat)}
+      end
+      
+      def describe_rules 
+        @rules.each do |rule|
+          Pils::log rule.display
+        end
+      end
+
+    end
+  end
+end
+

data/lib/pils/parsing/lexicon.rb

@@ -0,0 +1,100 @@
+# This file is part of Pils.
+#
+#     Pils is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU Lesser General Public License as
+#     published by the Free Software Foundation, either version 3 of
+#     the License, or (at your option) any later version.
+#
+#     Pils is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU Lesser General Public License for more details.
+#
+#     You should have received a copy of the
+#     GNU Lesser General Public License along with Pils.
+#     If not, see <http://www.gnu.org/licenses/>.
+
+
+module Pils
+  module Parsing
+    class Lexicon
+
+      attr_accessor :entries
+      attr_accessor :definite_articles
+
+      def initialize
+        @entries = Set.new
+        @definite_articles = [
+            'der', 'des', 'dem', 'den', 'die', 'der', 'den', 'die',
+            'die', 'der', 'der', 'die', 'die', 'der', 'den', 'die',
+            'das', 'des', 'dem', 'das', 'die', 'der', 'den', 'die'
+        ]
+        @indefinite_articles = [
+            'ein',  'eines', 'einem', 'einen', '', '', '', '',
+            'eine', 'einer', 'einer', 'eine',  '', '', '', '',
+            'ein',  'eines', 'einem', 'ein',   '', '', '', ''
+        ]
+      end
+
+      def definite_article(kase=:nom, number=:sg, gender=:m)
+        pos = 0
+        pos = 8 if gender==:f
+        pos = 16 if gender==:n
+        pos = pos + 4 if number==:pl
+        pos = pos + 1 if kase==:gen
+        pos = pos + 2 if kase==:dat
+        pos = pos + 3 if kase==:acc
+        @definite_articles[pos]
+      end
+
+      def indefinite_article(kase=:nom, number=:sg, gender=:m)
+        pos = 0
+        pos = 8 if gender==:f
+        pos = 16 if gender==:n
+        pos = pos + 4 if number==:pl
+        pos = pos + 1 if kase==:gen
+        pos = pos + 2 if kase==:dat
+        pos = pos + 3 if kase==:acc
+        @indefinite_articles[pos]
+      end
+
+
+      def add_wordform(new_form)
+        @entries << new_form
+      end
+
+
+      def get(params={})
+        temp = @entries
+        # Pils::log ">>>> " +  temp.to_a.collect{|a| ("%s(%s)" % [a.form,a.cat])}.join(', ')
+        return {} if (temp.empty? || params[:form] =~ /^\s*$/ )
+        if params.has_key?(:form) && !(params[:form].nil?)
+          # Pils::log "§FORM"
+          # temp.each do |t|
+          #   puts t
+          #   puts t.form
+          # end
+          # puts params[:form]
+          temp = temp.select{|t| !(t.form.nil?) && t.form.downcase==params[:form].downcase}
+        end
+        if params.has_key?(:cat) && !(params[:cat].nil?)
+          # Pils::log "§CAT"
+          temp = temp.select{|t| t.cat==params[:cat]}
+        end
+        if params.has_key?(:grammar)
+          g = Pils::Structures::Avm.new(params[:grammar])
+          temp = temp.select{|t| g < t.grammar}
+        end
+        temp
+      end
+
+
+      def describe
+        @entries.each do |entry|
+          Pils::log "// %s" % [entry.display.to_s]
+        end
+      end
+
+    end
+  end
+end

data/lib/pils/parsing/parser.rb

@@ -0,0 +1,310 @@
+# This file is part of Pils.
+#
+#     Pils is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU Lesser General Public License as
+#     published by the Free Software Foundation, either version 3 of
+#     the License, or (at your option) any later version.
+#
+#     Pils is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU Lesser General Public License for more details.
+#
+#     You should have received a copy of the
+#     GNU Lesser General Public License along with Pils.
+#     If not, see <http://www.gnu.org/licenses/>.
+
+module Pils
+  module Parsing
+    class Parser
+
+      attr_accessor :tokens
+
+      attr_accessor :syntax
+      attr_accessor :semantics
+
+      attr_accessor :syntax_position
+      attr_accessor :token_position
+
+      attr_accessor :stack
+
+      attr_accessor :grammar
+      attr_accessor :lexicon
+
+
+      def tokenize(str)
+        str.split(/\s+/)
+      end
+
+      def init(ptokens)
+        # set start states
+        @tokens = ptokens
+        @token_position = 0
+        # syntactic state
+        #   set state to first start symbol, stack to remaining start symbols
+
+        @syntax = Tree.new(@grammar.starting_cats.first, nil) # Pils::Parsing::
+        @syntax_position = 0
+
+        # semantic state
+        @semantics = {}
+        # stack
+        @stack = []
+
+        Pils::log @tokens
+      end
+
+      def word_substitutions
+        # current category in syntax
+        # current word
+        # is there a lexicon entry for form, cat, gram?
+
+        @lexicon_matches = @lexicon.get({
+                                            form: @tokens[@token_position],
+                                            cat: @head_cat.obj.cat,
+                                            grammar: @head_cat.obj.grammar
+                                        })
+        # Pils::log "Lexical matches: %i" % @lexicon_matches.count
+        # Pils::log "Lexical matches:\n - %s" % @lexicon_matches.collect{|m| m.display }.join("\n - ")
+        @lexicon_matches
+      end
+
+      def expand_cat
+        rules = @grammar.expand(@head_cat.obj)
+        # Pils::log "Erwartet wird: %s" % @head_cat.obj.display
+        # exit(0) if @head_cat.obj.display == "NP_nom_sg_m_3"
+        # Pils::log "    Ahnen       %s" % @head_cat.ancestors_from_root.collect{|a| a.obj }.join('>')
+        # Pils::log "    Descriptor  %s" % @head_path.join('>')
+
+
+        # Pils::log "Anwendbare Regeln: %i" %  rules.count
+        # rules.each_with_index do |rule,ind|
+        #   Pils::log "  %3i - %s ==> %s"% [ ind, rule.left.display, rule.right.collect{|r| r.display}.join(' ')]
+        # end
+
+        first_rule, *rules_tail = *rules
+
+        if rules_tail.count>0
+          rules_tail.each do |rule|
+            # create a new alternative state.
+            new_syntax = @syntax.clone
+            new_semantics = @semantics.clone
+            # apply rule!
+            new_child_list = []
+            new_head_cat = new_syntax.leaf_at(@syntax_position)
+            rule.right.each do |x|
+              new_child_list << Tree.new(x)
+            end
+            new_head_cat.set_children(new_child_list, true)
+            new_state = [new_syntax, new_semantics, @syntax_position, @token_position]
+            @stack << new_state
+          end
+        end
+        # expand current tree
+        # Pils::log @syntax.display
+        new_child_list = []
+        first_rule.right.each do |x|
+          new_child_list << Tree.new(x)
+        end
+        @head_cat.set_children(new_child_list, true)
+        # Pils::log @syntax.display
+        sync
+
+      end
+
+      def sync
+        @head_cat = @syntax.leaf_at(@syntax_position)
+        if @head_cat
+          @head_path = @head_cat.ancestors_from_root.collect{|a| a.obj.descriptor}.reject{|d| d.nil?}
+        end
+        # Pils::log @head_cat.obj.display
+
+      end
+
+      def parse!(max_iterations=50)
+        @result = true
+        @iter = 0
+        puts @token_position
+        puts @tokens
+        while (@result===true || @result===false) && (@tokens.size-@token_position>=1) && @iter<max_iterations
+          @result = parse
+          @iter = @iter + 1
+          # Pils::log ''
+          # Pils::log '+---+'
+          # Pils::log '+%3i+' % @iter
+          # Pils::log '+---+'
+
+        end
+        # Pils::log Pils::log '+  ---  +'
+        # Pils::log Pils::log @result
+        # Pils::log Pils::log '+  ---  +'
+        # Pils::log @result.class.name
+        if @result.kind_of?(Hash)
+          @tokens.each_with_index do |token,n|
+            @result[:syntax].set_wordform_at(n, token)
+          end
+        end
+        return @result
+      end
+
+      def parse
+        # categorial replacement?
+        # gibt es für das Blatt an aktueller Position eine Expansion?
+        # Pils::log '*' * 48
+        # Pils::log display_tokens
+        # Pils::log '*' * 48
+        # Pils::log @syntax.display
+        # Pils::log '*' * 48
+        # Pils::log @head_cat.obj if @head_cat
+        # Pils::log '*' * 48
+        # Pils::log @syntax.leaf_count
+        # Pils::log @syntax_position
+        # Pils::log @syntax.leaf_at(@syntax_position-1).obj unless @syntax_position==0
+        # Pils::log '*' * 48
+        sync
+        # Pils::log @syntax.leaf_count
+        # Pils::log @syntax.leaves
+        # Pils::log @syntax.leaves.count
+        # Pils::log @syntax.leaves.reject{|l| l.obj.nil? }
+        # Pils::log @syntax.leaves.reject{|l| l.obj.nil? }.count
+
+        # Pils::log @syntax_position
+        (0..(@syntax.leaf_count-1)).each do |u|
+          current_leaf = @syntax.leaf_at(u)
+          # Pils::log "  %2i %s" % [u, current_leaf]
+          # Pils::log "      %s" % [u, current_leaf.obj]
+        end
+        # stack:
+        # Pils::log "STACK:"
+        #@stack.each do |n|
+        #  Pils::log "    -- %2i %2i %s" % [n[2], n[3], n[0].display]
+        #end
+        # Pils::log @syntax.leaf_at(@syntax_position-1).obj unless @syntax_position==0
+        # Pils::log '*' * 48
+
+        # Pils::log "# A. CATEGORY EXPANSION"
+        # Pils::log @head_cat
+        # gib fehler aus, wenn nach sync keine head cat da ist
+        if @head_cat.nil? && !(@token_position > (@tokens.count-1))
+          retrieve_from_stack
+          return false
+        end
+
+        # Pils::log "     looking for expansions for #{@head_cat.obj}" if @head_cat
+
+        # Jetzt: Wenn es fuer diese Kategorie eine Ableitung gibt, fuehre diese durch.
+        success = false
+        if @head_cat
+          # Pils::log @grammar.expandible?(@head_cat.obj)
+          if @grammar.expandible?(@head_cat.obj)
+            # get first result.
+            # extend tree.
+            # set new tree as syntax.
+            expand_cat
+            success = true
+          end
+          return true if success
+          # Pils::log  Pils::log "KEINE CATEXPANSION MOEGLICH. Daher jetzt Wortabgleich."
+        end
+        # Wortabgleich!
+        # Passt das aktuelle Wort auf das momentane Token?
+        # Wenn ja: Setze beide Positionen eins weiter.
+        #   und trage die Semantik in die Semantikliste ein.
+        # Pils::log "# B. WORD EXPANSION"
+        # Pils::log "     looking for a word for #{@head_cat.obj}"
+        subs = word_substitutions
+        if subs.count > 0
+          # wir haben Wortersetzungen! Nimm die erste. Erzeuge einen
+          # neuen Zustand im aktuellen System.
+          # Setze die restlichen in die Warteliste.
+          head, *tail = *subs
+          if tail
+            tail.each do |t|
+              new_syntax = @syntax.clone
+              new_semantics = @semantics.clone
+              new_state = [new_syntax, new_semantics, @syntax_position, @token_position]
+              @stack << new_state
+            end
+            # Pils::log "Stack: %i" % @stack.size
+            # @stack.each do |n|
+            #   Pils::log "    -- %2i %2i %s" % [n[2], n[3], n[0].display]
+            # end
+          end
+          if head
+            # obtain semantics for word form
+            # get path descriptor from tree
+            # Pils::log "    --SEM- %s %s" % [head.form, head.semantics ]
+            semobj = @semantics
+            @head_path.each do |p|
+              semobj[p] = {} unless semobj.has_key?(p)
+              semobj = semobj[p]
+            end
+
+            head_path_descriptor = @head_path.collect{|h| "[:#{h}]"}.join('')
+            # Pils::log "HP : %s" % @head_path.to_s
+            # Pils::log "HPD: %s" % head_path_descriptor
+            # Pils::log @head_cat.ancestors.collect{|a| a.display}
+            # Pils::log @head_cat.ancestors_from_root.collect{|a| a.display}
+
+            head.semantics.keys.each do |key|
+              # Pils::log "      @semantics#{head_path_descriptor}[key] = head.semantics[key]"
+              instance_eval("@semantics#{head_path_descriptor}[key] = head.semantics[key]")
+              # semobj[key] = head[key]
+            end
+
+
+            @token_position = @token_position + 1
+            @syntax_position = @syntax_position + 1
+            sync
+            # TODO: Store semantics
+          end
+          success = true
+        end
+        # Pils::log "     now looking at #{@head_cat ? @head_cat.obj : 'END'}"
+        # Pils::log "     token pos %i / %i " % [@token_position, (@tokens.count-1)]
+        # check if end is nigh
+        if @head_cat.nil? && @token_position > (@tokens.count-1)
+          # Pils::log "END IS NIGH!"
+          # Pils::log @syntax.display
+          # Pils::log @semantics # JSON.pretty_generate(@semantics)
+          return {syntax: @syntax.clone, semantics: @semantics.clone}
+          # exit(0)
+        end
+        return true if success
+
+        retrieve_from_stack
+
+        # Pils::log @syntax.display
+        # Pils::log "Now expecting: %s" % @syntax.leaf_at(@syntax_position).obj.to_s
+
+        # Pils::log @stack.size
+        return success
+      end
+
+      # gets the next possible state from stack, discards the current one.
+      def retrieve_from_stack
+        # Pils::log "We retrieve the next object from the stack. %i" % @stack.size
+        return nil if @stack.nil? || @stack.empty?
+        new_state, *state_rest = @stack
+        @stack = state_rest
+        @syntax = new_state[0]
+        @semantics = new_state[1]
+        @syntax_position = new_state[2]
+        @token_position = new_state[3]
+        sync
+        # Pils::log "   becomes %i" % @stack.size
+        # @stack.each do |n|
+        #   Pils::log "    -- %2i %2i %s" % [n[2], n[3], n[0].display]
+        # end
+      end
+
+
+      def display_tokens
+        r = @tokens.slice(0,@token_position)
+        s = @tokens - r
+        t = [r, "*", s].flatten
+        t.join(" ")
+      end
+    end
+  end
+end