zenlish 0.2.02 → 0.2.06

data/lib/zenlish/inflect/unconditionally_true.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'nullary_input_expression'
 
 module Zenlish
@@ -5,7 +7,7 @@ module Zenlish
     class UnconditionallyTrue < NullaryInputExpression
       def success?(_headings, _lexeme, _heading_values)
         true
-      end     
+      end
     end # class
   end # module
-end # module
+end # module

data/lib/zenlish/lang/dictionary.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 unless defined?(Zenlish::Lang::Dictionary)
   require_relative '../feature/feature_struct_def_bearer'
   module Zenlish
@@ -9,15 +11,16 @@ unless defined?(Zenlish::Lang::Dictionary)
       sandbox = Object.new
       sandbox.extend(Zenlish::Lex::EmptyLexiconFactory)
       Dictionary = sandbox.create_empty_lexicon
-      self.extend(Feature::FeatureStructDefBearer)
+      extend(Feature::FeatureStructDefBearer)
 
-      # @param aLemma [String] is the canonical form, dictionary form, 
+      # @param aLemma [String] is the canonical form, dictionary form,
       #   or citation form of a headword.
       # @param aWClassName [String] the name of a word class.
       def self.add_entry(aLemma, aWClassName, aFeatureHash = nil, &aBlock)
         entry = Zenlish::Lex::LexicalEntry.new(aLemma)
         wclass = Dictionary.name2terminal[aWClassName]
         raise StandardError, "Undefined word class for '#{aLemma}'" unless wclass
+
         lexeme = Zenlish::Lex::Lexeme.new(wclass, entry, aFeatureHash)
         lexeme.instance_exec(&aBlock) if block_given?
         lexeme.freeze
@@ -56,7 +59,7 @@ unless defined?(Zenlish::Lang::Dictionary)
       add_entry('big', 'Adjective')
       add_entry('body', 'CommonNoun')
       add_entry('but', 'Coordinator')
-      add_entry('can', 'IrregularVerb')      
+      add_entry('can', 'IrregularVerbCan')
       add_entry('can', 'ModalVerbCan')
       add_entry('cause', 'RegularVerb')
       add_entry('change', 'RegularVerb')
@@ -65,15 +68,12 @@ unless defined?(Zenlish::Lang::Dictionary)
       end
       add_entry('contain', 'RegularVerb')
       add_entry('container', 'CommonNoun')
-      add_entry('could', 'ModalVerbCould')
-      add_entry('damage', 'RegularVerb')      
+      add_entry('damage', 'RegularVerb')
       add_entry('die', 'RegularVerb')
-      add_entry('difficult', 'Adjective')      
+      add_entry('difficult', 'Adjective')
       add_entry('different', 'Adjective')
       add_entry('do', 'AuxiliaryDo')
-      add_entry('do', 'IrregularVerbDo') do
-        forms past_simple: 'did', past_participle: 'done'
-      end
+      add_entry('do', 'IrregularVerbDo')
       add_entry('each', 'DistributiveDeterminer')
       add_entry('each', 'Pronoun')
       add_entry('exist', 'RegularVerb')
@@ -88,17 +88,22 @@ unless defined?(Zenlish::Lang::Dictionary)
       add_entry('good', 'Adjective')
       add_entry('have', 'IrregularVerbHave')
       add_entry('happen', 'RegularVerb')
-      add_entry('hear', 'IrregularLinkingVerb')
+      add_entry('hear', 'IrregularLinkingVerb') do
+        forms past_simple: 'heard', past_participle: 'heard'
+      end
       add_entry('here', 'Adverb')
       # example: ...from here (works as a pronoun of a place)
-      add_entry('here', 'CommonNoun', {'NUMBER' => enumeration(:singular),
-        'PARADIGM' => [identifier, 'Singular_only']})
-      add_entry('I', 'PersonalPronoun')
+      add_entry('here', 'CommonNoun', { 'NUMBER' => enumeration(:singular),
+        'PARADIGM' => [identifier, 'Singular_only'] })
+      add_entry('I', 'PersonalPronoun', { 'PERSON' => enumeration(:first),
+        'GENDER' => enumeration(:feminine, :masculine) })
       add_entry('if', 'SubordinatingConjunction')
       add_entry('in', 'Preposition')
       add_entry('inside', 'Preposition')
-      add_entry('it', 'PersonalPronoun')
-      add_entry('its', 'PossessiveDeterminer')
+      add_entry('it', 'PersonalPronoun', { 'PERSON' => enumeration(:third),
+        'PARADIGM' => [identifier, 'ppn_3rd_paradigm'] })
+      add_entry('its', 'PossessiveDeterminer', { 'PERSON' => enumeration(:third),
+        'PARADIGM' => [identifier, 'possdet_3rd_paradigm'] })
       add_entry('kind', 'CommonNoun')
       add_entry('know', 'IrregularVerbKnow') do
         forms past_simple: 'knew', past_participle: 'known'
@@ -113,18 +118,18 @@ unless defined?(Zenlish::Lang::Dictionary)
       end
       add_entry('many', 'Quantifier')
       add_entry('maybe', 'AdverbMaybe')
-      add_entry('me', 'PersonalPronoun')
       add_entry('moment', 'CommonNoun')
       add_entry('more', 'Adjective')
       add_entry('more', 'Adverb')
       add_entry('move', 'RegularVerb')
       add_entry('much', 'Adverb')
-      add_entry('my', 'PossessiveDeterminer')
+      add_entry('my', 'PossessiveDeterminer', { 'PERSON' => enumeration(:first),
+        'GENDER' => enumeration(:feminine, :masculine) })
       add_entry('near', 'Preposition')
       add_entry('near to', 'Preposition')
       add_entry('now', 'Adverb')
-      add_entry('now', 'CommonNoun', {'NUMBER' => enumeration(:singular),
-        'PARADIGM' => [identifier, 'Singular_only']})
+      add_entry('now', 'CommonNoun', { 'NUMBER' => enumeration(:singular),
+        'PARADIGM' => [identifier, 'Singular_only'] })
       add_entry('not', 'AdverbNot')
       add_entry('of', 'PrepositionOf')
       add_entry('on', 'Preposition')
@@ -134,10 +139,10 @@ unless defined?(Zenlish::Lang::Dictionary)
       add_entry('or', 'Coordinator')
       add_entry('other', 'Adjective')
       add_entry('part', 'CommonNoun')
-      add_entry('people', 'CommonNoun', {'NUMBER' => enumeration(:plural), 
-        'PARADIGM' => [identifier, 'Plural_only']})
-      add_entry('person', 'CommonNoun', {'NUMBER' => enumeration(:singular),
-        'PARADIGM' => [identifier, 'Singular_only']})
+      add_entry('people', 'CommonNoun', { 'NUMBER' => enumeration(:plural),
+        'PARADIGM' => [identifier, 'Plural_only'] })
+      add_entry('person', 'CommonNoun', { 'NUMBER' => enumeration(:singular),
+        'PARADIGM' => [identifier, 'Singular_only'] })
       add_entry('place', 'CommonNoun')
       add_entry('same', 'Adjective')
       add_entry('same', 'Pronoun')
@@ -158,11 +163,8 @@ unless defined?(Zenlish::Lang::Dictionary)
       add_entry('than', 'PrepositionThan')
       add_entry('that', 'RelativePronoun')
       add_entry('the', 'DefiniteArticle')
-      add_entry('them', 'PersonalPronoun')
       add_entry('then', 'LinkingAdverb')
-      add_entry('their', 'PossessiveDeterminer')
       add_entry('there', 'ExistentialThere')
-      add_entry('they', 'PersonalPronoun')
       add_entry('thing', 'CommonNoun')
       add_entry('think', 'IrregularVerbThink') do
         forms past_simple: 'thought', past_participle: 'thought'
@@ -187,8 +189,12 @@ unless defined?(Zenlish::Lang::Dictionary)
       add_entry('who', 'RelativePronoun')
       add_entry('with', 'Preposition')
       add_entry('word', 'CommonNoun')
-      add_entry('you', 'PersonalPronoun')
-      add_entry('your', 'PossessiveDeterminer')
+      add_entry('you', 'PersonalPronoun', { 'PERSON' => enumeration(:second),
+            'GENDER' => enumeration(:feminine, :masculine),
+            'PARADIGM' => [identifier, 'ppn_2nd_paradigm'] })
+      add_entry('your', 'PossessiveDeterminer', { 'PERSON' => enumeration(:second),
+        'GENDER' => enumeration(:feminine, :masculine),
+        'PARADIGM' => [identifier, 'possdet_2nd_paradigm'] })
 
       # Punctuation signs...
       add_entry(':', 'Colon')
@@ -197,4 +203,4 @@ unless defined?(Zenlish::Lang::Dictionary)
       add_entry('"', 'Quote')
     end # module
   end # module
-end # defined?
+end # defined?

data/lib/zenlish/lang/lemmatizer.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative '../trie/trie'
+
+module Zenlish
+  module Lang
+    class Lemmatizer
+      # @return [Trie:Trie] Trie (aka prefix tree) with all word forms from dictionary.
+      attr_reader :trie
+
+      def initialize(aLexicon)
+        @trie = Trie::Trie.new
+        initialize_trie(aLexicon)
+      end
+
+      def lemmatize(aWordForm, _hints = nil)
+        node = trie.search(aWordForm)
+        node&.value
+      end
+
+      private
+
+      def initialize_trie(aLexicon)
+        aLexicon.entries.each do |ent|
+          citation_form = ent.lemma
+          ent.lexemes.each do |lxm|
+            if lxm.wclass.kind_of?(Zenlish::WClasses::WordClass)
+              if lxm.wclass.invariable?
+                trie.add(citation_form, lxm)
+              else
+                w_forms = lxm.all_inflections
+                w_forms.each { |form| trie.add(form, lxm) }
+              end
+            end
+          end
+        end
+      end
+    end # class
+  end # module
+end # module

data/lib/zenlish/lang/zenlish_grammar.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Grammar for a simple subset of English language
 # It is called Zenlish
 
@@ -6,7 +8,7 @@ require_relative 'dictionary'
 
 ########################################
 # Define a grammar for a highly English-like language
-builder = Rley::Syntax::GrammarBuilder.new do
+builder = Rley::grammar_builder do
   add_terminals(*Zenlish::Lang::Dictionary.terminals)
 
   rule 'zenlish' => 'prose'
@@ -19,7 +21,7 @@ builder = Rley::Syntax::GrammarBuilder.new do
   #################
   # Simple sentence
   #################
-  rule 'simple_sentence' =>  'front_adverb simple_sentence'
+  rule 'simple_sentence' => 'front_adverb simple_sentence'
   rule 'front_adverb' => 'AdverbMaybe'
   rule 'front_adverb' => 'Adverb'
   rule 'simple_sentence' => 'declarative_simple_sentence'
@@ -107,7 +109,7 @@ builder = Rley::Syntax::GrammarBuilder.new do
   rule 'comparative_start' => 'ComparativeParticle'
   rule 'conjunctive_prefix' => 'ConjunctivePronoun noun_phrase verb_phrase'
   rule 'identifying_clause' => 'RelativePronoun tense_verb_phrase'
-  rule 'relative_clause_opt' =>  'relative_clause'
+  rule 'relative_clause_opt' => 'relative_clause'
   rule 'relative_clause_opt' => []
   rule 'relative_clause' => 'RelativePronoun tense_phrase'
   # Sentence 3-Bxa 'Lisa sees a living thing that is very big.
@@ -137,7 +139,7 @@ builder = Rley::Syntax::GrammarBuilder.new do
   rule 'tense' => 'AuxiliaryBe'
   rule 'tense' => 'AuxiliaryDo'
   rule 'tense' => 'ModalVerbCan'
-  rule 'tense' => 'ModalVerbCould'
+
 
   #############
   # NOUN PHRASE
@@ -232,12 +234,13 @@ builder = Rley::Syntax::GrammarBuilder.new do
   rule 'lexical_verb' => 'IrregularVerb'
   rule 'lexical_verb' => 'IrregularLinkingVerb'
   rule 'lexical_verb' => 'IrregularVerbBe'
+  rule 'lexical_verb' => 'IrregularVerbCan'
   rule 'lexical_verb' => 'IrregularVerbDo'
   rule 'lexical_verb' => 'IrregularVerbHave'
   rule 'lexical_verb' => 'IrregularVerbKnow'
   rule 'lexical_verb' => 'IrregularVerbSay'
   rule 'lexical_verb' => 'IrregularVerbThink'
- 
+
 
   rule 'linking_verb' => 'IrregularLinkingVerb'
 
@@ -293,7 +296,7 @@ builder = Rley::Syntax::GrammarBuilder.new do
   # complementation by a verb: gerund -ing form...
   rule 'preposition_object' => 'noun_phrase_opt lexical_verb post_head_vp'
   # preposition_object => "a gerund (a verb form ending in "-ing") that acts as a noun # Example: He beat Lee without overly trying.
-  rule 'preposition_object' => 'conjunctive_prefix'		# It's obvious from _what he said_.
+  rule 'preposition_object' => 'conjunctive_prefix'	# It's obvious from _what he said_.
   rule 'preposition_object' => []
 
   ######################

data/lib/zenlish/lex/empty_lexicon_factory.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../wclasses/all_word_classes'
 require_relative 'lexicon'
 
@@ -7,7 +9,7 @@ module Zenlish
       # Factory method. Helps in creating an "empty" lexicon.
       # It just contains the word classes of Zenlish but no headwords.
       # @return [Lexicon] the created lexicon object
-      def create_empty_lexicon()
+      def create_empty_lexicon
         lexicon = Lexicon.new
 
         add_word_classes(lexicon)
@@ -41,6 +43,7 @@ module Zenlish
         add_wclass(aLexicon, WClasses::IrregularLinkingVerb)
         add_wclass(aLexicon, WClasses::IrregularVerb)
         add_wclass(aLexicon, WClasses::IrregularVerbBe)
+        add_wclass(aLexicon, WClasses::IrregularVerbCan)
         add_wclass(aLexicon, WClasses::IrregularVerbDo)
         add_wclass(aLexicon, WClasses::IrregularVerbHave)
         add_wclass(aLexicon, WClasses::IrregularVerbKnow)
@@ -48,7 +51,6 @@ module Zenlish
         add_wclass(aLexicon, WClasses::IrregularVerbThink)
         add_wclass(aLexicon, WClasses::LinkingAdverb)
         add_wclass(aLexicon, WClasses::ModalVerbCan)
-        add_wclass(aLexicon, WClasses::ModalVerbCould)
         add_wclass(aLexicon, WClasses::PersonalPronoun)
         add_wclass(aLexicon, WClasses::PossessiveDeterminer)
         add_wclass(aLexicon, WClasses::Preposition)
@@ -73,7 +75,6 @@ module Zenlish
       def add_wclass(aLexicon, aClass)
         aLexicon.add_terminal(aClass.new.freeze)
       end
-
     end # module
   end # module
 end # module

data/lib/zenlish/lex/lexeme.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../feature/feature_struct_def_bearer'
 module Zenlish
   module Lex
@@ -20,7 +22,7 @@ module Zenlish
         anEntry.add_lexeme(self)
         if aWClass.kind_of?(WClasses::WordClass)
           unless wclass.extension.nil?
-            self.extend(wclass.extension)
+            extend(wclass.extension)
             init_extension(self)
           end
           p_struct = aWClass.kind_of?(WClasses::WordClass) ? aWClass.struct : nil
@@ -42,6 +44,11 @@ module Zenlish
         table.inflect(self, constraints)
       end
 
+      def all_inflections
+        table = paradigm
+        table.all_inflections(self)
+      end
+
       # @return [String] the base (dictionary) form.
       def lemma
         entry.lemma
@@ -57,4 +64,4 @@ module Zenlish
       alias base_form lemma
     end # class
   end # module
-end # module
+end # module

data/lib/zenlish/lex/lexical_entry.rb

@@ -1,14 +1,16 @@
+# frozen_string_literal: true
+
 module Zenlish
   module Lex
     # TODO: document
     class LexicalEntry
-      #@return [String] the lemma (dictionary) form of a word.
+      # @return [String] the lemma (dictionary) form of a word.
       attr_reader :lemma
 
       # @return [Array<Lexeme, Rley::Syntax::Terminal>]
       attr_reader :lexemes
 
-      # @param theLemma [String] lemma (= citation form), a word form used 
+      # @param theLemma [String] lemma (= citation form), a word form used
       #   conventionnaly to represent a lexeme.
       # @param aLexeme [Lexeme, NilClass] the lexeme to link with lexical entry.
       def initialize(theLemma, aLexeme = nil)
@@ -17,10 +19,10 @@ module Zenlish
         add_lexeme(aLexeme)
       end
 
-      # @param aLexeme [Lexeme, NilClass] the lexeme to link with lexical entry.      
+      # @param aLexeme [Lexeme, NilClass] the lexeme to link with lexical entry.
       def add_lexeme(aLexeme)
         lexemes << aLexeme if aLexeme
       end
     end # class
   end # module
-end # module
+end # module

data/lib/zenlish/lex/lexicon.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Zenlish
   module Lex
     # A lexicon is a collection of lexical entries.
@@ -5,7 +7,7 @@ module Zenlish
     class Lexicon
       # @return [Array<Lex::LexicalEntry>] entries in the lexicon
       attr_reader :entries
-      
+
       # @return [Hash{String => Lex::LexicalEntry}] the lexical entry for the given lemma.
       attr_reader :lemma2entry
 
@@ -23,7 +25,7 @@ module Zenlish
       end
 
       # @param aLemma[String] retrieve the lexeme form the given "head word".
-      # @param aWordClass [WordClasses::WordClass, NilClass] the word class of 
+      # @param aWordClass [WordClasses::WordClass, NilClass] the word class of
       #   the lexeme.
       # @return [Lex::Lexeme]
       def get_lexeme(aLemma, aWordClass = nil)
@@ -45,11 +47,12 @@ module Zenlish
 
           lexeme
         else
-          begin
-            lemma2entry.fetch(aLemma).lexemes.first
-          rescue NoMethodError => exc
-            $stderr.puts "Multiple lexemes for #{aLemma}"
-            raise exc
+          entry = lemma2entry.fetch(aLemma)
+          if entry.kind_of?(Array)
+            err_msg = "Multiple lexemes for #{aLemma}"
+            raise StandardError, err_msg
+          else
+            entry.lexemes.first
           end
         end
       end
@@ -82,7 +85,6 @@ module Zenlish
           aHash[aKey] = aValue
         end
       end
-
     end # class
   end # module
 end # module

data/lib/zenlish/lex/literal.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rley'
 
 module Zenlish
@@ -7,7 +9,7 @@ module Zenlish
       attr_reader :zlexeme
 
       # initialize(theLexeme, aTerminal, aPosition) ⇒ Token
-      # @param literalText [String] the portion of input text that represents 
+      # @param literalText [String] the portion of input text that represents
       #   an occurence of the lexeme.
       # @param aLexeme [Lex::Lexeme] the lexeme matched by the literal text.
       # @param aPosition [Integer] the position of the literal in the input.

data/lib/zenlish/lexer/lexer.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require_relative '../lex/literal'
+
+module Zenlish
+  module Lexer
+    class Lexer
+      # @return [StringScanner] Low-level scanner object.
+      attr_reader(:scanner)
+
+      # @return [Boolean] true, if lexer is currently busy to scan Zenlish.
+      attr_reader(:zenlish_mode)
+
+      # @return [Integer] Current line number (one-based)
+      attr_reader(:lineno)
+
+      # @return [Integer] Offset of start of current line within IO (one-based).
+      attr_reader(:line_start)
+
+      @@punct2name = {
+        ':' => 'Colon',
+        ',' => 'Comma',
+        '.' => 'Period',
+        '"' => 'Quote'
+      }.freeze
+
+      class ScanError < StandardError; end
+
+      # Constructor. Initialize a tokenizer for Skeem.
+      # @param source [String] Skeem text to tokenize.
+      def initialize(source)
+        @scanner = StringScanner.new('')
+        @zenlish_mode = true
+        reinitialize(source)
+      end
+
+      # @param source [String] Skeem text to tokenize.
+      def reinitialize(source)
+        @scanner.string = source
+        @lineno = 1
+        @line_start = 0
+      end
+
+      # @return [Array<Token>] | Returns a sequence of tokens
+      def tokens
+        tok_sequence = []
+        until @scanner.eos?
+          token = _next_token
+          tok_sequence << token unless token.nil?
+        end
+
+        return tok_sequence
+      end
+
+      private
+
+      def _next_token
+        skip_intertoken_spaces
+        if zenlish_mode
+          next_zenlish_token
+        else
+          next_json_token
+        end
+      end
+
+      def next_zenlish_token
+        curr_ch = scanner.peek(1)
+        return nil if curr_ch.nil? || curr_ch.empty?
+
+        token = nil
+
+        if ':,."'.include? curr_ch
+          # Delimiters, separators => single character token
+          token = build_token(@@punct2name[curr_ch], scanner.getch)
+        elsif (literal = scanner.scan(/[^\s:;,."]+/))
+          token = build_token('WORD', literal)
+        else # Unknown token
+          erroneous = curr_ch.nil? ? '' : scanner.scan(/./)
+          sequel = scanner.scan(/.{1,20}/)
+          erroneous += sequel unless sequel.nil?
+          raise ScanError, "Unknown token #{erroneous} on line #{lineno}"
+        end
+
+        return token
+      end
+
+      def build_token(aSymbolName, aLiteral, _format = :default)
+        begin
+          col = scanner.pos - aLiteral.size - @line_start + 1
+          pos = Rley::Lexical::Position.new(@lineno, col)
+          token = Rley::Lexical::Token.new(aLiteral, aSymbolName, pos)
+        rescue StandardError => e
+          puts "Failing with '#{aSymbolName}' and '#{aLiteral}'"
+          raise e
+        end
+
+        return token
+      end
+
+      def next_json_token
+        curr_ch = scanner.peek(1)
+        return nil if curr_ch.nil? || curr_ch.empty?
+
+        token = nil
+
+        if ':,."'.include? curr_ch
+          # Delimiters, separators => single character token
+          token = build_token(@@punct2name[curr_ch], scanner.getch)
+        elsif (literal = scanner.scan(/[^\s:;,."]+/))
+          token = build_token('WORD', literal)
+        else # Unknown token
+          erroneous = curr_ch.nil? ? '' : scanner.scan(/./)
+          sequel = scanner.scan(/.{1,20}/)
+          erroneous += sequel unless sequel.nil?
+          raise ScanError, "Unknown token #{erroneous} on line #{lineno}"
+        end
+
+        return token
+      end
+
+      def skip_intertoken_spaces
+        pre_pos = scanner.pos
+
+        loop do
+          ws_found = scanner.skip(/[ \t\f]+/) ? true : false
+          nl_found = scanner.skip(/(?:\r\n)|\r|\n/)
+          if nl_found
+            ws_found = true
+            next_line
+          end
+          break unless ws_found
+        end
+
+        curr_pos = scanner.pos
+        return if curr_pos == pre_pos
+      end
+
+      def next_line
+        @lineno += 1
+        @line_start = scanner.pos
+      end
+    end # class
+  end # module
+end # module