zenlish 0.2.05 → 0.2.07

data/lib/zenlish/lexer/lexer.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require_relative '../lex/literal'
+
+module Zenlish
+  module Lexer
+    class Lexer
+      # @return [StringScanner] Low-level scanner object.
+      attr_reader(:scanner)
+
+      # @return [Boolean] true, if lexer is currently busy to scan Zenlish.
+      attr_reader(:zenlish_mode)
+
+      # @return [Integer] Current line number (one-based)
+      attr_reader(:lineno)
+
+      # @return [Integer] Offset of start of current line within IO (one-based).
+      attr_reader(:line_start)
+
+      @@punct2name = {
+        ':' => 'Colon',
+        ',' => 'Comma',
+        '.' => 'Period',
+        '"' => 'Quote'
+      }.freeze
+
+      class ScanError < StandardError; end
+
+      # Constructor. Initialize a tokenizer for Skeem.
+      # @param source [String] Skeem text to tokenize.
+      def initialize(source)
+        @scanner = StringScanner.new('')
+        @zenlish_mode = true
+        reinitialize(source)
+      end
+
+      # @param source [String] Skeem text to tokenize.
+      def reinitialize(source)
+        @scanner.string = source
+        @lineno = 1
+        @line_start = 0
+      end
+
+      # @return [Array<Token>] | Returns a sequence of tokens
+      def tokens
+        tok_sequence = []
+        until @scanner.eos?
+          token = _next_token
+          tok_sequence << token unless token.nil?
+        end
+
+        return tok_sequence
+      end
+
+      private
+
+      def _next_token
+        skip_intertoken_spaces
+        if zenlish_mode
+          next_zenlish_token
+        else
+          next_json_token
+        end
+      end
+
+      def next_zenlish_token
+        curr_ch = scanner.peek(1)
+        return nil if curr_ch.nil? || curr_ch.empty?
+
+        token = nil
+
+        if ':,."'.include? curr_ch
+          # Delimiters, separators => single character token
+          token = build_token(@@punct2name[curr_ch], scanner.getch)
+        elsif (literal = scanner.scan(/[^\s:;,."]+/))
+          token = build_token('WORD', literal)
+        else # Unknown token
+          erroneous = curr_ch.nil? ? '' : scanner.scan(/./)
+          sequel = scanner.scan(/.{1,20}/)
+          erroneous += sequel unless sequel.nil?
+          raise ScanError, "Unknown token #{erroneous} on line #{lineno}"
+        end
+
+        return token
+      end
+
+      def build_token(aSymbolName, aLiteral, _format = :default)
+        begin
+          col = scanner.pos - aLiteral.size - @line_start + 1
+          pos = Rley::Lexical::Position.new(@lineno, col)
+          token = Rley::Lexical::Token.new(aLiteral, aSymbolName, pos)
+        rescue StandardError => e
+          puts "Failing with '#{aSymbolName}' and '#{aLiteral}'"
+          raise e
+        end
+
+        return token
+      end
+
+      def next_json_token
+        curr_ch = scanner.peek(1)
+        return nil if curr_ch.nil? || curr_ch.empty?
+
+        token = nil
+
+        if ':,."'.include? curr_ch
+          # Delimiters, separators => single character token
+          token = build_token(@@punct2name[curr_ch], scanner.getch)
+        elsif (literal = scanner.scan(/[^\s:;,."]+/))
+          token = build_token('WORD', literal)
+        else # Unknown token
+          erroneous = curr_ch.nil? ? '' : scanner.scan(/./)
+          sequel = scanner.scan(/.{1,20}/)
+          erroneous += sequel unless sequel.nil?
+          raise ScanError, "Unknown token #{erroneous} on line #{lineno}"
+        end
+
+        return token
+      end
+
+      def skip_intertoken_spaces
+        pre_pos = scanner.pos
+
+        loop do
+          ws_found = scanner.skip(/[ \t\f]+/) ? true : false
+          nl_found = scanner.skip(/(?:\r\n)|\r|\n/)
+          if nl_found
+            ws_found = true
+            next_line
+          end
+          break unless ws_found
+        end
+
+        curr_pos = scanner.pos
+        return if curr_pos == pre_pos
+      end
+
+      def next_line
+        @lineno += 1
+        @line_start = scanner.pos
+      end
+    end # class
+  end # module
+end # module

data/lib/zenlish/trie/base_trie_node.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Zenlish
+  module Trie
+    # Abstract class for representing nodes in the trie data structure.
+    class BaseTrieNode
+      # @return [Hash{String => TrieNode}] the successor nodes
+      attr_reader :succ
+
+      def initialize
+        @succ = {}
+      end
+
+      def include?(aKey)
+        succ.include?(aKey)
+      end
+
+      def add_succ(aKey, aNode)
+        succ[aKey] = aNode
+      end
+
+      def leaf?
+        succ.empty?
+      end
+    end # class
+  end # module
+end # module

data/lib/zenlish/trie/trie.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require_relative 'trie_root'
+require_relative 'trie_node'
+
+# https://raw.githubusercontent.com/first20hours/google-10000-english/master/20k.txt
+# https://raw.githubusercontent.com/dwyl/english-words/master/words_alpha.txt
+
+module Zenlish
+  module Trie
+    # A trie (aka prefix tree or digital tree) is a kind of search tree.
+    class Trie
+      #  @return [TrieRoot] The root node of the trie data structure
+      attr_reader :root
+
+      def initialize
+        @root = TrieRoot.new
+      end
+
+      def add(aWord, aValue)
+        append_node(root, aWord, 0, aValue)
+      end
+
+      def search(aWord)
+        search_node(root, aWord, 0)
+      end
+
+      private
+
+      def append_node(aNode, aText, anIndex, aValue)
+        key = aText[anIndex]
+        unless aNode.include?(key)
+          aNode.add_succ(key, TrieNode.new(key))
+        end
+        successor = aNode.succ[key]
+
+        if anIndex == aText.size - 1
+          current_value = successor.value
+          if current_value
+            if current_value.kind_of?(Array)
+              current_value << aValue
+            else
+             successor.value = [current_value, aValue]
+            end
+          else
+            successor.value = aValue
+          end
+        else
+          append_node(successor, aText, anIndex + 1, aValue)
+        end
+      end
+
+      def search_node(aNode, aText, anIndex)
+        key = aText[anIndex]
+        return nil unless aNode.include?(key)
+
+        successor = aNode.succ[key]
+        return successor if anIndex == aText.size - 1
+
+        search_node(successor, aText, anIndex + 1)
+      end
+=begin
+
+
+      def include?(word)
+        find_word(word) { |found, base| return found && base.word }
+      end
+
+      def add_word(word)
+        letters = word.chars
+        base    = @root
+
+        letters.each { |letter| base = add_character(letter, base.next) }
+
+        base.word = true
+      end
+
+      def find_word(word)
+        letters = word.chars
+        base    = @root
+
+        word_found =
+        letters.all? { |letter| base = find_character(letter, base.next) }
+
+        yield word_found, base if block_given?
+
+        base
+      end
+
+      def find_words_starting_with(prefix)
+        stack        = []
+        words        = []
+        prefix_stack = []
+
+        stack        << find_word(prefix)
+        prefix_stack << prefix.chars.take(prefix.size-1)
+
+        return [] unless stack.first
+
+        until stack.empty?
+          node = stack.pop
+
+          prefix_stack.pop and next if node == :guard_node
+
+          prefix_stack << node.value
+          stack        << :guard_node
+
+          words << prefix_stack.join if node.word
+
+          node.next.each { |n| stack << n }
+        end
+
+        words
+      end
+
+      private
+
+      def add_character(character, trie)
+        trie.find { |n| n.value == character } || add_node(character, trie)
+      end
+
+      def find_character(character, trie)
+        trie.find { |n| n.value == character }
+      end
+
+      def add_node(character, trie)
+        TrieNode.new(character).tap { |new_node| trie << new_node }
+      end
+=end
+    end # class
+  end # module
+end # module

data/lib/zenlish/trie/trie_node.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative 'base_trie_node'
+
+module Zenlish
+  module Trie
+    class TrieNode < BaseTrieNode
+      # @return [String] the matching character
+      attr_reader :key
+
+      # @return [Object, NilClass] Result when this character matches end-of-word.
+      attr_accessor :value
+
+      def initialize(aChar)
+        super()
+        @key = aChar
+        @value = nil
+      end
+    end # class
+  end # module
+end # module

data/lib/zenlish/trie/trie_root.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require_relative 'base_trie_node'
+
+module Zenlish
+  module Trie
+    class TrieRoot < BaseTrieNode
+    end # class
+  end # module
+end # module

data/lib/zenlish/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Zenlish
-  VERSION = '0.2.05'.freeze
+  VERSION = '0.2.07'
 end

data/lib/zenlish/wclasses/adjective.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'word_class'
 
 module Zenlish
@@ -7,8 +9,8 @@ module Zenlish
       # @return [FalseClass] Indicates that an adjective may inflect.
       def invariable?
         # TODO: set to false once we introduce the degrees (basic, comparative, superlative)
-        true 
-      end    
+        true
+      end
     end # class
   end # module
 end # module

data/lib/zenlish/wclasses/adverb.rb

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 require_relative 'word_class'
 
 module Zenlish
   module WClasses
     # TODO: document
-    class Adverb < WordClass   
+    class Adverb < WordClass
     end # class
   end # module
 end # module

data/lib/zenlish/wclasses/adverb_maybe.rb

@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 require_relative 'adverb'
 
 module Zenlish
   module WClasses
-    # The adverb maybe expresses uncertainty and is often placed in front 
+    # The adverb maybe expresses uncertainty and is often placed in front
     # of clauses.
     class AdverbMaybe < Adverb
     end # class

data/lib/zenlish/wclasses/adverb_not.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'adverb'
 
 module Zenlish

data/lib/zenlish/wclasses/all_word_classes.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Load the WordClass class hierarchy
 # Algorithm: load the leaf classes from hierarchy
 
@@ -43,4 +45,4 @@ require_relative 'fronting_quantifier'
 require_relative 'regular_verb'
 require_relative 'regular_verb_want'
 require_relative 'relative_pronoun'
-require_relative 'subordinating_conjunction'
+require_relative 'subordinating_conjunction'

data/lib/zenlish/wclasses/article.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'determiner'
 
 module Zenlish

data/lib/zenlish/wclasses/auxiliary.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'verb'
 
 module Zenlish

data/lib/zenlish/wclasses/auxiliary_be.rb

@@ -1,20 +1,20 @@
+# frozen_string_literal: true
+
 require_relative 'auxiliary'
 
 module Zenlish
   module WClasses
     class AuxiliaryBe < Auxiliary
-      def initialize
-        super()
-      end
+      # rubocop: disable Layout/SpaceInsideArrayLiteralBrackets
 
       private
 
       def init_feature_defs
         super
         # Create standard feature definitions for modal verb can.
-        feature_def_dsl {
+        feature_def_dsl do
           feature_def 'PARADIGM' => [identifier, 'Verb_be_inflection'] # 2nd item is default value
-        }
+        end
       end
 
       def init_paradigms
@@ -35,7 +35,8 @@ module Zenlish
           rule([dont_care,           dont_care,         equals(:past_participle)], literal('been'))
         end
         add_paradigm(table)
-      end    
+      end
+      # rubocop: enable Layout/SpaceInsideArrayLiteralBrackets
     end # class
   end # module
 end # module

data/lib/zenlish/wclasses/auxiliary_do.rb

@@ -1,21 +1,21 @@
+# frozen_string_literal: true
+
 require_relative 'auxiliary'
 
 module Zenlish
   module WClasses
     # TODO: document
     class AuxiliaryDo < Auxiliary
-      def initialize
-        super()
-      end
+      # rubocop: disable Layout/SpaceInsideArrayLiteralBrackets
 
       private
 
       def init_feature_defs
         super
         # Create standard feature definitions for modal verb can.
-        feature_def_dsl {
+        feature_def_dsl do
           feature_def 'PARADIGM' => [identifier, 'Verb_do_inflection'] # 2nd item is default value
-        }
+        end
       end
 
       def init_paradigms
@@ -34,6 +34,7 @@ module Zenlish
         end
         add_paradigm(table)
       end
+      # rubocop: enable Layout/SpaceInsideArrayLiteralBrackets
     end # class
   end # module
 end # module

data/lib/zenlish/wclasses/cardinal.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'numeral'
 
 module Zenlish
@@ -6,4 +8,4 @@ module Zenlish
     class Cardinal < Numeral
     end # class
   end # module
-end # module
+end # module

data/lib/zenlish/wclasses/common_noun.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'noun'
 require_relative '../inflect/inflection_table_builder'
 

data/lib/zenlish/wclasses/comparative_particle.rb

@@ -1,10 +1,12 @@
+# frozen_string_literal: true
+
 require_relative 'word_class'
 
 module Zenlish
   module WClasses
-    # TODO: document 
+    # TODO: document
     # Example: 'as' in 'same ... as'
-    class ComparativeParticle < WordClass   
+    class ComparativeParticle < WordClass
     end # class
   end # module
-end # module
+end # module

data/lib/zenlish/wclasses/conjunction.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'word_class'
 
 module Zenlish

data/lib/zenlish/wclasses/conjunctive_pronoun.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'pronoun'
 
 module Zenlish

data/lib/zenlish/wclasses/coordinator.rb

@@ -1,12 +1,14 @@
+# frozen_string_literal: true
+
 require_relative 'conjunction'
 
 module Zenlish
   module WClasses
-    # A coordinator (aka coordinating conjunction) is a word or phrase that 
+    # A coordinator (aka coordinating conjunction) is a word or phrase that
     # can join words, phrases, and clauses that share some logical relationhip
     # Common coordinators are enumerated in the FANBOYS acronym:
     # For, And, Nor, But, Or, Yet and So.
-    class Coordinator < WordClass    
+    class Coordinator < WordClass
     end # class
   end # module
 end # module

data/lib/zenlish/wclasses/definite_article.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'article'
 
 module Zenlish

data/lib/zenlish/wclasses/degree_adverb.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'adverb'
 
 module Zenlish

data/lib/zenlish/wclasses/demonstrative_determiner.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'determiner'
 require_relative '../inflect/inflection_table_builder'
 
@@ -10,7 +12,7 @@ module Zenlish
         super()
         init_feature_defs
         init_paradigms
-      end    
+      end
 
       # @return [FalseClass] Indicates that demonstrative determiners inflects.
       def invariable?
@@ -21,11 +23,11 @@ module Zenlish
 
       def init_feature_defs
         # Create standard feature definitions for irregular verbs.
-        feature_def_dsl {
+        feature_def_dsl do
           feature_def 'NUMBER' => enumeration(:singular, :plural)
           feature_def 'DISTANCE' => enumeration(:near, :far)
           feature_def 'PARADIGM' => [identifier, 'Demonstrative_det_paradigm'] # 2nd item is default value
-        }
+        end
       end
 
       def init_paradigms
@@ -37,7 +39,7 @@ module Zenlish
           rule([equals(:singular), equals(:near)],  func('base_form'))
           rule([equals(:singular), equals(:far)],   literal('that'))
           rule([equals(:plural),   equals(:near)],  literal('these'))
-          rule([equals(:plural),   equals(:far), ], literal('those'))
+          rule([equals(:plural),   equals(:far)],   literal('those'))
         end
         add_paradigm(table)
       end

data/lib/zenlish/wclasses/demonstrative_pronoun.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'pronoun'
 
 module Zenlish

data/lib/zenlish/wclasses/determiner.rb

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 require_relative 'word_class'
 
 module Zenlish
   module WClasses
     # TODO: document
-    class Determiner < WordClass   
+    class Determiner < WordClass
     end # class
   end # module
 end # module

data/lib/zenlish/wclasses/distributive_determiner.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'determiner'
 
 module Zenlish

data/lib/zenlish/wclasses/existential_there.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'pronoun'
 
 module Zenlish

data/lib/zenlish/wclasses/fronting_quantifier.rb

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 require_relative 'quantifier'
 
 module Zenlish
   module WClasses
     # Quantifiers that appear in front of determiners: all, both
-    class  FrontingQuantifier < Quantifier    
+    class FrontingQuantifier < Quantifier
     end # class
   end # module
 end # module

data/lib/zenlish/wclasses/indefinite_article.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'article'
 
 module Zenlish

data/lib/zenlish/wclasses/indefinite_pronoun.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'pronoun'
 
 module Zenlish
@@ -6,4 +8,4 @@ module Zenlish
     class IndefinitePronoun < Pronoun
     end # class
   end # module
-end # module
+end # module