bracket_notation 1.0.3

data/lib/bracket_notation/parser.rb

@@ -0,0 +1,99 @@
+#--
+# This file is part of BracketNotation.
+# 
+# BracketNotation is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# BracketNotation is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with BracketNotation.  If not, see <http://www.gnu.org/licenses/>.
+#++
+# BracketNotation is a parser for generating syntax trees from sentences
+# annotated with the kind of bracket notation that is commonly used by
+# linguists. The result is a tree structure with nodes that describe the phrases
+# and constituents of the sentence.
+#
+# BracketNotation was inspired by Yoichiro Hasebe's RSyntaxTree[http://yohasebe.com/rsyntaxtree/],
+# and small portions of his code have been incorporated in the parser.
+#
+# Author:: Cody Brimhall (mailto:brimhall@somuchwit.com)
+# Copyright:: Copyright (c) 2010 Cody Brimhall
+# License:: Distributed under the terms of the GNU General Public License, v. 3
+
+module BracketNotation # :nodoc:
+  # This class represents a parser for sentences annotated with the kind of
+  # bracket notation that is commonly used by linguists. After being checked for
+  # obvious problems, the input string is scanned for tokens, which are
+  # evaluated to produce an expression tree.
+  class Parser
+    attr_reader :input
+    
+    # This class is an Exception subclass that reports errors in the input
+    # validation process.
+    class ValidationError < RuntimeError; end
+    
+    # Saves the input string, as well as a copy of the input string that has
+    # been normalized and validated.
+    def initialize(input)
+      validation_error "Parser input cannot be nil" if input.nil?
+      
+      @input = input
+      @data = scrub(input)
+      validate
+    end
+    
+    # Scans and evaluates the input string, returning an expression tree.
+    def parse
+      scanner = Scanner.new(@data)
+      evaluator = Evaluator.new(scanner.scan)
+      expression = evaluator.evaluate
+    end
+    
+    private
+    
+    # Normalizes the input string to make it easier to parse.
+    def scrub(str)
+      output = str.gsub(/\t/, "")
+      output.gsub!(/\s+/, " ")
+      output.gsub!(/\] \[/, "][")
+      output.gsub!(/ \[/, "[")
+      
+      return output
+    end
+    
+    # Checks to see if the input is valid, i.e. it has a length, no unnamed
+    # nodes, and the bracket-nesting is balanced.
+    def validate
+      validation_error("Input string can't be empty.") if @data.length < 1
+      validation_error("All opening brackets must have a label.") if /\[\s*\[/ =~ @data
+      
+      # Count the opening and closing brackets to make sure they're balanced
+      chars = @data.gsub(/[^\[\]]/, "").split(//)
+      validation_error("Opening and closing brackets must be balanced.") if chars.length % 2 != 0
+      
+      open_count, close_count = 0, 0
+      
+      chars.each do |char|
+        case char
+          when '[': open_count += 1
+          when ']': close_count += 1
+        end
+        
+        break if open_count < close_count
+      end
+      
+      validation_error("Opening and closing brackets must be properly nested.") if open_count != close_count
+    end
+    
+    # Raises a validation exception with the given message
+    def validation_error(message)
+      raise ValidationError, message
+    end
+  end
+end

data/lib/bracket_notation/scanner.rb

@@ -0,0 +1,129 @@
+#--
+# This file is part of BracketNotation.
+# 
+# BracketNotation is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# BracketNotation is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with BracketNotation.  If not, see <http://www.gnu.org/licenses/>.
+#++
+# BracketNotation is a parser for generating syntax trees from sentences
+# annotated with the kind of bracket notation that is commonly used by
+# linguists. The result is a tree structure with nodes that describe the phrases
+# and constituents of the sentence.
+#
+# BracketNotation was inspired by Yoichiro Hasebe's RSyntaxTree[http://yohasebe.com/rsyntaxtree/],
+# and small portions of his code have been incorporated in the parser.
+#
+# Author:: Cody Brimhall (mailto:brimhall@somuchwit.com)
+# Copyright:: Copyright (c) 2010 Cody Brimhall
+# License:: Distributed under the terms of the GNU General Public License, v. 3
+
+module BracketNotation # :nodoc:
+  # This class represents a scanner for sentences annotated with the kind of
+  # bracket notation that is commonly used by linguists. The scanner reads the
+  # input string and generates a list of Token instances.
+  class Scanner
+    include Enumerable
+    
+    attr_reader :input
+    
+    UNRESERVED_CHARACTER = /^[^\[\]\s]$/
+    LBRACKET_CHARACTER = "["
+    RBRACKET_CHARACTER = "]"
+    EOL_CHARACTER = nil
+    
+    # Saves the input string.
+    def initialize(input)
+      @input = input
+      @pos = 0
+      @chunk_size = 1
+      @last_read = "\n"
+      @tokens = nil
+    end
+    
+    # Returns an array of all the tokens produced by the scanner.
+    def scan
+      return @tokens unless @tokens.nil?
+      
+      @tokens = []
+      token = nil
+      @tokens << token while (token = next_token)
+      
+      return @tokens
+    end
+    
+    # Enumerates the list of tokens, passing each in turn to the provided block.
+    def each(&block)
+      scan.each &block
+    end
+    
+    private
+    
+    # Generate and return the next token in the token stream.
+    def next_token
+      return nil if @last_read.nil?
+      
+      # Scan the input string for the next token, ignoring white space (and
+      # anything else that isn't a recognized character)
+      token = nil
+      while(token.nil?)
+        token = case read_char
+          when UNRESERVED_CHARACTER: name_token
+          when LBRACKET_CHARACTER: Token.LBRACKET
+          when RBRACKET_CHARACTER: Token.RBRACKET
+          when EOL_CHARACTER: Token.EOL
+          else nil
+        end
+      end
+      
+      return token
+    end
+    
+    # Read a single character and update the position pointer.
+    def read_char
+      return @last_read if @last_read.nil? # Already at end of line
+      
+      @last_read = input[@pos, @chunk_size]
+      @pos += @chunk_size
+      
+      return @last_read || Token::EOL
+    end
+    
+    # Look ahead to see what the next char will be, without updating @last_read
+    # or the position pointer.
+    def peek_char
+      if @last_read.nil?
+        return @last_read
+      end
+      
+      return input[@pos, @chunk_size]
+    end
+    
+    # Gobble up the string of unreserved characters to make a name token.
+    def name_token
+      value = String.new(@last_read)
+      
+      # Read through the subsequent unreserved characters to build the name token.
+      while(peek_char =~ UNRESERVED_CHARACTER)
+        value << read_char
+      end
+      
+      return Token.NAME(value)
+    end
+    
+    # Go back to the beginning of the input string and prepare to generate the.
+    # tokens again
+    def reset
+      @pos = 0
+      @last_read = "\n"
+    end
+  end
+end

data/lib/bracket_notation/token.rb

@@ -0,0 +1,79 @@
+#--
+# This file is part of BracketNotation.
+# 
+# BracketNotation is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# BracketNotation is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with BracketNotation.  If not, see <http://www.gnu.org/licenses/>.
+#++
+# BracketNotation is a parser for generating syntax trees from sentences
+# annotated with the kind of bracket notation that is commonly used by
+# linguists. The result is a tree structure with nodes that describe the phrases
+# and constituents of the sentence.
+#
+# BracketNotation was inspired by Yoichiro Hasebe's RSyntaxTree[http://yohasebe.com/rsyntaxtree/],
+# and small portions of his code have been incorporated in the parser.
+#
+# Author:: Cody Brimhall (mailto:brimhall@somuchwit.com)
+# Copyright:: Copyright (c) 2010 Cody Brimhall
+# License:: Distributed under the terms of the GNU General Public License, v. 3
+
+module BracketNotation # :nodoc:
+  # This class represents a token in a stream of characters. All tokens have a
+  # type, and some of them (e.g. +NAME+ tokens) have corresponding values.
+  class Token
+    attr_reader :type
+    attr_reader :value
+    
+    # Constants that identify the different types of tokens
+    LBRACKET = "LBRACKET"
+    RBRACKET = "RBRACKET"
+    NAME = "NAME"
+    EOL = "EOL"
+    
+    # Initializes and returns a new token of type +LBRACKET+.
+    def self.LBRACKET; return self.new(LBRACKET); end
+    
+    # Initializes and returns a new token of type +RBRACKET+.
+    def self.RBRACKET; return self.new(RBRACKET); end
+    
+    # Initializes and returns a new token of type +NAME+ with the given value.
+    def self.NAME(value); return self.new(NAME, value); end
+    
+    # Initializes and returns a new token of type +EOL+.
+    def self.EOL; return self.new(EOL); end
+    
+    # Saves the token type, as well as an optional value.
+    def initialize(type, value = nil)
+      @type = type
+      @value = value
+    end
+    
+    # Provides a human-friendly string representation of a token instance.
+    def inspect # :nodoc:
+      output = "#{@type}"
+      output << " \"#{@value}\"" unless @value.nil?
+      
+      return output
+    end
+    
+    # Compares the receiver with another object, returning true only if the
+    # other object is also an instance of Token, and only if the two tokens
+    # share the same +type+ and +value+.
+    def ==(rvalue)
+      if self.class != rvalue.class
+        return super
+      end
+      
+      return @type == rvalue.type && @value == rvalue.value
+    end
+  end
+end

data/lib/bracket_notation/version.rb

@@ -0,0 +1,40 @@
+#--
+# This file is part of BracketNotation.
+# 
+# BracketNotation is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# BracketNotation is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with BracketNotation.  If not, see <http://www.gnu.org/licenses/>.
+#++
+# BracketNotation is a parser for generating syntax trees from sentences
+# annotated with the kind of bracket notation that is commonly used by
+# linguists. The result is a tree structure with nodes that describe the phrases
+# and constituents of the sentence.
+#
+# BracketNotation was inspired by Yoichiro Hasebe's RSyntaxTree[http://yohasebe.com/rsyntaxtree/],
+# and small portions of his code have been incorporated in the parser.
+#
+# Author:: Cody Brimhall (mailto:brimhall@somuchwit.com)
+# Copyright:: Copyright (c) 2010 Cody Brimhall
+# License:: Distributed under the terms of the GNU General Public License, v. 3
+
+module BracketNotation # :nodoc:
+  module Version # :nodoc:
+    MAJOR = 1
+    MINOR = 0
+    MAINT = 3
+    
+    # Returns the current version string.
+    def self.to_s;
+      return [MAJOR, MINOR, MAINT].join(".")
+    end
+  end
+end

data/test/functional/evaluator_test.rb

@@ -0,0 +1,48 @@
+#--
+# This file is part of BracketNotation.
+# 
+# BracketNotation is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# BracketNotation is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with BracketNotation.  If not, see <http://www.gnu.org/licenses/>.
+#++
+# BracketNotation is a parser for generating syntax trees from sentences
+# annotated with the kind of bracket notation that is commonly used by
+# linguists. The result is a tree structure with nodes that describe the phrases
+# and constituents of the sentence.
+#
+# BracketNotation was inspired by Yoichiro Hasebe's RSyntaxTree[http://yohasebe.com/rsyntaxtree/],
+# and small portions of his code have been incorporated in the parser.
+#
+# Author:: Cody Brimhall (mailto:brimhall@somuchwit.com)
+# Copyright:: Copyright (c) 2010 Cody Brimhall
+# License:: Distributed under the terms of the GNU General Public License, v. 3
+
+require 'test/test_helper'
+
+class EvaluatorTest < Test::Unit::TestCase
+  include BracketNotation
+  
+  context "the evaluator" do
+    setup do
+      @tokens = [Token.LBRACKET, Token.NAME("S"), Token.LBRACKET, Token.NAME("DP"), Token.LBRACKET, Token.NAME("D"), Token.NAME("the"), Token.RBRACKET, Token.LBRACKET, Token.NAME("NP"), Token.LBRACKET, Token.NAME("N"), Token.NAME("boy"), Token.RBRACKET, Token.RBRACKET, Token.RBRACKET, Token.LBRACKET, Token.NAME("VP"), Token.LBRACKET, Token.NAME("V"), Token.NAME("ate"), Token.RBRACKET, Token.LBRACKET, Token.NAME("DP"), Token.LBRACKET, Token.NAME("D"), Token.NAME("the"), Token.RBRACKET, Token.LBRACKET, Token.NAME("NP"), Token.LBRACKET, Token.NAME("N"), Token.NAME("bread"), Token.RBRACKET, Token.RBRACKET, Token.RBRACKET, Token.RBRACKET, Token.RBRACKET, Token.EOL]
+      @evaluator = Evaluator.new(@tokens)
+    end
+    
+    should "produce an expression" do
+      assert_kind_of Expression, @evaluator.evaluate
+    end
+    
+    should "produce a root node" do
+      assert_nil @evaluator.evaluate.parent
+    end
+  end
+end

data/test/functional/parser_test.rb

@@ -0,0 +1,79 @@
+#--
+# This file is part of BracketNotation.
+# 
+# BracketNotation is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# BracketNotation is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with BracketNotation.  If not, see <http://www.gnu.org/licenses/>.
+#++
+# BracketNotation is a parser for generating syntax trees from sentences
+# annotated with the kind of bracket notation that is commonly used by
+# linguists. The result is a tree structure with nodes that describe the phrases
+# and constituents of the sentence.
+#
+# BracketNotation was inspired by Yoichiro Hasebe's RSyntaxTree[http://yohasebe.com/rsyntaxtree/],
+# and small portions of his code have been incorporated in the parser.
+#
+# Author:: Cody Brimhall (mailto:brimhall@somuchwit.com)
+# Copyright:: Copyright (c) 2010 Cody Brimhall
+# License:: Distributed under the terms of the GNU General Public License, v. 3
+
+require 'test/test_helper'
+
+class ParserTest < Test::Unit::TestCase
+  include BracketNotation
+  
+  context "the parser" do
+    setup do
+      @valid_input = "[S [DP [D the] [NP [N boy]]] [VP [V ate] [DP [D the] [NP [N bread]]]]]"
+      @input_with_unnamed_phrase = "[ [DP [D the] [NP [N boy]]] [VP [V ate] [DP [D the] [NP [N bread]]]]]"
+      @input_with_too_many_brackets = "[S [DP [D the] [NP [N boy]]] [VP [V ate] [DP [D the] [NP [N bread]]]]]]"
+      @input_with_too_few_brackets = "[S [DP [D the] [NP [N boy]]] [VP [V ate] [DP [D the] [NP [N bread]]]]"
+      @parser = nil
+    end
+    
+    should "validate good input" do
+      assert_nothing_raised do
+        @parser = Parser.new(@valid_input)
+      end
+    end
+    
+    should "not validate unnamed phrases" do
+      assert_raise Parser::ValidationError do
+        @parser = Parser.new(@input_with_unnamed_phrase)
+      end
+    end
+    
+    should "not validate too many brackets" do
+      assert_raise Parser::ValidationError do
+        @parser = Parser.new(@input_with_too_many_brackets)
+      end
+    end
+    
+    should "not validate too few brackets" do
+      assert_raise Parser::ValidationError do
+        @parser = Parser.new(@input_with_too_few_brackets)
+      end
+    end
+    
+    should "not validate the empty string" do
+      assert_raise Parser::ValidationError do
+        @parser = Parser.new("")
+      end
+    end
+    
+    should "not validate nil" do
+      assert_raise Parser::ValidationError do
+        @parser = Parser.new(nil)
+      end
+    end
+  end
+end