treat 0.1.1

data/lib/treat/lexicalizers/tag/stanford.rb

@@ -0,0 +1,86 @@
+module Treat
+  module Lexicalizers
+    module Tag
+      class Stanford
+        # Require the Ruby-Java bridge.
+        silently do
+          require 'rjb'
+          jar = "#{Treat.bin}/stanford_tagger/stanford-postagger.jar"
+          unless File.readable?(jar)
+            raise "Could not find stanford tagger JAR file in #{jar}."+
+            " You may need to set Treat.bin to a custom value."
+          end
+          Rjb::load(
+            "#{Treat.bin}/stanford_tagger/stanford-postagger.jar", 
+            ['-Xms256M', '-Xmx512M']
+          )
+          MaxentTagger = ::Rjb::import('edu.stanford.nlp.tagger.maxent.MaxentTagger')
+          Word = ::Rjb::import('edu.stanford.nlp.ling.Word')
+          List = ::Rjb::import('java.util.ArrayList')
+        end
+        # A list of models to use by language.
+        # Other models are available; see the models/ folder
+        # in the Stanford Tagger distribution files.
+        LanguageToModel = {
+          eng: 'english-left3words-distsim.tagger',
+          ger: 'german-fast.tagger',
+          fra: 'french.tagger',
+          ara: 'arabic-fast.tagger',
+          chi: 'chinese.tagger'
+        }
+        # Hold one tagger per language.
+        @@taggers = {}
+        # Hold the user-set options for each language.
+        @@options = {}
+        # Hold the default options.
+        DefaultOptions =  {}
+        # Tag the word using one of the Stanford taggers.
+        def self.tag(entity, options = {})
+          lang = entity.language
+          # Find the model.
+          if options[:model]
+            model = options[:model]
+          else
+            model = LanguageToModel[lang]
+            if model.nil?
+              raise Treat::Exception "There exists no Stanford" +
+              "tagger model for language #{lang}."
+            end
+          end
+          # Reinitialize the tagger if the options have changed.
+          if options != @@options[lang]
+            @@options[lang] = DefaultOptions.merge(options)
+            @@taggers[lang] = nil # Reset the tagger
+          end
+          if @@taggers[lang].nil?
+            model = "#{Treat.bin}/stanford_tagger/models/#{model}"
+            unless File.readable?(model)
+              raise "Could not find a tagger model for language #{lang}: looking in #{model}."
+            end
+            silence_streams(STDOUT, STDERR) do
+              @@taggers[lang] =
+              MaxentTagger.new(model)
+            end
+          end
+          list = List.new
+          id_list = {}
+          i = 0
+          [entity].each do |word|    # Fix...
+            list.add(Word.new(word.to_s))
+            id_list[i] = word
+            i += 1
+          end
+          it = nil
+          it = @@taggers[lang].apply(list).iterator
+          i = 0
+          while it.has_next
+            w = it.next
+            id_list[i].set :tag, w.tag
+            i += 1
+          end
+          w.tag
+        end
+      end
+    end
+  end
+end

data/lib/treat/processors.rb

@@ -0,0 +1,45 @@
+module Treat
+  # Category for processor groups.
+  #
+  # A processor group is a group of algorithms for the building
+  # of trees representing textual entities.
+  #
+  # The processor groups include:
+  #
+  #   - Chunkers : split a text into zone objects.
+  #   - Segmenters : split a text or zone into sentence objects.
+  #   - Tokenizers : split a sentence into Token objects.
+  #   - Parsers: split a sentence into a tree of constituents
+  #     containing other constituents and Token objects, representing
+  #     the syntactic structure.
+  module Processors
+    # Chunkers split a text into zones.
+    module Chunkers
+      extend Group
+      self.type = :transformer
+      self.targets = [:document, :text]
+    end
+    # Segmenters split a text or zone into sentences.
+    module Segmenters
+      extend Group
+      self.type = :transformer
+      self.targets = [:document, :text, :zone]
+    end
+    # Tokenizers splits a sentence into Token objects.
+    module Tokenizers
+      extend Group
+      self.type = :transformer
+      self.targets = [:document, :text, :zone, :sentence, :constituent]
+    end
+    # Parsers split a sentence into constituent objects
+    # representing its syntactic structure, with the
+    # Token objects as children of the constituents.
+    module Parsers
+      extend Group
+      self.type = :transformer
+      self.targets = [:document, :text, :zone, :sentence, :constituent]
+    end
+    # Makes all the groups autoloadable and creates the delegators.
+    extend Treat::Category
+  end
+end

data/lib/treat/processors/chunkers/txt.rb

@@ -0,0 +1,27 @@
+module Treat
+  module Processors
+    module Chunkers
+      # This class separates a plain text file into 
+      # zones based on a very naive analysis of the 
+      # file.
+      class Txt
+        # Return an array of Zone objects found in the text.
+        def self.chunk(text, options = {})
+          zones = text.to_s.split("\n")
+          zones.each do |zone|
+            next if zone.strip == ''
+            if false # fix
+              text << Entities::List.new(zone)
+            end
+            if zone.length < 60
+              text << Entities::Title.new(zone)
+            else
+              text << Entities::Paragraph.new(zone)
+            end
+          end
+          text
+        end
+      end
+    end
+  end
+end

data/lib/treat/processors/parsers/enju.rb

@@ -0,0 +1,214 @@
+module Treat
+  module Processors
+    module Parsers
+      # The Enju class is a wrapper for the Enju syntactic
+      # parser for English. Given a file or string input,
+      # the parser formats it runs it through Enju, and
+      # parses the XML output by Enju using the Nokogiri
+      # XML reader. It creates wrappers for the sentences,
+      # syntactical constituents and  tokens that Enju identified.
+      #
+      # Original paper:
+      # Takuya Matsuzaki, Yusuke Miyao, and Jun'ichi Tsujii.
+      # 2007. Efficient HPSG Parsing with Supertagging and
+      # CFG-filtering. In Proceedings of IJCAI 2007.
+      class Enju
+        # Require the 'open13' library for interaction
+        # with the background Enju process.
+        require 'open3'
+        @@parsers = []
+        @@i = 0
+        # Require the Nokogiri XML parser.
+        require 'nokogiri'
+        # Maps Enju categories to Treat categories.
+        CategoryMap = {
+          'ADJ' => :adjective,
+          'ADV' => :adverb,
+          'CONJ' => :conjunction,
+          'COOD' => :conjunction,
+          'C' => :complementizer,
+          'D' => :determiner,
+          'N' => :noun,
+          'P' => :preposition,
+          'PN' => :punctuation,
+          'SC' => :conjunction,
+          'V' => :verb,
+          'PRT' => :particle
+        }
+        # Return the process running Enju.
+        def self.proc
+          if @@parsers.size < @@options[:processes]
+            @@parsers << ::Open3.popen3("enju -xml -i")
+          end
+          @@i += 1
+          @@i = 0 if @@i == @@parsers.size
+          @@parsers[@@i-1]
+        end
+        # Parse the entity into its syntactical constituents
+        # using Enju
+        def self.parse(entity, options = {})
+          options[:processes] ||= 1
+          @@options = options
+          stdin, stdout = proc
+          if entity.to_s.count('.') == 0
+            remove_last = true
+            text = entity.to_s + '.'
+          else
+            remove_last = false
+            text = entity.to_s
+          end
+          stdin.puts(text + "\n")
+          parsed = build(stdout.gets, remove_last)
+          if not parsed.nil?
+            entity.remove_all!
+            parsed.children.each do |child|
+              entity << child
+            end
+          else
+            warn "Couldn't parse the text '#{entity.to_s}'."
+          end
+          entity
+        end
+        # Parses an Enju XML output file using the Nogoriki
+        # XML reader and converts that structure into a tree
+        # of wrappers for textual entities.
+        def self.build(xml, remove_last = false)
+          # Read in the XML file.
+          xml_reader = Nokogiri::XML::Reader.from_memory(xml)
+          current_element = nil
+          previous_depth = 0
+          id_table = {}
+          edges_table = {}
+          # Read the XML file entity by entity.
+          while xml_reader.read
+            # The depth in the XML tree.
+            current_depth = xml_reader.depth
+            # If we are at the end of the children stack, pop up.
+            if previous_depth > current_depth
+              current_element = current_element.parent
+            end
+            # If an end element has been reached,
+            # change the depth and pop up on next
+            # iteration.
+            if xml_reader.node_type ==
+              Nokogiri::XML::Reader::TYPE_END_ELEMENT
+              previous_depth = current_depth
+              next
+            end
+            attributes = xml_reader.attributes
+            prefix = ['schema', 'lexentry', 'type']
+            # If the entity has entributes, add them.
+            unless attributes.empty?
+              new_attributes = {}
+              edges = {}
+              id = attributes.delete('id')
+              pred = attributes.delete('pred')
+              attributes.each_pair do |attribute, value|
+                if ['arg1', 'arg2'].include?(attribute)
+                  edges[value] = pred
+                else
+                  if attribute == 'cat'
+                    if xml_reader.name == 'tok'
+                      if value.length > 1 && ['P', 'X'].include?(value[-1]) &&
+                        value != 'PN'
+                        new_attributes[:saturated] = (value[-1] == 'P')
+                        value = value[0..-2]
+                      end
+                      cat = CategoryMap[value]
+                      new_attributes[:cat] = cat
+                    else
+                      new_attributes[:enju_cat] = value
+                      xcat = attributes['xcat'].split(' ')[0]
+                      xcat ||= ''
+                      tags = Treat::Resources::Tags::EnjuCatXcatToPTB.select do |m|
+                        m[0] == value && m[1] == xcat
+                      end
+                      if tags.empty?
+                        tag = 'UK'
+                      else
+                        tag = tags[0][2]
+                      end
+                      new_attributes[:enju_xcat] = xcat
+                      attributes.delete('xcat')
+                      new_attributes[:tag] = tag
+                    end
+                  else
+                    pre = prefix.include?(attribute) ? 'enju_' : ''
+                    new_attributes[:"#{pre+attribute}"] = value
+                  end
+                end
+              end
+              attributes.delete('arg1')
+              attributes.delete('arg2')
+            end
+            # Handle naming conventions.
+            if attributes.has_key?('pos')
+              new_attributes[:tag] = new_attributes[:pos]
+              new_attributes.delete :pos
+            end
+            # Create the appropriate entity for the
+            # element.
+            current_value = ''
+            attributes = new_attributes
+            case xml_reader.name
+            when 'sentence'
+              current_element = Treat::Entities::Sentence.new('')
+              id_table[id] = current_element.id
+              edges_table[current_element.id] = edges
+              current_element.features = attributes
+            when 'cons'
+              current_element = current_element <<
+              Treat::Entities::Phrase.new('')
+              id_table[id] = current_element.id
+              edges_table[current_element.id] = edges
+              current_element.features = attributes
+            when 'tok'
+              tmp_attributes = attributes
+              tmp_edges = edges
+            else
+              current_value = xml_reader.value.gsub(/\s+/, "")
+              if !current_value.empty?
+                current_element = current_element <<
+                Treat::Entities::Entity.from_string(current_value)
+                if current_element.is_a?(Treat::Entities::Word)
+                  current_element.features = tmp_attributes
+                  id_table[id] = current_element.id
+                  edges_table[current_element.id] = tmp_edges                  
+                end
+              end
+            end
+            previous_depth = current_depth
+          end
+          # Add the edges to the entity.
+          unless current_element.nil?
+            root = current_element.root
+            edges_table.each_pair do |id2, edges2|
+              # Next if there are no edges.
+              next if edges2.nil?
+              entity = root.find(id2)
+              edges2.each_pair do |argument, type|
+                # Skip this argument if we don't know
+                # the target node.
+                next if argument == 'unk'
+                entity.associate(id_table[argument], type)
+              end
+            end
+            # Link the head and sem_head to their entities.
+            root.each_constituent do |constituent|
+              constituent.set :head,
+              root.find(id_table[constituent.head])
+              constituent.set :sem_head,
+              root.find(id_table[constituent.sem_head])
+            end
+          end
+          # Remove the period we added at the end.
+          if remove_last
+            last = current_element.punctuations[-1]
+            current_element.remove!(last)
+          end
+          current_element
+        end
+      end
+    end
+  end
+end

data/lib/treat/processors/parsers/stanford.rb

@@ -0,0 +1,60 @@
+module Treat
+  module Processors
+    module Parsers
+      class Stanford
+        # Require the Ruby-Java bridge.
+        silently { require 'rjb' }
+        jar = "#{Treat.bin}/stanford_parser/stanford-parser.jar"
+        unless File.readable?(jar)
+          raise "Could not find stanford parser JAR file in #{jar}."+
+          " You may need to set Treat.bin to a custom value."
+        end
+        Rjb::load(jar, ['-Xms256M', '-Xmx512M'])
+        LexicalizedParser = ::Rjb::import('edu.stanford.nlp.parser.lexparser.LexicalizedParser')
+        @@parsers = {}
+        def self.parse(entity, options = {})
+          lang = Treat::Resources::Languages.describe(entity.language).to_s
+          pcfg = "#{Treat.bin}/stanford_parser/grammar/#{lang.upcase}PCFG.ser.gz"
+          unless File.readable?(pcfg)
+            raise "Could not find a language model for #{lang}: looking in #{pcfg}."
+          end
+          @@parsers[lang] ||= LexicalizedParser.new(pcfg) # Fix - check that exists.
+          parse = @@parsers[lang].apply(entity.to_s)
+          entity.remove_all!          
+          recurse(parse, entity)
+          entity
+        end
+        def self.recurse(java_node, ruby_node)
+          # Leaf
+          if java_node.num_children == 0
+            ruby_child = Treat::Entities::Entity.from_string(java_node.value)
+            labels = java_node.labels.iterator
+            while labels.has_next
+              label = labels.next
+              ruby_child.set :begin_char, label.begin_position
+              ruby_child.set :end_char, label.end_position
+              ruby_child.set :tag, ruby_node.tag
+            end
+            ruby_node << ruby_child
+          else
+            if java_node.num_children == 1
+              return recurse(java_node.children[0], ruby_node)
+            end
+            java_node.children.each do |java_child|
+              dependencies = java_child.dependencies.iterator
+              # while dependencies.has_next
+                #dependency = dependencies.next
+              # end
+              ruby_child = Treat::Entities::Phrase.new
+              ruby_child.set :tag, java_child.value 
+              ruby_node << ruby_child
+              unless java_child.children.empty?
+                recurse(java_child, ruby_child)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/treat/processors/segmenters/punkt.rb

@@ -0,0 +1,48 @@
+module Treat
+  module Processors
+    module Segmenters
+      # An adapter for the 'punk-segmenter' gem, which segments
+      # texts into sentences based on an unsupervised, language
+      # independent algorithm.
+      # 
+      # Original paper: Kiss, Tibor and Strunk, Jan (2006): 
+      # Unsupervised Multilingual Sentence Boundary Detection. 
+      # Computational Linguistics 32: 485-525.
+      class Punkt
+        silently { require 'punkt-segmenter' }
+        # Hold one copy of the segmenter per language.
+        @@segmenters = {}
+        # Hold only one trainer per language.
+        @@trainers = {}
+        # Texts to train the segmenter on.
+        @@training_texts = {
+          eng:  "A minute is a unit of measurement of time or of angle. The minute is a unit of time equal to 1/60th of an hour or 60 seconds by 1. In the UTC time scale, a minute occasionally has 59 or 61 seconds; see leap second. The minute is not an SI unit; however, it is accepted for use with SI units. The symbol for minute or minutes is min. The fact that an hour contains 60 minutes is probably due to influences from the Babylonians, who used a base-60 or sexagesimal counting system. Colloquially, a min. may also refer to an indefinite amount of time substantially longer than the standardized length."
+        }
+        # Segment a text using the Punkt segmenter gem.
+        #
+        # Options: 
+        #   :training_text => (String) Text to train the segmenter on.
+        def self.segment(entity, options = {})
+          lang = entity.language
+          training_text = options[:training_text] ? 
+          options[:training_text] : @@training_texts[lang]
+          unless training_text
+            raise "No training text available for language #{lang}."
+          end
+          if @@trainers[lang].nil?
+            @@trainers[lang] = ::Punkt::Trainer.new
+            @@trainers[lang].train(training_text)
+            @@segmenters[lang] = 
+            ::Punkt::SentenceTokenizer.new(@@trainers[lang].parameters)
+          end
+          result = @@segmenters[lang].sentences_from_text(entity.to_s, 
+            :output => :sentences_text)
+          result.each do |sentence|
+            entity << Entities::Entity.from_string(sentence)
+          end
+          entity
+        end
+      end
+    end
+  end
+end