treat 0.1.1

data/lib/treat/entities/zones.rb

@@ -0,0 +1,17 @@
+module Treat
+  module Entities
+    # Represents a zone of text 
+    # (Title, Paragraph, List, Quote).
+    class Zone < Entity
+    end
+    # Represents a title, subtitle, logical header.
+    class Title < Zone
+    end
+    # Represents a paragraph.
+    class Paragraph < Zone
+    end
+    # Represents a list.
+    class List < Zone
+    end
+  end
+end

data/lib/treat/exception.rb

@@ -0,0 +1,5 @@
+module Treat
+  # Exception class for the Treat library.
+  class Exception < ::Exception
+  end
+end

data/lib/treat/extractors.rb

@@ -0,0 +1,41 @@
+module Treat
+  # Extractors extract specific information out of texts.
+  module Extractors
+    # Extracts a DateTime object containing a timestamp
+    # from string representation of date/time.
+    module Time
+      extend Group
+      self.type = :computer
+      self.targets = [:word, :constituent, :symbol]
+    end
+    # Extract the topic from a text.
+    module Topics
+      extend Group
+      self.type = :annotator
+      self.targets = [:collection, :document, :text, :zone, :sentence]
+    end
+    # Extract the topic from a text.
+    module TopicWords
+      extend Group
+      self.type = :annotator
+      self.targets = [:collection, :document, :text, :zone, :sentence]
+    end
+    module Statistics
+      extend Group
+      self.type = :computer
+      self.targets = [:entity]
+      self.default = :none
+    end
+    module NamedEntity
+      extend Group
+      self.type = :computer
+      self.targets = [:entity]
+    end
+    module KeySentences
+      extend Group
+      self.type = :computer
+      self.targets = [:collection, :document, :text, :zone, :sentence]
+    end
+    extend Treat::Category
+  end
+end

data/lib/treat/extractors/key_sentences/topics_frequency.rb

@@ -0,0 +1,49 @@
+module Treat
+  module Extractors
+    module KeySentences
+      class TopicsFrequency
+        
+        def self.key_sentences(entity, options = {})
+          options[:threshold] ||= 4
+          @@topics = options[:topic_words]
+          if Treat::Entities.rank(entity.type) <
+            Treat::Entities.rank(:sentence)
+            raise Treat::Exception, 'Cannot get the key ' +
+            'sentences of an entity smaller than a sentence.'
+          else
+            sentence_scores = {}
+            sentences = []
+            entity.each_sentence do |sentence|
+              sentence_scores[sentence.id] = score_sentence(sentence)
+            end
+            sentence_scores.each do |sid, score|
+              if score >= options[:threshold]
+                s = entity.find(sid)
+                s.set :is_key_sentence?, true
+                sentences << s
+              end
+            end
+          end
+          sentences
+        end
+        
+        def self.score_sentence(sentence)
+          sentence.set :topic_score, 0
+          sentence.each_word do |word|
+            found = false
+            @@topics.each do |i, topic_words|
+              if topic_words.include?(word.to_s)
+                sentence.set :topic_score,
+                (sentence.topic_score + 1)
+                found = true
+              end
+            end
+            word.set :is_keyword?, found
+          end
+          sentence.topic_score
+        end
+        
+      end
+    end
+  end
+end

data/lib/treat/extractors/named_entity/abner.rb

@@ -0,0 +1,20 @@
+module Treat
+  module Extractors
+    module NamedEntity
+      class Abner
+        # Require the Ruby-Java bridge.
+        silently do
+          require 'rjb'
+          Rjb::load('', ['-Xms256M', '-Xmx512M'])
+          puts Rjb.import('tagger')
+        end
+        @@tagger = nil
+        def self.named_entity(entity)
+          @@tagger ||= AbnerTagger.new
+          @@tagger.tokenize(entity)
+        end
+      end
+    end
+  end
+end
+

data/lib/treat/extractors/named_entity/stanford.rb

@@ -0,0 +1,174 @@
+module Treat
+  module Extractors
+    module NamedEntity
+      class Stanford
+        # Require the Ruby-Java bridge.
+        silently do 
+          require 'rjb'
+          Rjb::load(nil, ['-Xms256M', '-Xmx1024M'])
+          Rjb::add_jar('/ruby/treat/bin/treat/treat.jar')
+          Rjb::add_jar('/ruby/treat/bin/stanford/xom.jar')
+          Rjb::add_jar('/ruby/treat/bin/stanford/joda-time.jar')
+          Rjb::add_jar('/ruby/treat/bin/stanford/stanford-corenlp.jar')
+          StanfordCoreNLP = Rjb::import('edu.stanford.nlp.pipeline.StanfordCoreNLP')
+          Annotation = Rjb::import('edu.stanford.nlp.pipeline.Annotation')
+          NamedEntityTagAnnotation = Rjb::import('edu.stanford.nlp.ling.CoreAnnotations$NamedEntityTagAnnotation')
+          Properties = Rjb::import('java.util.Properties')
+        end
+        @@classifier = nil
+        def self.named_entity(entity, options = {})
+          properties = Properties.new
+          properties.set_property('annotators', 'tokenize, ssplit, pos, lemma, ner')
+          properties.set_property('pos.model', '/ruby/treat/bin/stanford/taggers/english-left3words-distsim.tagger')
+          properties.set_property('ner.model.3class', '/ruby/treat/bin/stanford/classifiers/all.3class.distsim.crf.ser.gz')
+          properties.set_property('ner.model.7class', '/ruby/treat/bin/stanford/classifiers/muc.7class.distsim.crf.ser.gz')
+          properties.set_property('ner.model.MISCclass', '/ruby/treat/bin/stanford/classifiers/conll.4class.distsim.crf.ser.gz')
+          properties.set_property('parser.model', '/ruby/treat/bin/stanford_parser/grammar/englishPCFG.ser.gz')
+          silence_stream(STDOUT) do
+            pipeline = StanfordCoreNLP.new(properties)
+          end
+          stanford_entity = Annotation.new(entity.to_s)
+          pipeline.annotate(stanford_entity)
+          puts stanford_entity.java_methods
+          puts stanford_entity.get_string(NamedEntityTagAnnotation)
+        end
+      end
+    end
+  end
+end
+
+
+=begin
+
+
+
+CRFBiasedClassifier = Rjb::import('edu.stanford.nlp.ie.crf.CRFBiasedClassifier')
+Properties = Rjb::import('java.util.Properties')
+List = ::Rjb::import('java.util.ArrayList')
+Word = ::Rjb::import('edu.stanford.nlp.ling.Word')
+CoreAnnotations = ::Rjb::import('edu.stanford.nlp.ling.CoreAnnotations')
+if @@classifier == nil
+  properties = Properties.new
+  options.each_pair do |option,value|
+    #properties.set_property('trainFile', )... Set the options.
+  end
+  @@classifier = CRFBiasedClassifier.new(properties)
+  @@classifier.load_classifier("/ruby/treat/bin/stanford_ner/classifiers/conll.4class.distsim.crf.ser.gz")
+end
+w = Word.new('Obama')
+#puts @@classifier.java_methods
+puts CoreAnnotations.public_methods.inspect
+puts @@classifier.classify(w).get()
+
+
+/*
+ * To change this template, choose Tools | Templates
+ * and open the template in the editor.
+ */
+
+package corenlp;
+import edu.stanford.nlp.ling.CoreAnnotations.CollapsedCCProcessedDependenciesAnnotation;
+import edu.stanford.nlp.ling.CoreAnnotations.CorefGraphAnnotation;
+import edu.stanford.nlp.ling.CoreAnnotations.NamedEntityTagAnnotation;
+import edu.stanford.nlp.ling.CoreAnnotations.PartOfSpeechAnnotation;
+import edu.stanford.nlp.ling.CoreAnnotations.SentencesAnnotation;
+import edu.stanford.nlp.ling.CoreAnnotations.TextAnnotation;
+import edu.stanford.nlp.ling.CoreAnnotations.TokensAnnotation;
+import edu.stanford.nlp.ling.CoreAnnotations.TreeAnnotation;
+import edu.stanford.nlp.ling.CoreLabel;
+import edu.stanford.nlp.pipeline.*;
+import edu.stanford.nlp.trees.Tree;
+import edu.stanford.nlp.trees.semgraph.SemanticGraph;
+import edu.stanford.nlp.util.CoreMap;
+import edu.stanford.nlp.util.IntTuple;
+import edu.stanford.nlp.util.Pair;
+import edu.stanford.nlp.util.Timing;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import java.util.Properties;
+/**
+ *
+ * @author Karthi
+ */
+public class Main {
+
+    /**
+     * @param args the command line arguments
+     */
+    public static void main(String[] args) throws IOException, ClassNotFoundException {
+//        // TODO code application liogic here
+//        System.out.println(System.getProperty("sun.arch.data.model"));
+////        String str="-cp stanford-corenlp-2010-11-12.jar:stanford-corenlp-models-2010-11-06.jar:xom-1.2.6.jar:jgrapht-0.7.3.jar -Xms3g edu.stanford.nlp.pipeline.StanfordCoreNLP -file <input.txt>";
+////        args=str.split(" ");
+////        StanfordCoreNLP.main(args);
+//        Timing tim = new Timing();
+//        Properties props = null;
+//        props.setProperty("annotators", "ssplit, ner, parse, dcoref");
+//
+//        StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
+//    props = pipeline.getProperties();
+//    long setupTime = tim.report();
+//    String fileName = "input.txt";
+//      ArrayList<File> files=null;
+//      files.add(new File(filename));
+//      pipeline.processFiles(pipeline, files, props);
+//
+//
+
+
+        // creates a StanfordCoreNLP object, with POS tagging, lemmatization, NER, parsing, and coreference resolution
+    Properties props = new Properties();
+    FileInputStream in = new FileInputStream("Main.properties");
+
+    props.load(in);
+    in.close();
+    StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
+
+    // read some text in the text variable
+    String text = "The doctor can consult with other doctors about this patient. If that is the case, the name of the doctor and the names of the consultants have to be maintained. Otherwise, only the name of the doctor is kept. "; // Add your text here!
+
+    // create an empty Annotation just with the given text
+    Annotation document = new Annotation(text);
+
+    // run all Annotators on this text
+    pipeline.annotate(document);
+    System.out.println(document);
+    
+    // these are all the sentences in this document
+    // a CoreMap is essentially a Map that uses class objects as keys and has values with custom types
+    List<CoreMap> sentences = (List<CoreMap>) document.get(SentencesAnnotation.class);
+    System.out.println(sentences);
+    for(CoreMap sentence: sentences) {
+      // traversing the words in the current sentence
+      // a CoreLabel is a CoreMap with additional token-specific methods
+      for (CoreLabel token: sentence.get(TokensAnnotation.class)) {
+        // this is the text of the token
+        String word = token.get(TextAnnotation.class);
+        // this is the POS tag of the token
+        String pos = token.get(PartOfSpeechAnnotation.class);
+        // this is the NER label of the token
+        String ne = token.get(NamedEntityTagAnnotation.class);
+      }
+
+      // this is the parse tree of the current sentence
+      Tree tree = sentence.get(TreeAnnotation.class);
+System.out.println(tree);
+      // this is the Stanford dependency graph of the current sentence
+      SemanticGraph dependencies = sentence.get(CollapsedCCProcessedDependenciesAnnotation.class);
+      System.out.println(dependencies);
+    }
+
+    // this is the coreference link graph
+    // each link stores an arc in the graph; the first element in the Pair is the source, the second is the target
+    // each node is stored as <sentence id, token id>. Both offsets start at 1!
+    List<Pair<IntTuple, IntTuple>> graph = document.get(CorefGraphAnnotation.class);
+    System.out.println(graph);
+
+    }
+
+}
+=end

data/lib/treat/extractors/statistics/frequency.rb

@@ -0,0 +1,22 @@
+module Treat
+  module Extractors
+    module Statistics
+      class Frequency
+        # Find the frequency of a given string value.
+        def self.statistics(entity, options={})
+          if entity.is_leaf?
+            w = entity.value.downcase
+            if entity.token_registry[:value][w].nil?
+              0
+            else
+              entity.token_registry[:value][w].size
+            end
+          else
+            raise Treat::Exception,
+            'Cannot get the frequency of a non-terminal entity.'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/treat/extractors/statistics/frequency_of.rb

@@ -0,0 +1,17 @@
+module Treat
+  module Extractors
+    module Statistics
+      class FrequencyOf
+        # Find the frequency of a given string value.
+        def self.statistics(entity, options = {})
+          w = options[:value]
+          if entity.token_registry[:value][w].nil?
+            0
+          else
+            entity.token_registry[:value][w].size
+          end
+        end
+      end
+    end
+  end
+end

data/lib/treat/extractors/statistics/position_in.rb

@@ -0,0 +1,13 @@
+module Treat
+  module Extractors
+    module Statistics
+      class PositionIn
+        # Find the position of the current entity
+        # inside the parent entity with type entity_type.
+        def self.statistics(entity)
+          raise Treat::Exception, 'Could you implement this?'
+        end
+      end
+    end
+  end
+end

data/lib/treat/extractors/statistics/transition_matrix.rb

@@ -0,0 +1,105 @@
+module Treat
+  module Extractors
+    module Statistics
+      class TransitionMatrix
+
+        # Find the transition matrix.
+        def self.statistics(entity, options={})
+
+          normalize = options[:normalize] || true
+          features = options[:features] || [:tag]
+          condition = options[:condition] || lambda { |e| true }
+          entity_types = options[:entity_types] ? options[:entity_types] :
+          [options[:entity_type]]
+          relationships = options[:relationships] || 
+          [:parent, :left, :right, :children]
+
+          # Create lambdas to generate the arrays.
+          empty_prototype = {}; features.each { |f| empty_prototype[f] = {} }
+          empty = lambda { Marshal.load(Marshal.dump(empty_prototype)) }
+          empty2_prototype = {}; relationships.each { |r| empty2_prototype[r] = empty.call }
+          empty2 = lambda { Marshal.load(Marshal.dump(empty2_prototype)) }
+
+          # Deep (recursive) merger.
+          merger = lambda do |key,v1,v2|
+            Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : v2
+          end
+
+          # Master matrix.
+          mm = nil
+
+          entity.each_entity(*entity_types) do |target|
+            
+            next unless condition.call(target)
+
+            # Initialize the empty transition matrix.
+            tm = empty.call
+
+            # Calculate the transition probabilities.
+            features.each do |f1|
+
+              v1 = target.send(f1)
+              tm[f1][v1] = empty2.call
+
+              relationships.each do |relationship|
+                tm[f1][v1][relationship] = empty.call
+                
+                features.each do |f2|
+                  relatives = target.send(relationship)
+                  relatives = [relatives] unless relatives.is_a? Array
+                  relatives.each do |relative|
+                    unless relative.nil?
+                      next if relative.nil? || !relative.has?(f2)
+                      v2 = relative.send(f2)
+                      tm[f1][v1][relationship][f2][v2] ||= 0.0
+                      tm[f1][v1][relationship][f2][v2] += 1.0
+                    end
+                  end
+                  
+                  tm[f1][v1][:edge] = empty.call
+                  
+                  target.edges.each do |id, edge_type|
+                    s = target.ancestor_with_type :sentence
+                    if s
+                      x = s.find(id)
+                      next unless relative.has?(f2)
+                      v2 = x.send(f2)
+                      tm[f1][v1][:edge][f2][v2] ||= 0.0
+                      tm[f1][v1][:edge][f2][v2] += 1.0
+                    end
+                  end
+                             
+                end
+              end
+            end
+            
+            mm = mm ? mm.merge(tm, &merger) : tm
+          end
+          if normalize
+            normalize(mm)
+          else
+            mm
+          end
+        end
+
+        # Normalize the transition probabilities.
+        def self.normalize(tm)
+          tm.each do |f1, as|
+            as.each do |a, dirs|
+              dirs.each do |dir, f2s|
+                f2s.each do |f2, vals|
+                  sum = vals.values.inject(0) {|n,x| n+x }.to_f
+                  vals.each do |val, count|
+                    vals[val] = count/sum
+                  end
+                end
+              end
+            end
+          end
+          tm
+        end
+
+      end
+    end
+  end
+end