treat 0.1.4 → 0.2.0

data/bin/INFO

@@ -1 +0,0 @@
-This is where Treat will look for the Stanford JAR files by default. You can change this to another directory by setting Treat.bin = '/path/to/your/folder/' at runtime.

data/examples/benchmark.rb

@@ -1,81 +0,0 @@
-require 'benchmark'
-require 'treat'
-
-Benchmark.bmbm do |x|
-
-  Treat.edulcorate
-
-=begin
-  # Readers
-  x.report("Read:PDF") { doc = Document 'pages/hhmm_article.pdf'; doc.read }
-  x.report("Read:TXT") { doc = Document 'pages/kant_short.txt'; doc.read }
-  x.report("Read:YAML") { doc = Document 'pages/nanotechnology_article.yml'; doc.read }
-  # x.report("Read:XML") { doc = Document 'pages/test.xml'; doc.read }
-  x.report("Read:Image") { doc = Document 'pages/novel_page.jpg'; doc.read }
-  
-  # Read collection of texts.
-  coll = Collection 'pages'
-  coll.read
-=end
-
-  # Processors.
-  # x.report("Cluster:LDA") { coll.cluster(:lda) }
-  x.report("Chunk:txt ") { text.chunk(:txt) }
-  x.report("Segment:punkt ") { text.segment(:punkt) }
-  x.report("Segment:tactful ") { text.segment(:tactful) }
-  x.report("Segment:stanford ") { text.segment(:stanford) }
-  x.report("Tokenize:macintyre ") { text.tokenize(:macintyre) }
-  x.report("Tokenize:multilingual "){ text.tokenize(:multilingual) }
-  x.report("Tokenize:perl "){ text.tokenize(:perl) }
-  x.report("Tokenize:stanford ") { text.tokenize(:stanford) }
-  x.report("Parse:enju") { text = text.parse(:enju) }
-  # x.report("Parse:stanford") { text = text.parse(:stanford) }
-  # x.report("Parse:link") { text = text.parse(:link) }
-
-  doc = Document 'pages/kant_short.txt'
-  text = doc.read.text.chunk.segment.tokenize
-
-  # Formatters.
-  yaml = nil; xml = nil
-  x.report("Serialize:yaml") { yaml = text.serialize(:yaml) }
-  x.report("Serialize:xml") { xml = text.serialize(:xml) }
-  x.report("Visualize:tree") { text.visualize(:tree) }
-  x.report("Visualize:txt") { text.visualize(:txt) }
-  # x.report("Visualize:dot") { text.visualize(:dot) }
-  # x.report("Visualize:standoff") { text.visualize(:standoff) }
-  # x.report("Visualize:simple_html") { text.visualize(:html) }
-  # Clean: html
-
-  # Detectors
-  x.report("Langugage:what_language ") { text.language(:what_language) }
-  x.report("Encoding:r_chardet19 ") { text.encoding(:r_chardet19) }
-  x.report("Format:file ") { text.format(:file) }
-
-  # Extractors
-  x.report("Date:chronic") { '2007/02/12'.date(:chronic) }
-  x.report("Date:native") { '2007/02/12'.date(:native) }
-  x.report("Time:chronic") { '2007/02/12'.time(:chronic) }
-  x.report("Topic:reuters") { text.topic }
-  x.report("Statistics:frequency:") { text.each_token { |token| token.statistics(:frequency) } }
-  # x.report("Statistics:position:") { text.each_token { |token| token.statistics(:position) } }
-
-  # Inflectors
-  # x.report("Lemma:elemma") { text.each_word { |word| word.lemma(:elemma) } }
-  x.report("Stem:porter_r") { text.each_word { |word| word.stem(:porter) } }
-  x.report("Stem:porter_c") { text.each_word { |word| word.stem(:porter_c) } }
-  x.report("Stem:uea") { text.each_word { |word| word.stem(:uea) } }
-  x.report("Declense:granger") { text.each_word { |word| word.declense(:granger) } }
-  # x.report("Inflect:granger") { text.each_noun { |word| word.plural(:granger) } }
-
-  # Statistics
-  x.report("Entity:word_count") { text.word_count }
-
-  # puts text.words_with_cat(:noun).inspect
-
-  # Lexicalizers
-  x.report("Tag:stanford") { text.each_word { |word| word.tag(:stanford) } }
-  # x.report("Tag:brill") { text.each_word { |word| word.tag(:brill) } }
-  # x.report("Tag:lingua") { text.each_word { |word| word.tag(:lingua) } }
-  # x.report("Lex:wordnet") { text.each_word { |word| word.lex(:wordnet) } }
-
-end

data/examples/keywords.rb

@@ -1,148 +0,0 @@
-require 'benchmark'
-require 'treat'
-
-Treat.edulcorate
-
-c = Collection.from_serialized('texts/corpus.yml')
-
-=begin
-c.each_text do |t|
-  t.chunk.segment.parse(:stanford)
-  puts "Done text #{t.id}."
-end
-
-c.serialize(:yaml).save("economist/corpus.yml")
-
-=end
-
-topic_words = c.topic_words(
-  :lda,
-  :topics => 5,
-  :words_per_topic => 5,
-  :iterations => 20
-)
-  
-c.each_document do |d|
-  
-  sentences = d.key_sentences(
-    :topics_frequency,
-    :topic_words => topic_words,
-    :threshold => 4
-  )
-
-  tm = d.statistics(
-    :transition_matrix,
-    :features => [:tag],
-    :entity_type => :word,
-    :condition => lambda do |word|
-      word.has?(:is_keyword?) &&
-      word.is_keyword?
-    end
-  )
-
-  sentences.each do |sentence|
-    sentence.each_word do |word|
-      score = word.statistics(
-        :transition_probability,
-        :transition_matrix => tm,
-        :relationships => [:parent, :left, :right, :children]
-      )
-      if word.has?(:is_keyword?) && 
-         word.is_keyword?
-           score += 0.5 
-      end
-      if score > 1
-        puts word.to_s
-      end
-    end
-  end
-  
-end
-
-
-
-Treat.edulcorate
-Treat.bin = '/ruby/nat/bin'
-
-c = Collection 'economist'
-c.each_document { |doc| doc.chunk.segment.tokenize }
-
-topic_words = c.topic_words(
-  :lda,
-  :topics => 5,
-  :words_per_topic => 5,
-  :iterations => 20
-)
-
-keywords = c.keywords(
-  :topics_frequency,
-  :topic_words => topic_words,
-  :tf_idf_threshold => 180
-)
-
-puts keywords.inspect
-
-abort
-
-c = Phrase 'a test clause'
-c.parse
-puts c.visualize(:tree)
-puts c.visualize(:inspect)
-puts c.visualize(:short_value)
-puts c.visualize(:standoff)
-puts c.visualize(:tree)
-
-c.serialize(:yaml).save('test.yml')
-c.serialize(:xml).save('test.xml')
-
-d = Phrase 'test.yml'
-d.print_tree
-d = Phrase 'test.xml'
-d.print_tree
-
-puts d.words[0].position_in_parent
-abort
-
-w = Word 'running'
-puts w.stem(:porter_c)
-puts w.stem(:porter)
-puts w.stem(:uea)
-
-w = Word 'run'
-
-puts w.infinitive(:linguistics)
-puts w.present_participle(:linguistics)
-puts w.plural(:linguistics)
-
-w = Word 'table'
-
-puts w.synonyms.inspect
-puts w.antonyms.inspect
-puts w.hyponyms.inspect
-puts w.hypernyms.inspect
-
-n = Number 2
-puts n.ordinal_words(:linguistics)
-puts n.cardinal_words(:linguistics)
-
-s = Sentence 'A sentence to parse.'
-s.dup.parse(:enju).print_tree
-s.dup.parse(:stanford).print_tree
-
-s = Sentence 'A sentence to tokenize'
-s.dup.tokenize(:macintyre).print_tree
-s.dup.tokenize(:multilingual).print_tree
-s.dup.tokenize(:perl).print_tree
-s.dup.tokenize(:punkt).print_tree
-s.dup.tokenize(:stanford).print_tree
-s.dup.tokenize(:tactful).print_tree
-
-
-=begin
-c = Collection 'economist'
-# c.each_document { |d| d.chunk.segment.tokenize }
-c.documents[0].chunk.segment
-c.sentences[0].parse(:enju)
-c.each_word { |word| word.stem }
-c.visualize(:dot, features: [:tag]).save('test.dot')
-=end

data/lib/treat/detectors.rb

@@ -1,31 +0,0 @@
-module Treat
-  # Detectors detect a specific meta-information about
-  # an entity, such as encoding, format and language.
-  # 
-  # Detectors are language-independent, and thus there
-  # are default algorithms specified for each of them.
-  module Detectors
-    # Group for algorithms that detect encoding.
-    module Encoding 
-      extend Group
-      self.type = :annotator
-      self.targets = [:document]
-      self.default = :r_chardet19
-    end
-    # Group for algorithms that support format detection.
-    module Format
-      extend Group
-      self.type = :annotator
-      self.targets = [:document]
-      self.default = :file
-    end
-    # Group for algorithms that do language detection.
-    module Language
-      extend Group
-      self.type = :annotator
-      self.targets = [:entity]
-      self.default = :what_language
-    end
-    extend Treat::Category
-  end
-end

data/lib/treat/detectors/encoding/r_chardet19.rb

@@ -1,27 +0,0 @@
-module Treat
-  module Detectors
-    module Encoding
-      # Require the 'rchardet19' gem.
-      silence_warnings { require 'rchardet19' }
-      # A wrapper for the 'rchardet19' gem, which
-      # detects the encoding of a file.
-      class RChardet19
-        # Returns the encoding of the document according
-        # to the 'rchardet19' gem.
-        #
-        # Options: none.
-        def self.encoding(document, options={})
-          r = CharDet.detect(document.file)
-          if r.encoding
-            Treat::Feature.new({
-              r.encoding.
-              gsub('-', '_').downcase.intern =>
-            r.confidence}).best
-          else
-            :unknown
-          end
-        end
-      end
-    end
-  end
-end

data/lib/treat/detectors/format/file.rb

@@ -1,36 +0,0 @@
-module Treat
-  module Detectors
-    module Format
-      # A wrapper for the *NIX 'file' command,
-      # witch uses etc/magic to detect the format
-      # of a file.
-      class File
-        # Returns an identifier representing
-        # the format of a file using the *NIX
-        # 'file' command.
-        # 
-        # Options: none.
-        def self.format(entity, options = {})
-          format = nil
-          create_temp_file(:txt, entity.to_s) do |tmp|
-            format = `file #{tmp}` 
-          end
-          if format.scan('text')
-            :txt
-          elsif format.scan('XML')
-            :xml
-          elsif format.scan('HTML')
-            :html
-          elsif format.scan('image')
-            :image
-          elsif format.scan('PDF')
-            :pdf
-          else
-            raise Treat::Exception,
-            "Unsupported text format #{format}."
-          end
-        end
-      end
-    end
-  end
-end

data/lib/treat/detectors/language/what_language.rb

@@ -1,29 +0,0 @@
-module Treat
-  module Detectors
-    module Language
-      # Require the 'whatlanguage' gem.
-      silence_warnings { require 'whatlanguage'  }
-      # Adaptor for the 'whatlanguage' gem, which
-      # performs probabilistic language detection.
-      class WhatLanguage < LanguageDetector
-        # Keep only once instance of the gem class.
-        @@detector = nil
-        # Detect the language of an entity using the
-        # 'whatlanguage' gem. Return an identifier
-        # corresponding to the ISO-639-2 code for the
-        # language.
-        def self.language(entity, options = {})
-          predetection = super(entity, options)
-          return predetection if predetection
-          @@detector ||= ::WhatLanguage.new(:possibilities)
-          possibilities = @@detector.process_text(entity.to_s)
-          lang = {}
-          possibilities.each do |k,v|
-            lang[Treat::Languages.find(k)] = v
-          end
-          Treat::Feature.new(lang).best
-        end
-      end
-    end
-  end
-end

data/lib/treat/entities/constituents.rb

@@ -1,15 +0,0 @@
-module Treat
-  module Entities
-    # Represents any syntactic constituent
-    # of a sentence.
-    class Constituent < Entity
-    end
-    # Represents a phrase inside a sentence
-    # or by itself.
-    class Phrase < Constituent
-    end
-    # Represents a clause inside a sentence.
-    class Clause < Constituent
-    end
-  end
-end

data/lib/treat/entities/sentence.rb

@@ -1,8 +0,0 @@
-module Treat
-  module Entities
-    # Represents a sentence.
-    class Sentence < Entity
-      def subject(l = nil, o = {}); link(l, o.merge({:linkage => :subject})); end
-    end
-  end
-end

data/lib/treat/extractors/named_entity/abner.rb

@@ -1,20 +0,0 @@
-module Treat
-  module Extractors
-    module NamedEntity
-      class Abner
-        # Require the Ruby-Java bridge.
-        silence_warnings do
-          require 'rjb'
-          Rjb::load('', ['-Xms256M', '-Xmx512M'])
-          puts Rjb.import('tagger')
-        end
-        @@tagger = nil
-        def self.named_entity(entity)
-          @@tagger ||= AbnerTagger.new
-          @@tagger.tokenize(entity)
-        end
-      end
-    end
-  end
-end
-

data/lib/treat/extractors/named_entity/stanford.rb

@@ -1,174 +0,0 @@
-module Treat
-  module Extractors
-    module NamedEntity
-      class Stanford
-        # Require the Ruby-Java bridge.
-        silence_warnings do 
-          require 'rjb'
-          Rjb::load(nil, ['-Xms256M', '-Xmx1024M'])
-          Rjb::add_jar('/ruby/treat/bin/treat/treat.jar')
-          Rjb::add_jar('/ruby/treat/bin/stanford/xom.jar')
-          Rjb::add_jar('/ruby/treat/bin/stanford/joda-time.jar')
-          Rjb::add_jar('/ruby/treat/bin/stanford/stanford-corenlp.jar')
-          StanfordCoreNLP = Rjb::import('edu.stanford.nlp.pipeline.StanfordCoreNLP')
-          Annotation = Rjb::import('edu.stanford.nlp.pipeline.Annotation')
-          NamedEntityTagAnnotation = Rjb::import('edu.stanford.nlp.ling.CoreAnnotations$NamedEntityTagAnnotation')
-          Properties = Rjb::import('java.util.Properties')
-        end
-        @@classifier = nil
-        def self.named_entity(entity, options = {})
-          properties = Properties.new
-          properties.set_property('annotators', 'tokenize, ssplit, pos, lemma, ner')
-          properties.set_property('pos.model', '/ruby/treat/bin/stanford/taggers/english-left3words-distsim.tagger')
-          properties.set_property('ner.model.3class', '/ruby/treat/bin/stanford/classifiers/all.3class.distsim.crf.ser.gz')
-          properties.set_property('ner.model.7class', '/ruby/treat/bin/stanford/classifiers/muc.7class.distsim.crf.ser.gz')
-          properties.set_property('ner.model.MISCclass', '/ruby/treat/bin/stanford/classifiers/conll.4class.distsim.crf.ser.gz')
-          properties.set_property('parser.model', '/ruby/treat/bin/stanford-parser/grammar/englishPCFG.ser.gz')
-          silence_stream(STDOUT) do
-            pipeline = StanfordCoreNLP.new(properties)
-          end
-          stanford_entity = Annotation.new(entity.to_s)
-          pipeline.annotate(stanford_entity)
-          puts stanford_entity.java_methods
-          puts stanford_entity.get_string(NamedEntityTagAnnotation)
-        end
-      end
-    end
-  end
-end
-
-
-=begin
-
-
-
-CRFBiasedClassifier = Rjb::import('edu.stanford.nlp.ie.crf.CRFBiasedClassifier')
-Properties = Rjb::import('java.util.Properties')
-List = ::Rjb::import('java.util.ArrayList')
-Word = ::Rjb::import('edu.stanford.nlp.ling.Word')
-CoreAnnotations = ::Rjb::import('edu.stanford.nlp.ling.CoreAnnotations')
-if @@classifier == nil
-  properties = Properties.new
-  options.each_pair do |option,value|
-    #properties.set_property('trainFile', )... Set the options.
-  end
-  @@classifier = CRFBiasedClassifier.new(properties)
-  @@classifier.load_classifier("/ruby/treat/bin/stanford_ner/classifiers/conll.4class.distsim.crf.ser.gz")
-end
-w = Word.new('Obama')
-#puts @@classifier.java_methods
-puts CoreAnnotations.public_methods.inspect
-puts @@classifier.classify(w).get()
-
-
-/*
- * To change this template, choose Tools | Templates
- * and open the template in the editor.
- */
-
-package corenlp;
-import edu.stanford.nlp.ling.CoreAnnotations.CollapsedCCProcessedDependenciesAnnotation;
-import edu.stanford.nlp.ling.CoreAnnotations.CorefGraphAnnotation;
-import edu.stanford.nlp.ling.CoreAnnotations.NamedEntityTagAnnotation;
-import edu.stanford.nlp.ling.CoreAnnotations.PartOfSpeechAnnotation;
-import edu.stanford.nlp.ling.CoreAnnotations.SentencesAnnotation;
-import edu.stanford.nlp.ling.CoreAnnotations.TextAnnotation;
-import edu.stanford.nlp.ling.CoreAnnotations.TokensAnnotation;
-import edu.stanford.nlp.ling.CoreAnnotations.TreeAnnotation;
-import edu.stanford.nlp.ling.CoreLabel;
-import edu.stanford.nlp.pipeline.*;
-import edu.stanford.nlp.trees.Tree;
-import edu.stanford.nlp.trees.semgraph.SemanticGraph;
-import edu.stanford.nlp.util.CoreMap;
-import edu.stanford.nlp.util.IntTuple;
-import edu.stanford.nlp.util.Pair;
-import edu.stanford.nlp.util.Timing;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import java.util.Properties;
-/**
- *
- * @author Karthi
- */
-public class Main {
-
-    /**
-     * @param args the command line arguments
-     */
-    public static void main(String[] args) throws IOException, ClassNotFoundException {
-//        // TODO code application liogic here
-//        System.out.println(System.getProperty("sun.arch.data.model"));
-////        String str="-cp stanford-corenlp-2010-11-12.jar:stanford-corenlp-models-2010-11-06.jar:xom-1.2.6.jar:jgrapht-0.7.3.jar -Xms3g edu.stanford.nlp.pipeline.StanfordCoreNLP -file <input.txt>";
-////        args=str.split(" ");
-////        StanfordCoreNLP.main(args);
-//        Timing tim = new Timing();
-//        Properties props = null;
-//        props.setProperty("annotators", "ssplit, ner, parse, dcoref");
-//
-//        StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
-//    props = pipeline.getProperties();
-//    long setupTime = tim.report();
-//    String fileName = "input.txt";
-//      ArrayList<File> files=null;
-//      files.add(new File(filename));
-//      pipeline.processFiles(pipeline, files, props);
-//
-//
-
-
-        // creates a StanfordCoreNLP object, with POS tagging, lemmatization, NER, parsing, and coreference resolution
-    Properties props = new Properties();
-    FileInputStream in = new FileInputStream("Main.properties");
-
-    props.load(in);
-    in.close();
-    StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
-
-    // read some text in the text variable
-    String text = "The doctor can consult with other doctors about this patient. If that is the case, the name of the doctor and the names of the consultants have to be maintained. Otherwise, only the name of the doctor is kept. "; // Add your text here!
-
-    // create an empty Annotation just with the given text
-    Annotation document = new Annotation(text);
-
-    // run all Annotators on this text
-    pipeline.annotate(document);
-    System.out.println(document);
-    
-    // these are all the sentences in this document
-    // a CoreMap is essentially a Map that uses class objects as keys and has values with custom types
-    List<CoreMap> sentences = (List<CoreMap>) document.get(SentencesAnnotation.class);
-    System.out.println(sentences);
-    for(CoreMap sentence: sentences) {
-      // traversing the words in the current sentence
-      // a CoreLabel is a CoreMap with additional token-specific methods
-      for (CoreLabel token: sentence.get(TokensAnnotation.class)) {
-        // this is the text of the token
-        String word = token.get(TextAnnotation.class);
-        // this is the POS tag of the token
-        String pos = token.get(PartOfSpeechAnnotation.class);
-        // this is the NER label of the token
-        String ne = token.get(NamedEntityTagAnnotation.class);
-      }
-
-      // this is the parse tree of the current sentence
-      Tree tree = sentence.get(TreeAnnotation.class);
-System.out.println(tree);
-      // this is the Stanford dependency graph of the current sentence
-      SemanticGraph dependencies = sentence.get(CollapsedCCProcessedDependenciesAnnotation.class);
-      System.out.println(dependencies);
-    }
-
-    // this is the coreference link graph
-    // each link stores an arc in the graph; the first element in the Pair is the source, the second is the target
-    // each node is stored as <sentence id, token id>. Both offsets start at 1!
-    List<Pair<IntTuple, IntTuple>> graph = document.get(CorefGraphAnnotation.class);
-    System.out.println(graph);
-
-    }
-
-}
-=end