rbbt-text 0.2.1 → 0.5.0

data/lib/rbbt/nlp/nlp.rb

@@ -0,0 +1,235 @@
+require 'rbbt'
+require 'rbbt/util/tmpfile'
+require 'rbbt/util/persistence'
+require 'rbbt/util/resource'
+require 'rbbt/ner/annotations'
+require 'rbbt/ner/annotations/annotated'
+require 'rbbt/nlp/genia/sentence_splitter'
+require 'digest/md5'
+
+
+module NLP
+
+  extend LocalPersist
+  self.local_persistence_dir = '/tmp/crap'
+
+  #Rbbt.software.opt.StanfordParser.define_as_install Rbbt.share.install.software.StanfordParser.find
+  #Rbbt.software.opt.StanfordParser.produce
+
+  Rbbt.software.opt.Geniass.define_as_install Rbbt.share.install.software.Geniass.find
+  Rbbt.software.opt.Geniass.produce
+
+  Rbbt.software.opt.Gdep.define_as_install Rbbt.share.install.software.Gdep.find
+  Rbbt.software.opt.Gdep.produce
+
+  NEW_LINE_MASK = "\t\t \t  \t"
+
+  def self.geniass_sentence_splitter(text)
+    offsets = []
+
+    cleaned = text.gsub("\n",NEW_LINE_MASK)
+    TmpFile.with_file(cleaned) do |fin|
+      TmpFile.with_file do |fout|
+        CMD.cmd("cd #{Rbbt.software.opt.Geniass.find}; ./geniass #{ fin } #{ fout }")
+
+        
+        Open.write(fin, Open.read(fin).gsub(NEW_LINE_MASK, "\n"))
+        Open.write(fout, Open.read(fout).gsub("\n", '|').gsub(NEW_LINE_MASK, "\n"))
+        # Addapted from sentence2standOff.rb in Geniass package
+
+        inTxtStrict = Open.open(fin)
+        inTxtNew = Open.open(fout)
+
+        marker = "|"[0]
+        position = 0
+        sentenceCount = 1
+        target = ''
+        targetNew = ''
+        start = 0
+        finish = 0
+
+        while(!inTxtNew.eof?) do
+          targetNew = inTxtNew.getc
+          target = inTxtStrict.getc
+          position += 1
+          if targetNew == marker
+            sentenceCount += 1
+            finish = position - 1
+            offsets << [start, finish] if finish - start > 10
+            if targetNew == target
+              start = position
+            else
+              targetNew = inTxtNew.getc
+              while targetNew != target do
+                target = inTxtStrict.getc
+                position += 1
+              end
+              start = position - 1
+            end
+          end
+        end
+
+        finish = position - 1
+        offsets << [start, finish] if finish > start
+
+        inTxtStrict.close
+        inTxtNew.close
+      end
+    end
+
+    offsets.collect do |s,e|
+      sentence = text[s..e]
+      next if sentence.nil?
+      #sentence.gsub!(NEW_LINE_MASK, "\n")
+      Segment.annotate sentence, s
+      sentence
+    end
+  end
+
+  module GdepToken
+    attr_accessor :num, :token, :lemma, :chunk, :pos, :bio, :link, :dep
+    include Segment
+
+    def self.annotate(token, offset = nil, num = nil, lemma = nil, chunk = nil, pos = nil, bio = nil, link = nil, dep = nil)
+      token.extend GdepToken
+
+      token.offset = offset
+      token.num = num
+      token.lemma = lemma
+      token.chunk = chunk
+      token.pos = pos
+      token.bio = bio
+      token.link = link
+      token.dep = dep
+
+      token
+    end
+  end
+
+  module GdepChunk
+    attr_accessor :type, :parts, :segment_types
+    include Segment
+
+    def self.annotate(string, offset = nil, type = nil, parts = nil)
+      string.extend GdepChunk
+
+      string.offset = offset
+      string.type = type
+      string.parts = parts
+      
+      string
+    end
+  end
+
+  def self.merge_vp_chunks(chunk_list)
+    vp = nil
+    new_chunks = []
+    chunk_list.each do |chunk|
+      if chunk.type =~ /^VP/
+        if vp.nil?
+          vp = chunk
+        else
+          vp << chunk
+          vp.parts.concat chunk.parts
+        end
+      else
+        new_chunks << vp if not vp.nil?
+        new_chunks << chunk
+        vp = nil
+      end
+    end
+
+    new_chunks
+  end
+
+  def self.gdep_chunks(sentence, segment_list)
+    chunks = []
+
+    chunk_start = "B"[0]
+    chunk_inside = "I"[0]
+
+    last = GdepToken.annotate("LW")
+
+    chunk_segments = []
+    segment_list.each do |segment|
+      if segment.chunk[0] == chunk_inside and not segment.offset.nil?
+        chunk_segments << segment
+      else
+        if chunk_segments.any?
+          cstart = chunk_segments.first.offset
+          cend = chunk_segments.last.end
+          chunk = sentence[cstart..cend]
+          GdepChunk.annotate(chunk, cstart, last.chunk.sub(/^.-/,''), chunk_segments)
+          chunks << chunk
+        end
+
+        if segment.offset.nil?
+          chunk_segments = []
+        else
+          chunk_segments = [segment]
+        end
+      end
+      last = segment
+    end
+
+    chunks
+  end
+
+  def self.gdep_parse_sentences(sentences)
+    sentences = Array === sentences ? sentences : [sentences]
+
+    input = sentences.collect{|sentence| sentence.gsub(/\n/, NEW_LINE_MASK)} * "\n"
+    sentence_tokens = TmpFile.with_file(input) do |fin|
+      out = local_persist(Digest::MD5.hexdigest(input), :Chunks, :string) do
+        CMD.cmd("cd #{Rbbt.software.opt.Gdep.find}; ./gdep #{ fin }").read
+      end
+
+      out.split(/^$/).collect do |sentence|
+        tokens = sentence.split(/\n/).collect do |line|
+          next if line.empty?
+          num, token, lemma, chunk, pos, bio, link, dep = line.split(/\t/)
+          GdepToken.annotate(token, nil, num, lemma, chunk, pos, bio, link, dep)
+        end.compact
+      end
+    end
+
+    sentences.zip(sentence_tokens).collect do |sentence, tokens|
+      Segment.align(sentence, tokens)
+    end
+  end
+
+
+  def self.gdep_parse_sentences_extension(sentences)
+    require Rbbt.software.opt.Gdep.ruby["Gdep.so"].find
+    gdep = Gdep.new
+    if not gdep.gdep_is_loaded
+      Misc.in_dir Rbbt.software.opt.Gdep.find do
+        gdep.load_gdep 
+      end
+    end
+
+    sentences = Array === sentences ? sentences : [sentences]
+
+    sentence_tokens = sentences.collect{|sentence|
+      Gdep.new.tag(sentence).split(/\n/).collect do |line|
+        next if line.empty?
+        token, lemma, pos, chunk = line.split(/\t/)
+        GdepToken.annotate(token, nil, nil, lemma, chunk, pos)
+        token
+      end.compact
+    }
+
+    sentences.zip(sentence_tokens).collect do |sentence, tokens|
+      Segment.align(sentence, tokens)
+      tokens
+    end
+  end
+
+  def self.gdep_chunk_sentences(sentences)
+    sentences = Array === sentences ? sentences : [sentences]
+    NLP.gdep_parse_sentences_extension(sentences).zip(sentences).collect do |segment_list, sentence|
+      chunk_list = NLP.gdep_chunks(sentence, segment_list)
+      NLP.merge_vp_chunks(chunk_list)
+    end
+  end
+end

data/share/install/software/ABNER

@@ -1,9 +1,5 @@
 #!/bin/bash
 
-INSTALL_HELPER_FILE="$1"
-RBBT_SOFTWARE_DIR="$2"
-source "$INSTALL_HELPER_FILE"
-
 name='ABNER'
 url="http://pages.cs.wisc.edu/~bsettles/abner/abner.jar"
 

data/share/install/software/ChemicalTagger

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+INSTALL_HELPER_FILE="$1"
+RBBT_SOFTWARE_DIR="$2"
+source "$INSTALL_HELPER_FILE"
+
+name='ChemicalTagger'
+url="https://bitbucket.org/lh359/chemicaltagger/downloads/chemicalTagger-1.0-jar-with-dependencies.jar"
+
+PKG_DIR=`opt_dir $name`
+[ -d $PKG_DIR ] || mkdir -p $PKG_DIR
+[ -t "$OPT_SRC_DIR/ChemicalTagger.jar" ] || wget "$url" -O "$OPT_SRC_DIR/ChemicalTagger.jar"
+cp "$OPT_SRC_DIR/ChemicalTagger.jar" "$PKG_DIR/ChemicalTagger.jar"
+[ -t  "$OPT_JAR_DIR/ChemicalTagger.jar" ] || ln -sf  "$PKG_DIR/ChemicalTagger.jar"  "$OPT_JAR_DIR/ChemicalTagger.jar"
+
+cat > /tmp/RbbtChemicalTagger.java <<EOF
+import uk.ac.cam.ch.wwmm.chemicaltagger.Utils;
+import uk.ac.cam.ch.wwmm.chemicaltagger.ChemistryPOSTagger ;
+import uk.ac.cam.ch.wwmm.chemicaltagger.ChemistrySentenceParser;
+import uk.ac.cam.ch.wwmm.chemicaltagger.POSContainer;
+import uk.ac.cam.ch.wwmm.chemicaltagger.Utils;
+import org.antlr.runtime.Token;
+import org.antlr.runtime.tree.Tree;
+import org.apache.commons.lang.StringUtils;
+import nu.xom.Document;
+import java.util.ArrayList;
+
+class RbbtChemicalTagger{
+    public static String[] match(String text){
+
+        ChemistryPOSTagger chemPos = ChemistryPOSTagger.getDefaultInstance();
+
+        POSContainer posContainer = chemPos.runTaggers(text);
+
+        ChemistrySentenceParser chemistrySentenceParser = new ChemistrySentenceParser(posContainer);
+
+        chemistrySentenceParser.parseTags();
+
+        Tree tree = chemistrySentenceParser.getParseTree();
+
+        ArrayList tokens = new ArrayList();
+        ArrayList molecules = new ArrayList();
+        ArrayList new_tokens = new ArrayList();
+        tokens.add(tree);
+        while (tokens.size() > 0){
+            for(int tree_i = 0; tree_i < tokens.size(); tree_i++){
+                Tree subtree = (Tree) tokens.get(tree_i);
+                int type = subtree.getType();
+                if (type == 78){
+                    molecules.add(subtree.getChild(0));
+                }else{
+                    for(int tree_j = 0; tree_j < subtree.getChildCount(); tree_j++){
+                        Tree child = subtree.getChild(tree_j);
+                        new_tokens.add(child);
+                    }
+                }
+            }
+            tokens = new_tokens;
+            new_tokens = new ArrayList();
+        }
+
+        String[] matches = new String[molecules.size()];
+        for(int molecule_i = 0; molecule_i < molecules.size(); molecule_i++){
+            Tree molecule = (Tree) molecules.get(molecule_i);
+            matches[molecule_i] = molecule.getText();
+        }
+        return matches;
+    }
+
+    public static void main(String[] args){
+        String text  = "Alternatively, rearrangement of O-(w-haloalkyl)esters 34 of 2-carboethoxy-N-hydroxypyridine-2-selone affords azonianaphthalenium halides 37 in 79% yield";
+        match(text);
+    }
+}
+
+EOF
+
+env |grep JAVA
+
+
+(env CLASSPATH="$OPT_JAR_DIR/ChemicalTagger.jar:$CLASSPATH" /home/mvazquezg/software/opt/java/jdk/bin/javac /tmp/RbbtChemicalTagger.java && jar uf "$PKG_DIR/ChemicalTagger.jar" -C /tmp RbbtChemicalTagger.class) || (rm "$PKG_DIR/ChemicalTagger.jar" && rmdir $PKG_DIR)

data/share/install/software/Gdep

@@ -0,0 +1,115 @@
+#!/bin/bash
+
+INSTALL_HELPER_FILE="$1"
+RBBT_SOFTWARE_DIR="$2"
+source "$INSTALL_HELPER_FILE"
+
+name='Gdep'
+url="http://people.ict.usc.edu/~sagae/parser/gdep/gdep-beta2.tgz"
+
+
+get_pkg "$name" "$url"
+uncompress_pkg "$name"
+
+cd "$(echo $OPT_BUILD_DIR/`ls $OPT_BUILD_DIR |head -n 1`)" 
+
+pwd > /tmp/pwd
+ls > /tmp/ls
+
+cat >> Makefile <<'EOF'
+libgdep.so: $(OBJS) ksdep.h
+	$(CPP) -o libgdep.so $(CFLAGS) $(OBJS) -shared
+EOF
+
+cat Makefile| sed 's/\(CFLAGS\s*=\)/\1 -fPIC/' > /tmp/clean.Makefile
+mv /tmp/clean.Makefile Makefile
+
+make gdep
+make libgdep.so
+
+mkdir lib
+mv libgdep.so lib
+
+build "$name" "$extra"
+
+cd "$OPT_BUILD_DIR/$name"
+
+mkdir ruby
+cat > ruby/extconf.rb <<'EOF'
+require 'mkmf-rice'
+dir_config('gdep')
+have_library('gdep')
+create_makefile('Gdep')
+EOF
+
+cat > ruby/Gdep.cpp <<'EOF'
+#include "rice/Class.hpp"
+#include "rice/String.hpp"
+
+#include <stdio.h>
+#include <fstream>
+#include <map>
+#include <list>
+#include <iostream>
+#include <sstream>
+#include "../maxent.h"
+#include "../common.h"
+
+using namespace Rice;
+using namespace std;
+void init_morphdic();
+extern void load_ne_models();
+string bidir_postag(const string & s, const vector<ME_Model> & vme, const vector<ME_Model> & cvme, bool dont_tokenize);
+
+
+vector<ME_Model> vme(16);
+vector<ME_Model> vme_chunking(16);
+bool gdep_loaded = false;
+
+void load_gdep()
+{
+  init_morphdic();
+
+  for (int i = 0; i < 16; i++) {
+    char buf[1000];
+    sprintf(buf, "./models_medline/model.bidir.%d", i);
+    vme[i].load_from_file(buf);
+  }
+
+
+  for (int i = 0; i < 8; i +=2 ) {
+    char buf[1000];
+    sprintf(buf, "./models_chunking/model.bidir.%d", i);
+    vme_chunking[i].load_from_file(buf);
+  }
+
+  load_ne_models();
+
+  gdep_loaded = true;
+}
+
+bool gdep_is_loaded(){
+  return gdep_loaded;
+}
+
+string tag(string line){
+  return(bidir_postag(line, vme, vme_chunking, false));
+}
+
+extern "C"
+void Init_Gdep()
+{
+  Class rb_cGdep =
+    define_class("Gdep")
+    .define_method("load_gdep", &load_gdep)
+    .define_method("tag", &tag)
+    .define_method("gdep_is_loaded", &gdep_is_loaded);
+}
+EOF
+
+cd ruby
+ruby extconf.rb --with-gdep-dir="$OPT_DIR/$name"  
+make
+
+
+

data/share/install/software/Geniass

@@ -0,0 +1,118 @@
+#!/bin/bash
+
+INSTALL_HELPER_FILE="$1"
+RBBT_SOFTWARE_DIR="$2"
+source "$INSTALL_HELPER_FILE"
+
+name='Geniass'
+url="http://www-tsujii.is.s.u-tokyo.ac.jp/~y-matsu/geniass/geniass-1.00.tar.gz"
+
+
+get_pkg "$name" "$url"
+uncompress_pkg "$name"
+
+cd "$(echo $OPT_BUILD_DIR/`ls $OPT_BUILD_DIR |head -n 1`)" 
+
+pwd > /tmp/pwd
+ls > /tmp/ls
+
+cat >> Makefile <<'EOF'
+libgeniass.so: maxent.o  blmvm.o
+	$(CXX) $(CFLAGS) -o $@ -shared $^
+EOF
+
+cat Makefile| sed 's/\(CFLAGS\s*=\)/\1 -fPIC/' > /tmp/clean.Makefile
+mv /tmp/clean.Makefile Makefile
+
+make geniass
+make libgeniass.so
+
+mkdir lib
+mv libgeniass.so lib
+
+build "$name" "$extra"
+
+cd "$OPT_BUILD_DIR/$name"
+
+mkdir ruby
+cat > ruby/extconf.rb <<'EOF'
+require 'mkmf-rice'
+dir_config('geniass')
+have_library('geniass')
+create_makefile('Geniass')
+EOF
+
+cat > ruby/Geniass.cpp <<'EOF'
+#include "rice/Class.hpp"
+#include "rice/String.hpp"
+
+#include <iostream>
+#include <iomanip>
+#include <string>
+#include <list>
+#include <cstdio>
+#include <cstdlib>
+#include <fstream>
+#include <sstream>
+#include "../maxent.h"
+
+using namespace Rice;
+using namespace std;
+
+ME_Model model;
+bool geniass_loaded = false;
+
+void load_geniass(){
+    printf("loading model");
+    string modelFile = "model1-1.0";
+    model.load_from_file(modelFile.c_str());
+    geniass_loaded = true;
+    printf("..done\n");
+}
+
+bool geniass_is_loaded(){ return(geniass_loaded); };
+
+void split(string& str, vector<string>& tokens)
+{
+    istringstream in(str);
+    char c;
+
+    while (in){
+        string token;
+        token = "";
+        while (in.get(c) && (c != '\t')) token.push_back(c);
+        tokens.push_back(token);
+    }
+}
+
+string label(string line){
+    vector<string> tokens;
+    split(line, tokens);
+    ME_Sample s;
+
+    for(vector<string>::const_iterator token = tokens.begin() + 1;
+        token != tokens.end(); ++token){
+        s.add_feature(*token);
+    }
+    (void) model.classify(s);
+    return(s.label);
+}
+
+extern "C"
+void Init_Geniass()
+{
+  Class rb_cGeniass =
+    define_class("Geniass")
+    .define_method("load_geniass", &load_geniass)
+    .define_method("label", &label)
+    .define_method("geniass_is_loaded", &geniass_is_loaded);
+}
+
+EOF
+
+cd ruby
+ruby extconf.rb --with-geniass-dir="$OPT_DIR/$name"  
+make
+
+
+