bio-exominer 0.0.3

data/bin/pack_exominer_symbols

@@ -0,0 +1,38 @@
+#! /usr/bin/env ruby
+#
+# Pack symbol file
+#
+# Copyright (C) 2013 Pjotr Prins <pjotr.prins@thebird.nl> 
+#
+
+gempath = File.dirname(File.dirname(__FILE__))
+$: << File.join(gempath,'lib')
+
+require 'msgpack'
+require 'bio-exominer/symbols'
+
+include BioExominer
+
+if ARGV[0] == '-d'
+  ARGV.shift
+
+  print "Unpacking symbols.bin..."
+  u = MessagePack::Unpacker.new(File.new('symbols.bin','rb'))
+  begin
+    u.each do |obj|
+      print obj[0],"\t",(obj[1] ? obj[1].join('|') : "NA"),"\t",obj[2],"\n"
+    end
+  rescue EOFError
+  end
+else
+
+  print "Writing symbols.bin..."
+  bin = MessagePack::Packer.new(File.new('symbols.bin','wb'))
+
+  ARGF.each_line do | line |
+    symbol,aliases,descr = Symbols::parse_line(line)
+    bin.write([symbol,aliases,descr])
+  end
+  bin.flush
+
+end

data/features/bio-exominer.feature

@@ -0,0 +1,9 @@
+Feature: something something
+  In order to something something
+  A user something something
+  something something something
+
+  Scenario: something something
+    Given inspiration
+    When I create a sweet new gem
+    Then everyone should see how awesome I am

data/features/support/env.rb

@@ -0,0 +1,13 @@
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../../lib')
+require 'bio-exominer'
+
+require 'rspec/expectations'

data/lib/bio-exominer.rb

@@ -0,0 +1,14 @@
+# Please require your code below, respecting the naming conventions in the
+# bioruby directory tree.
+#
+# For example, say you have a plugin named bio-plugin, the only uncommented
+# line in this file would be 
+#
+#   require 'bio/bio-plugin/plugin'
+#
+# In this file only require other files. Avoid other source code.
+
+require 'bio-exominer/rdf'
+require 'bio-exominer/textparser'
+require 'bio-exominer/exominer'
+

data/lib/bio-exominer/exominer.rb

@@ -0,0 +1,3 @@
+
+module BioExominer
+end

data/lib/bio-exominer/rdf.rb

@@ -0,0 +1,38 @@
+module BioExominer
+
+require 'uri'
+
+  # FIXME: use bioruby-rdf modules instead! It is all there now.
+  module RDF
+
+    def RDF::valid_uri? uri
+      uri =~ /^([!#$&-;=?_a-z~]|%[0-9a-f]{2})+$/i
+    end
+
+    # An identifier is used for the subject and predicate in RDF. This is a case-sensitive
+    # (shortened) URI. You can change default behaviour for identifiers using the options 
+    # --transform-ids (i.e. in the input side, rather than the output side)
+    #
+    def RDF::make_identifier(s)
+      id = s.strip.gsub(/[^[:print:]]/, '').gsub(/[#)(,]/,"").gsub(/[%]/,"perc").gsub(/(\s|\.|\$|\/|\\)+/,"_")
+      # id = URI::escape(id)
+      id = id.gsub(/\|/,'_')
+      id = id.gsub(/\-|:/,'_')
+      if id != s 
+        # logger = Bio::Log::LoggerPlus['bio-table']
+        $stderr.print "\nWARNING: Changed identifier <#{s}> to <#{id}>"
+      end
+      if not RDF::valid_uri?(id)
+        raise "Invalid URI after mangling <#{s}> to <#{id}>!"
+      end
+      valid_id = if id =~ /^\d/
+                   'r' + id
+                 else
+                   id
+                 end
+      valid_id
+    end
+
+  end
+end
+

data/lib/bio-exominer/symbols.rb

@@ -0,0 +1,49 @@
+module BioExominer
+
+  require 'msgpack'
+  require 'bio-exominer/rdf'
+
+  module Symbols
+
+    # Make a full URI out of a symbol
+    def Symbols::uri(symbol,hugo)
+      if hugo[symbol]
+        # http://bio2rdf.org/hugo:RAD51C
+        "hgnc:"+RDF::make_identifier(symbol)
+      else
+        "ncbigene:"+RDF::make_identifier(symbol)  # remove all non-printable
+      end
+    end
+
+    def Symbols::parse_line(line)
+      symbol,aliases,descr = line.strip.split(/\t/)
+      aliases = 
+        if aliases == 'NA'
+          nil
+        else
+          aliases.split(/\|/)
+        end
+      return symbol,aliases,(descr ? descr.strip : "")
+    end
+
+    def Symbols::each(fn)
+      is_bin = fn =~ /.bin$/
+
+      if is_bin
+        u = MessagePack::Unpacker.new(File.new(fn,'rb'))
+        begin
+          u.each do |obj|
+            # print obj[0],"\t",(obj[1] ? obj[1].join('|') : "NA"),"\t",obj[2],"\n"
+            yield obj[0],obj[1],obj[2]
+          end
+        rescue EOFError
+        end
+      else
+        File.open(fn).each_line do | line |
+          yield parse_line(line)
+        end
+      end
+    end
+  end
+
+end

data/lib/bio-exominer/textparser.rb

@@ -0,0 +1,124 @@
+# Text parsing
+
+module BioExominer
+
+  module TextParser 
+
+    MAX_SIZE = 120
+    SKIP_TOKENS = %w{ can has Ma van large was polymerase had far a impact was East early 
+      face Park ali and team tag ras ac tail at al age ac TA tag small this pure such
+      type gene pmc but is ten org we an term not as by lost et out how up per for
+      end beta der The Ten Out At No How pdf Ding Jan To cell gov even Jun
+      Sun DNA Nat in hit doc bin with set Nov unknown key link to cgi
+      and or RDF NPG
+    }
+
+    # L3MBTL
+
+    def TextParser::valid_token? token
+      return false if token.strip == "" 
+      return false if token =~ /^(\d|[,])+$/
+      return false if token !~ /[a-zA-Z]/  # at least one word char
+      true
+    end
+
+    def TextParser::add tokens, word
+      return if SKIP_TOKENS.include?(word)
+      # return if word.size < 2
+      tokens[word] ||= 0 
+      tokens[word] += 1
+    end
+
+    def TextParser::rm_punctuation w
+      return nil if w == nil
+      word = w.dup
+      if word =~ /^\[\d+\]/
+        word = word.sub(/^\[\d+\]/,'')
+      end
+      word = word.sub(/^\(/,'')
+      word = word.sub(/\)$/,'')     
+      word = word.sub(/[,:;!]$/,'') # remove punctuation
+      word = word.sub(/^[`"']/,'')  # remove starting quotes
+      word = word.sub(/[`"']$/,'')  # remove ending quotes
+      word
+    end
+
+    # Return tokens with count
+    def TextParser::tokenize buf
+      tokens = {}
+      list = buf.split(/[\r\n\s]+/)
+      list.each_with_index do | word,idx |
+        n1 = p1 = nil
+        p1 = rm_punctuation(list[idx-1]) if idx>0
+        w1 = rm_punctuation(word)
+        n1 = rm_punctuation(list[idx+1]) if idx<list.size
+        next if w1.size < 2
+        next if p1 =~ /table|dataset|supplement|figure|chapter|section|paragraph/i 
+        # Filter out letters+name
+        if w1 =~ /^[A-Z]/ and w1.capitalize == w1
+          next if n1 and n1.size == 1
+          next if p1 and p1.size == 1
+          next if n1 and n1.size == 2 and n1 =~ /^[A-Z][A-Z]/
+          next if p1 and p1.size == 2 and p1 =~ /^[A-Z][A-Z]/
+        end
+        if w1.size == 2 and w1 =~ /^[A-Z][A-Z]/
+          next if p1 and p1 =~ /^[A-Z]/ and p1.capitalize == p1
+          next if n1 and n1 =~ /^[A-Z]/ and n1.capitalize == n1
+        end
+        # Filter out all lowercase small names
+        next if w1.size < 4 and w1 == w1.downcase and w1 !~ /\d/
+        # Remove brackets and braces in first and last positions
+        add(tokens,w1) if TextParser.valid_token?(word)
+        # p [word,w1,TextParser.valid_token?(word)]
+        add(tokens,word) if TextParser.valid_token?(word) and word != w1
+        # split on dash or underscore
+        if word =~ /-|_/
+          word.split(/-|_/).each do |w|
+            add(tokens,w) if TextParser.valid_token?(w)
+          end
+        end
+      end
+      # p tokens
+      tokens
+    end
+
+    # Return a list of tokens with count and context
+    def TextParser::tokenize_with_context buf, context_type = :sentence
+      tokens_context = {}
+      tokens_count = {}
+      # Split buf into sentences based on dots or newlines
+      sentences = 
+        if context_type == :line or context_type == 'line'
+          buf.split(/\n/)
+        else
+          buf.split(/\.\s+/)
+        end
+      sentences.each do | sentence1 |
+        sentence = sentence1.strip.gsub(/(\r|\n)\s*/,' ') 
+        # remove quotes
+        sentence = sentence.gsub(/"/,'')
+        tokens = tokenize(sentence)
+        tokens.each { | token, count |
+          # shorten the sentence
+          sentence2 = 
+            if sentence.size > MAX_SIZE+2
+              half_size = MAX_SIZE/2
+              pos = sentence.index(token)
+              start = (pos-half_size<0 ? 0 : pos-half_size)
+              stop  = pos+half_size
+              s2 = sentence[start..stop]
+              s2.sub(/^\w+\s+/,'').sub(/\s+\w+$/,'')
+            else
+              sentence
+            end
+          tokens_count[token] ||= 0
+          tokens_count[token] += count
+          tokens_context[token] ||= []
+          tokens_context[token] << sentence2
+        }
+      end
+      return tokens_count, tokens_context
+    end
+  end
+
+end

data/scripts/4store.sh

@@ -0,0 +1,30 @@
+#! /bin/sh
+#
+#  Options
+#
+#    -r     Restart server
+#    -d     Delete DB and restart server
+
+dbname=exominer
+if [ ! -z $1 ] ; then
+  dbname=$1
+  shift
+fi
+
+echo Starting DB $dbname
+
+if [ "$1" = "-r" ]; then
+  killall 4s-httpd
+  killall 4s-backend
+fi
+
+if [ "$1" = "-d" ]; then
+  killall 4s-httpd
+  killall 4s-backend
+  4s-backend-setup $dbname
+fi
+
+
+4s-backend $dbname
+4s-httpd -p 8081 $dbname
+

data/scripts/example.sh

@@ -0,0 +1,9 @@
+#! /bin/sh
+
+
+./bin/exominer --rdf --name tcga_bc --hugo --tag 'title=Comprehensive molecular portraits of human breast tumours' --tag 'year=2012;species=human;type=breast cancer' -s ncbi_symbols.bin < tcga_bc.txt > tcga_bc.rdf
+
+curl -T tcga_bc.rdf -H 'Content-Type: application/x-turtle' http://localhost:8081/data/exominer.rdf
+
+~/opt/bin/sparql-query http://localhost:8081/sparql/ 'SELECT * WHERE { ?s ?p ?o } LIMIT 5'
+

data/scripts/example_rdf.sh

@@ -0,0 +1,7 @@
+#! /bin/sh
+
+
+./bin/exominer --rdf --name tcga_bc --hugo --doi doi:10.1038/nature11412 --tag 'title=Comprehensive molecular portraits of human breast tumours' --tag 'year=2012;species=human;type=breast cancer' < tcga_bc.txt 
+./bin/exominer --rdf --name tcga_bc --hugo --doi doi:10.1038/nature11412 --tag 'title=Comprehensive molecular portraits of human breast tumours' --tag 'year=2012;species=human;type=breast cancer' < tcga_bc.txt > tcga_bc.rdf
+
+curl -T tcga_bc.rdf -H 'Content-Type: application/x-turtle' http://localhost:8081/data/exominer.rdf

data/scripts/load_rdf.sh

@@ -0,0 +1,15 @@
+#! /bin/sh
+
+PORT=8081
+
+for rdf in $* ; do
+  echo "Loading $rdf"
+  # ---- test syntax
+  rapper -i turtle $rdf > /dev/null
+
+  uri=http://localhost:$PORT/data/http://biobeat.org/data/$rdf
+
+  curl -X DELETE $uri
+  curl -T $rdf -H 'Content-Type: application/x-turtle' $uri
+done
+

data/spec/bio-exominer_spec.rb

@@ -0,0 +1,8 @@
+# require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+# 
+# describe "BioExominer" do
+#   it "fails" do
+#     fail "hey buddy, you should probably rename this file and start specing for real"
+#   end
+# end
+

data/spec/rdf_spec.rb

@@ -0,0 +1,28 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+require 'minitest'
+
+class TestRDF < MiniTest::Test
+  
+  include BioExominer
+
+  # def test_uri_escape
+    # assert_equal !RDF::escape("")
+  # end
+
+  def test_uri_validator
+    # invalid
+    assert !RDF::valid_uri?("use`quote")
+    # assert !RDF::valid_uri?("use%7quote")
+
+    # valid
+    assert RDF::valid_uri?("use%07quote")
+
+  end
+
+  def test_make_identifier
+    assert_equal RDF::make_identifier("AA"), "AA"
+    assert_equal RDF::make_identifier("use:colon:"), "use_colon_"
+    assert_equal RDF::make_identifier("use|pipe"), "use_pipe"
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,19 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+# require 'rspec'
+require 'rubygems'
+gem 'minitest' # ensures you're using the gem, and not the built in MT
+require 'minitest/autorun'
+
+require 'bio-exominer'
+
+include BioExominer 
+
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+# Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+# RSpec.configure do |config|
+  
+# end

data/spec/text_parser_spec.rb

@@ -0,0 +1,59 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+class TestTextParser < MiniTest::Test
+  
+  include BioExominer
+
+  BUF =<<TEXT
+
+  Hello world. Test gene GEN1.X. This is with context! I don't believe this is true,
+  and that you can do this. Love Ruby, love RDF. Love the combination.
+  Fish the token out of a very long sentence
+  Fish the token out of a very long sentence
+  Fish the token out of a very long sentence
+  Fish the token out of a very long sentence
+  Fish the token out of a very long sentence
+  Fish the token "GEN2.X" out of a very long sentence
+  Fish the token out of a very long sentence
+  Fish the token out of a very long sentence
+  Fish the token out of a very long sentence
+  Fish the token out of a very long sentence
+  Fish the token out of a very long sentence
+  Fish the token out of a very long sentence
+  Fish the token out of a very long sentence
+TEXT
+
+  def test_tokenize_with_context
+    counts,match = TextParser::tokenize_with_context(BUF)
+    assert_equal counts['world'],1
+    assert_equal counts['Love'],2
+    assert_equal match['world'], ['Hello world']
+    assert_equal match['Love'], ['Love Ruby, love RDF', 'Love the combination']
+    assert_equal match['context'], ['This is with context! I don\'t believe this is true, and that you can do this']
+    assert_equal match['GEN1.X'], ['Test gene GEN1.X']
+    assert_equal match['GEN2.X'], ['Fish the token out of a very long sentence Fish the token GEN2.X out of a very long sentence Fish the token out of a']
+    # ---- Line based context
+    counts,match = TextParser::tokenize_with_context(BUF,context=:line)
+    assert_equal match['GEN2.X'], ['Fish the token GEN2.X out of a very long sentence']
+  end
+
+  BUF2 =<<TEXT2
+  valid token figure S11 table XX p53
+  Invalid MD, and RD Jester, Wikkel W, Wokkel WOS 
+TEXT2
+
+  def test_valid_tokens 
+    match = TextParser::tokenize(BUF2)
+    assert match['token']
+    assert !match['S11']
+    assert !match['XX']
+    assert !match['Wokkel']
+    assert match['p53']
+    assert match['WOS']
+    assert !match['MD']
+    assert !match['Invalid']
+    assert !match['RD']
+    assert !match['Jester']
+    assert !match['Wikkel']
+    assert !match['W']
+  end
+end