biblicit 2.1.0 → 2.2.0

data/biblicit.gemspec

@@ -26,7 +26,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'rake'
   gem.add_development_dependency 'rspec'
   gem.add_development_dependency 'pry'
-  gem.add_development_dependency 'pry-debugger'
 
   gem.requirements << 'For PDFs, Poppler or XPDF (try "which pdftotext")'
   gem.requirements << 'For Postscript files, Ghostscript (try "which ps2ascii")'

data/lib/biblicit/extractor.rb

@@ -1,7 +1,6 @@
 # encoding: UTF-8
 
 require 'biblicit/cb2bib'
-require 'biblicit/citeseer'
 require 'biblicit/parscit'
 
 require 'tempfile'
@@ -30,7 +29,7 @@ module Biblicit
 
     def self.extract_from_file(file, opts)
       file = File.realpath(file)
-      tools = opts.delete(:tools) || [:parshed, :citeseer]
+      tools = opts.delete(:tools) || [:parshed]
 
       result = {}
 
@@ -38,11 +37,7 @@ module Biblicit
         `#{SH_DIR}/convert_to_text.sh #{file.shellescape} #{in_txt.path}`
 
         if tools.include?(:parshed)
-          result.merge!( parshed: ParsCit.extract(in_txt, opts) )
-        end
-
-        if tools.include?(:citeseer)
-          result.merge!( citeseer: CiteSeer.extract(in_txt, opts) )
+          result.merge!( ParsCit.extract(in_txt, opts) )
         end
 
         if tools.include?(:cb2bib)

data/lib/biblicit/parscit.rb

@@ -17,20 +17,32 @@ module ParsCit
     attr_reader :result
 
     def initialize(in_txt, opts={})
+      mode = (opts.fetch :include_citations, false) ? 'extract_all' : 'extract_header'
       ENV['CRFPP_HOME'] ||= "#{File.dirname(`which crf_test`)}/../"
-      output = `#{PERL_DIR}/bin/citeExtract.pl -q -m extract_all #{in_txt.path}`
+      output = `#{PERL_DIR}/bin/citeExtract.pl -q -m #{mode} #{in_txt.path}`
       @result = parse(Nokogiri::XML output)
     end
 
   private
 
     def parse(xml)
-      parsed = xml.css("algorithm[name=ParsHed]")
+      result = {}
+
+      parshed = xml.css("algorithm[name=ParsHed]")
+      result[:parshed] = {
+        title: parshed.css('title').text.gsub(/\s+/,' ').strip,
+        authors: parshed.css('author').map { |a| a.text.gsub(/\s+/,' ').strip },
+        abstract: parshed.css('abstract').text#,
+        #confidence: parshed.css('title').attr('confidence').value.to_f
+      }
 
-      result = {
-        title: parsed.css('title').text.gsub(/\s+/,' ').strip,
-        authors: parsed.css('author').map { |a| a.text.gsub(/\s+/,' ').strip },
-        abstract: parsed.css('abstract').text
+      svm = xml.css('algorithm[name="SVM HeaderParse"]')
+      result[:citeseer] = {
+        title: svm.css('title').text,
+        authors: svm.css('author > name').map { |n| n.text.strip }.reject(&:blank?).uniq,
+        author_emails: svm.css('author > email').map { |n| n.text.strip }.reject(&:blank?).uniq,
+        abstract: svm.css('abstract').text,
+        valid: svm.css('validHeader').first.try(:text) == '1'
       }
 
       citations = xml.css('algorithm[name=ParsCit] > citationList > citation').map do |node|

data/lib/biblicit/version.rb

@@ -2,6 +2,6 @@
 
 module Biblicit
 
-  VERSION = '2.1.0'
+  VERSION = '2.2.0'
 
 end

data/parscit/bin/citeExtract.pl

@@ -33,8 +33,8 @@ use Getopt::Std;
 use strict 'vars';
 use lib $FindBin::Bin . "/../lib";
 
-use lib "/home/wing.nus/tools/languages/programming/perl-5.10.0/lib/5.10.0";
-use lib "/home/wing.nus/tools/languages/programming/perl-5.10.0/lib/site_perl/5.10.0";
+#use lib "/home/wing.nus/tools/languages/programming/perl-5.10.0/lib/5.10.0";
+#use lib "/home/wing.nus/tools/languages/programming/perl-5.10.0/lib/site_perl/5.10.0";
 
 # Dependencies
 use File::Spec;
@@ -42,6 +42,9 @@ use File::Basename;
 
 # Local libraries
 use ParsCit::Controller;
+
+use HeaderParse::API::Parser;
+use HeaderParse::Config::API_Config;
 	
 # USER customizable section
 my $tmpfile	.= $0; 
@@ -58,6 +61,7 @@ my $progname	= $1;
 my $PARSCIT		= 1;
 my $PARSHED		= 2;
 my $SECTLABEL	= 4; # Thang v100401
+my $SVM       = 8;
 
 my $default_input_type	= "raw";
 my $output_version		= "110505";
@@ -228,6 +232,14 @@ if (($mode & $PARSCIT) == $PARSCIT)
   if (scalar(@export_types) != 0) { BiblioScript(\@export_types, $$pc_xml, $out); }
 }
 
+# SVM HEADER PARSE
+if (($mode & $SVM) == $SVM)
+{
+  my $svm_xml = HeaderParse::API::Parser::extractHeader($text_file);
+
+  $rxml .= $$$svm_xml . "\n";
+}
+
 $rxml .= "</algorithms>";
 
 if (defined $out) 
@@ -267,7 +279,7 @@ sub ParseMode
 	} 
 	elsif ($arg eq "extract_header") 
 	{
-		return $PARSHED;
+		return ($PARSHED | $SVM);
 	} 
 	elsif ($arg eq "extract_citations") 
 	{
@@ -279,7 +291,7 @@ sub ParseMode
 	} 
 	elsif ($arg eq "extract_all") 
 	{
-		return ($PARSHED | $PARSCIT | $SECTLABEL);
+		return ($PARSHED | $PARSCIT | $SECTLABEL | $SVM);
 	} 
 	else 
 	{

data/{svm-header-parse/HeaderParseService → parscit}/lib/HeaderParse/API/Parser.pm

@@ -20,6 +20,27 @@ use HeaderParse::Config::API_Config;
 use vars qw($ServerURL $repositoryLocation $algVersion);
 
 
+sub extractHeader {
+    my ($textFile) = @_;
+
+    my $jobID;
+    while($jobID = rand(time)) {
+	unless(-f $offlineD."$jobID") {
+	    last;
+	}
+    }
+
+    my ($status, $msg, $rXML) = _parseHeader($textFile, $jobID);
+
+    if ($status <= 0) {
+      my $error = "Error: $msg";
+	    return \$error;
+    }
+    else {
+      return \$rXML;
+    }
+}
+
 sub _parseHeader{
     my ($fileID, $jobID) = @_;
     my ($header, $faultMessage, $rResponse, $success, $papertext);

data/{svm-header-parse/HeaderParseService → parscit}/lib/HeaderParse/Config/API_Config.pm

@@ -19,12 +19,6 @@ use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
 @ISA = qw(Exporter);
 @EXPORT =  qw($Classifier $offlineD $Database_Dir $Data_Dir $Tmp_Dir $nMinHeaderLength $nMaxHeaderLength $ServerURL $ServerPort $algName $algVersion);
 
-#$Database_Dir = "$FindBin::Bin/../lib/HeaderParse/database";
-#$Data_Dir = "$FindBin::Bin/../lib/HeaderParse/data/";
-#$offlineD = "$FindBin::Bin/../lib/HeaderParse/OfflineFiles/";
-
-$HeaderParseHome = "$FindBin::Bin/HeaderParseService";
-
 if ($ENV{'SVM_LIGHT_HOME'}.length) {
   $Classifier = "$ENV{'SVM_LIGHT_HOME'}/svm_classify" 
 }
@@ -32,10 +26,17 @@ else {
   $Classifier = "svm_classify5"; # assume on path
 }
 
-$Database_Dir = "$HeaderParseHome/resources/database/";
-$Data_Dir = "$HeaderParseHome/resources/data/";
-$offlineD = "$HeaderParseHome/resources/models/";
-$Tmp_Dir = "$HeaderParseHome/tmp";
+my $path;
+BEGIN
+{
+	if ($FindBin::Bin =~ /(.*)/) { $path = $1; }
+}
+
+$Resource_Dir = "$path/../resources/headerParse";
+$Database_Dir = "$Resource_Dir/database/";
+$Data_Dir = "$Resource_Dir/data/";
+$offlineD = "$Resource_Dir/models/";
+$Tmp_Dir = "$path/../tmp";
 
 $nMinHeaderLength = 50;
 $nMaxHeaderLength = 2500;