biblicit 1.0 → 2.0.3

data/parscit/bin/sectLabel/genericSect/crossValidation.rb

@@ -0,0 +1,98 @@
+#!/usr/bin/env ruby
+require 'find'
+
+pwd = File.dirname(__FILE__)
+
+@CRFPP  = "#{ENV['CRFPP_HOME']}/bin"
+@RESOURCES   = "#{pwd}/../../../crfpp/traindata/"
+@TEST_DIR = "#{pwd}/run"
+@CONLLEVAL = "#{pwd}/../../conlleval.pl"
+if ARGV.length == 2
+dataFile = ARGV[0]
+numFold  = ARGV[1].to_i
+f = File.open("#{dataFile}")
+contentArray = Array.new
+while !f.eof do
+	l = f.gets.chomp
+	contentArray << l
+end
+f.close
+
+numInstances = (contentArray.length/numFold).to_i
+
+iFold = 1
+while iFold <= numFold do
+	startIndex = (iFold - 1)*numInstances
+	endIndex = iFold*numInstances - 1
+	if iFold == numFold
+		endIndex = contentArray.length - 1
+	end
+
+	i = 0
+	f = File.open("#{@TEST_DIR}/gs_train.txt","w")
+	g = File.open("#{@TEST_DIR}/gs_test.txt","w")
+	while i < contentArray.length do
+		str = contentArray[i]
+		if i >= startIndex and i <= endIndex
+			g.write("#{str}\n")
+		else
+			f.write("#{str}\n")
+		end
+		i = i + 1
+	end
+	f.close
+	g.close
+
+	cmd = "#{pwd}/../single2multi.pl -in #{@TEST_DIR}/gs_train.txt -out #{@TEST_DIR}/gs_train_#{iFold}.txt"
+	puts "#{cmd}"
+	system(cmd)
+	cmd = "#{pwd}/../single2multi.pl -in #{@TEST_DIR}/gs_test.txt -out #{@TEST_DIR}/gs_test_#{iFold}.txt"
+	system(cmd)
+
+	#create train and test file
+	cmd = "ruby extractFeature.rb #{@TEST_DIR}/gs_train_#{iFold}.txt  > #{@TEST_DIR}/gs_train_#{iFold}.train "
+	system(cmd)
+	cmd = "ruby extractFeature.rb #{@TEST_DIR}/gs_test_#{iFold}.txt > #{@TEST_DIR}/gs_test_#{iFold}.test"
+	system(cmd)
+
+	#run crf
+
+	cmd = "#{@CRFPP}/crf_learn #{@RESOURCES}/genericSect.template #{@TEST_DIR}/gs_train_#{iFold}.train #{@TEST_DIR}/gs_train_#{iFold}.model"
+	system(cmd)
+
+	cmd = "#{@CRFPP}/crf_test  -m #{@TEST_DIR}/gs_train_#{iFold}.model #{@TEST_DIR}/gs_test_#{iFold}.test > #{@TEST_DIR}/gs_test_#{iFold}.out"
+	system(cmd)
+
+	cmd = "cat #{@TEST_DIR}/gs_test_#{iFold}.out >> #{@TEST_DIR}/gs_#{numFold}_result.txt"
+	system(cmd)
+
+	iFold = iFold + 1
+end
+cmd = "#{@CONLLEVAL} -r  -d \"\t\" <  #{@TEST_DIR}/gs_#{numFold}_result.txt"
+puts "#{cmd}"
+system(cmd)
+
+cmd = "rm #{@TEST_DIR}/gs_*"
+system(cmd)
+
+else
+	puts "Usage ruby crossValidation.rb dataFile numFold"
+
+end
+
+### v100401 ###
+#processed 2366 tokens with 2366 phrases; found: 2366 phrases; correct: 2259.
+#accuracy:  95.48%; precision:  95.48%; recall:  95.48%; FB1:  95.48
+#         abstract: precision: 100.00%; recall: 100.00%; FB1: 100.00  210
+#  acknowledgments: precision:  98.08%; recall: 100.00%; FB1:  99.03  104
+#       background: precision:  91.67%; recall:  39.29%; FB1:  55.00  12
+#categories-and-subject-descriptors: precision: 100.00%; recall: 100.00%; FB1: 100.00  165
+#      conclusions: precision:  94.82%; recall:  96.83%; FB1:  95.81  193
+#      discussions: precision:  80.95%; recall:  47.22%; FB1:  59.65  21
+#       evaluation: precision:  93.16%; recall:  72.19%; FB1:  81.34  117
+#    general-terms: precision: 100.00%; recall: 100.00%; FB1: 100.00  142
+#     introduction: precision:  99.05%; recall:  99.52%; FB1:  99.29  211
+#         keywords: precision:  99.52%; recall: 100.00%; FB1:  99.76  210
+#           method: precision:  88.33%; recall:  98.36%; FB1:  93.07  677
+#       references: precision: 100.00%; recall: 100.00%; FB1: 100.00  211
+#    related-works: precision: 100.00%; recall:  88.57%; FB1:  93.94  93

data/parscit/bin/sectLabel/genericSect/extractFeature.rb

@@ -0,0 +1,104 @@
+#!/usr/bin/env ruby
+#Author Nguyen Thuy Dung
+require 'find'
+#get relative pos in ingeter, values range from 0-10
+def getPos (val)
+	if val == 0
+		return 0
+	end
+	i = 1
+	while i <= 10 do
+		if val <= (i/10.0) 
+			return i
+		end
+		i = i + 1
+	end
+	return -1
+end
+
+#process generic headers: "related work" becomes "related_work"
+def getHeader(str)
+	str = str.strip
+	while str.index(" ") != nil do
+		str = str.sub(" ", "_")
+	end
+	return str.downcase
+end
+
+
+f = File.open("#{ARGV[0]}")
+hea_array = Array.new
+ahea_array = Array.new
+while !f.eof do
+	l = f.gets.chomp.strip
+	if l != ""
+		tmp_array = l.split("|||")
+		if tmp_array.length == 1
+			hea_array << tmp_array[0].strip			
+			ahea_array << "?"
+		else
+			hea_array << tmp_array[1].strip			
+			ahea_array << tmp_array[0].strip
+		end
+	else
+		index = 0
+		while index < hea_array.length do
+			if hea_array.length == 1
+				pos = 0
+			else
+				pos = getPos(index*1.0/(hea_array.length - 1))	
+			end
+			currHeader = getHeader(hea_array.at(index))
+			assignedHeader = getHeader(ahea_array.at(index))
+			tmp = hea_array.at(index).split(" ")
+			len = tmp.length
+			if  len > 3
+				len = 3
+			end	
+			firstWord = tmp.at(0)
+			secondWord = "null"
+			if len >= 2
+				secondWord = tmp.at(1)
+			end
+			puts "index=#{index} pos=#{pos}/10 firstWord=#{firstWord} secondWord=#{secondWord}  currHeader=#{currHeader} #{assignedHeader}"
+			index = index + 1
+		end
+		puts ""
+		hea_array = Array.new
+		ahea_array = Array.new
+	end
+end
+f.close
+if hea_array.length > 0
+index = 0
+while index < hea_array.length do
+	if hea_array.length == 1
+		pos = 0
+	else
+		pos = getPos(index*1.0/(hea_array.length - 1))	
+	end
+	currHeader = getHeader(hea_array.at(index))
+	assignedHeader = getHeader(ahea_array.at(index))
+	tmp = hea_array.at(index).split(" ")
+	len = tmp.length
+	if  len > 3
+		len = 3
+	end	
+	
+	firstWord = tmp.at(0).strip
+	secondWord = "null"
+	if /[0-9]+.?/.match(firstWord) and len > 1
+		firstWord = tmp.at(1).strip
+		if len > 2
+			secondWord = tmp.at(2)
+		end
+	else
+		if len > 1
+			secondWord = tmp.at(1)
+		end
+	end
+	puts "index=#{index} pos=#{pos}/10 firstWord=#{firstWord} secondWord=#{secondWord}  currHeader=#{currHeader} #{assignedHeader}"
+	index = index + 1
+end
+puts ""
+end

data/parscit/bin/sectLabel/genericSectExtract.rb

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+require 'find'
+# Emma
+
+pwd = File.dirname(__FILE__)
+
+@CRFPP  = "#{ENV['CRFPP_HOME']}/bin"
+@SRC    = "#{pwd}/genericSect"
+@DATA   = "#{pwd}/../../resources/sectLabel/"
+@TEST_DIR = "/tmp/"
+
+name  = "#{Time.now.to_i}-#{Process.pid}"
+
+cmd = "ruby #{@SRC}/extractFeature.rb #{ARGV[0]} > #{@TEST_DIR}/#{name}.test"
+system(cmd)
+
+cmd = "#{@CRFPP}/crf_test -m #{@DATA}/genericSect.model  #{@TEST_DIR}/#{name}.test >  #{@TEST_DIR}/#{name}.out"
+system(cmd)
+
+
+if ARGV[1] != nil
+	g = File.open("#{ARGV[1]}", "w")
+
+	cmd = "chmod 777 #{ARGV[1]}"
+	system(cmd)
+end
+
+f = File.open("#{@TEST_DIR}/#{name}.out")
+while !f.eof do
+	str = f.gets.chomp.strip
+	if str != ""
+		l = str.split(" ")
+		output = l.at(l.length-1)
+		while output.index("-") != nil
+			output = output.sub("-", " ")
+		end
+		if output == "related works"
+			output = "related work"
+		end
+		if ARGV[1] == nil
+			puts "#{output}"
+		else
+			g.write("#{output}\n")
+		end
+	end
+end
+f.close
+if ARGV[1] != nil
+	g.close
+end
+
+File.unlink("#{@TEST_DIR}/#{name}.out")
+File.unlink("#{@TEST_DIR}/#{name}.test")

data/parscit/bin/sectLabel/getStructureInfo.pl

@@ -0,0 +1,156 @@
+#!/usr/bin/perl -wT
+# Author: Luong Minh Thang <luongmin@comp.nus.edu.sg>, generated at Thu, 01 Apr 2010 01:15:34
+
+# Modified from template by Min-Yen Kan <kanmy@comp.nus.edu.sg>
+
+require 5.0;
+use strict;
+use Getopt::Long;
+
+# I do not know a better solution to find a lib path in -T mode.
+# So if you know a better solution, I'd be glad to hear.
+# See this http://www.perlmonks.org/?node_id=585299 for why I used the below code
+use FindBin;
+FindBin::again(); # to get correct path in case 2 scripts in different directories use FindBin
+my $path;
+BEGIN {
+  if ($FindBin::Bin =~ /(.*)/) {
+    $path = $1;
+  }
+}
+use lib "$path/../../lib";
+use SectLabel::PreProcess;
+
+### USER customizable section
+$0 =~ /([^\/]+)$/; my $progname = $1;
+my $outputVersion = "1.0";
+### END user customizable section
+
+sub License {
+  print STDERR "# Copyright 2009 \251 by Luong Minh Thang\n";
+}
+
+### HELP Sub-procedure
+sub Help {
+  print STDERR "usage: $progname -h\t[invokes help]\n";
+  print STDERR "       $progname -in inFile -out outFile [-print]\n";
+  print STDERR "Options:\n";
+  print STDERR "\t-q\tQuiet Mode (don't echo license)\n";
+}
+my $QUIET = 0;
+my $HELP = 0;
+my $inFile = undef;
+my $outFile = undef;
+my $isPrint = 0;
+$HELP = 1 unless GetOptions('in=s' => \$inFile,
+			    'out=s' => \$outFile,
+			    'print' => \$isPrint,
+			    'h' => \$HELP,
+			    'q' => \$QUIET);
+
+if ($HELP || !defined $inFile || !defined $outFile) {
+  Help();
+  exit(0);
+}
+
+if (!$QUIET) {
+  License();
+}
+
+### Untaint ###
+$inFile = untaintPath($inFile);
+$outFile = untaintPath($outFile);
+my $envPath = $ENV{'PATH'};
+$envPath = untaintPath($envPath);
+$ENV{'PATH'} = $envPath;
+### End untaint ###
+
+
+open (IN, "<:utf8", "$inFile") || die "#Can't open file \"$inFile\"";
+my $text = "";
+while(<IN>){
+  $text .= $_;
+}
+close IN;
+
+my @lines = split(/\n/, $text);
+my $numLines = scalar(@lines);
+my ($bodyLength, $citationLength, $bodyEndId) =
+	    SectLabel::PreProcess::findCitationText(\@lines, 0, $numLines);
+
+my ($headerLength, $bodyStartId);
+($headerLength, $bodyLength, $bodyStartId) =
+	    SectLabel::PreProcess::findHeaderText(\@lines, 0, $bodyLength);
+
+if($isPrint){
+  open(OF, ">:utf8", "$outFile.header");
+  my @headers = @lines[0..$headerLength];
+  foreach(@headers){
+    print OF "$_\n";
+  }
+  close OF;
+  
+  open(OF, ">:utf8", "$outFile.body");
+  my @bodys = @lines[($headerLength+1)..($bodyEndId-1)];
+  foreach(@bodys){
+    print OF "$_\n";
+  }
+  close OF;
+  
+  open(OF, ">:utf8", "$outFile.citation");
+  my @citations = @lines[$bodyEndId..($numLines-1)];
+  foreach(@citations){
+    print OF "$_\n";
+  }
+  close OF;
+}
+#my $header = join(/\n/, @headers);
+#print STDERR 
+
+open(OF, ">:utf8", "$outFile.info");
+print OF "header\t".$headerLength."\n";
+print OF "body\t".$bodyLength."\n";
+print OF "reference\t".$citationLength."\n";
+close OF;
+
+# sanity check
+my $totalLength = $headerLength + $bodyLength + $citationLength;
+if($numLines != $totalLength){
+  print STDOUT "Die in getStructureInfo(): different num lines $numLines != $totalLength\n"; # to display in Web
+  die "Die in getStructureInfo(): different num lines $numLines != $totalLength\n";
+}
+
+sub untaintPath {
+  my ($path) = @_;
+
+  if ( $path =~ /^([-_\/\w\.\d: ]+)$/ ) { #\p{C}\p{P}
+    $path = $1;
+  } else {
+    die "Bad path $path\n";
+  }
+
+  return $path;
+}
+
+sub untaint {
+  my ($s) = @_;
+  if ($s =~ /^([\w \-\@\(\),\.\/<>]+)$/) { #\p{C}\p{P}
+    $s = $1;               # $data now untainted
+  } else {
+    die "Bad data in $s";  # log this somewhere
+  }
+  return $s;
+}
+
+sub execute {
+  my ($cmd) = @_;
+  print STDERR "Executing: $cmd\n";
+  $cmd = untaint($cmd);
+  system($cmd);
+}
+
+sub newTmpFile {
+  my $tmpFile = `date '+%Y%m%d-%H%M%S-$$'`;
+  chomp($tmpFile);
+  return $tmpFile;
+}