biblicit 1.0 → 2.0.3

data/parscit/lib/ParsHed/Config.pm

@@ -0,0 +1,49 @@
+package ParsHed::Config;
+
+$algorithmName		= "ParsHed";
+$algorithmVersion	= "110505";
+
+## Tr2crfpp
+## Paths relative to ParsCit root dir ($FindBin::Bin/..)
+$tmpDir			= "tmp";
+$dictFile		= "resources/parsCitDict.txt";
+$keywordFile	= "resources/parsHed/keywords";
+$bigramFile		= "resources/parsHed/bigram";
+
+$crf_test   = "$ENV{'CRFPP_HOME'}/bin/crf_test";
+
+$modelFile		= "resources/parsHed/parsHed.model";
+#$oldModelFile	= "resources/parsHed/archive/parsHed.090316.model";
+$oldModelFile = "unsupported for space reasons";
+
+# Flags for different types of CRF features for line-level training
+$isFullFormToken	= 1;
+$isFirstToken  		= 1;
+$isLastToken  		= 1;
+$isSecondToken  	= 1;
+$isSecondLastToken	= 1;
+$isBack1 			= 1;
+$isForw1 			= 1;
+$isKeyword  		= 1;
+$isBigram 			= 0;
+
+# List of tags trained in parsHed
+# Those with value 0 do not have frequent keyword features
+%hash = (	'abstract'    => 1,
+	 		'address'     => 1 ,
+	 		'affiliation' => 1,
+	 		'author'      => 0, #
+	 		'date'        => 0, #
+	 		'degree'      => 1,
+	 		'email'       => 0, #
+	 		'intro'       => 1,
+	 		'keyword'     => 1,
+	 		'note'        => 1,
+	 		'page'        => 1,
+	 		'phone'       => 0, #
+	 		'pubnum'      => 1,
+	 		'title'       => 1,
+	 		'web'         => 0	);
+$tags = \%hash;
+
+1;

data/parscit/lib/ParsHed/Controller.pm

@@ -0,0 +1,143 @@
+package ParsHed::Controller;
+#
+# This package is used to pull together various citation
+# processing modules in the ParsCit distribution, serving
+# as a script for handling the entire citation processing
+# control flow.  
+# The extractHeader subroutine should be
+# the only needed API element to return XML output.
+#
+# Luong Minh Thang 25 May, 09. Adopted from Parscit Controller Isaac Councill, 07/23/07
+# 
+
+require 'dumpvar.pl';
+use strict;
+use ParsCit::PreProcess;
+use ParsHed::PostProcess;
+use ParsHed::Tr2crfpp;
+use ParsHed::Tr2crfpp_token; # for token-level model
+use ParsCit::CitationContext;
+use ParsHed::Config;
+use CSXUtil::SafeText qw(cleanXML);
+
+###
+# Main API method for generating an XML document including all
+# header data.  Returns a reference XML document.
+###
+sub extractHeader 
+{
+    my ($text_file, $is_token_level) = @_; 
+
+	# Thang 10/11/09: add confidence score option - 1: enable, 0: disable
+    my $conf_level = 1; 
+    my ($status, $msg, $xml) = extractHeaderImpl($text_file, $is_token_level, $conf_level);
+
+    if ($status > 0) 
+	{
+		return \$xml;
+    } 
+	else 
+	{
+		my $error = "Error: $msg";
+		return \$error;
+    }
+}
+
+
+###
+# Main script for actually walking through the steps of
+# header processing.  Returns a status code (0 for failure),
+# an error message (may be blank if no error), a reference 
+# to an XML document.
+#
+# $is_token_level: flag to enable previous token-level model 
+# (for performance comparison).
+# Todo: catch errors and return $status < 0
+###
+sub extractHeaderImpl 
+{
+	# Thang 10/11/09: $confLevel to add confidence info
+	my ($text_file, $is_token_level, $conf_level) = @_;
+
+  	my ($status, $msg) = (1, "");
+
+	# Open input text file
+  	if (!open (IN, "<:utf8", "$text_file")) { return (-1, "Could not open text file $text_file: $!"); }
+    
+  	my $buf = "";
+	while (<IN>) 
+	{
+    	chomp;
+
+		# Remove ^M character at the end of the file if any
+		s/\cM$//; 
+	
+		# Skip comments
+    	if (/^\#/) 
+		{ 
+			next; 
+		}
+		# Skip blank lines
+		elsif (/^\s+$/) 
+		{ 
+			next; 
+		}
+    	else 
+		{
+			# sample RE for header stop.
+	      	if (/INTRODUCTION/i) { last; }
+      
+      		if($is_token_level)
+			{
+				$buf .= "$_";
+				$buf .= " +L+ ";
+      		} 
+			else 
+			{
+				$buf .= "$_\n";
+      		}
+    	}
+  	}
+  	close IN;
+
+	# For compatible reason
+  	if($is_token_level) { $buf = "<title> $buf </title>\n"; }
+
+  	# Run tr2crfpp to prepare feature files
+	my $tmpfile;
+  	if($is_token_level)
+	{
+    	$tmpfile = ParsHed::Tr2crfpp_token::prepData(\$buf, $text_file);
+  	} 
+	else 
+	{
+    	$tmpfile = ParsHed::Tr2crfpp::prepData(\$buf, $text_file);
+  	}
+
+  	my $xml = undef;
+  	# run crf_test, output2xml
+	my $outfile = $tmpfile."_dec";
+
+  	if($is_token_level)
+	{
+    	if (ParsHed::Tr2crfpp_token::decode($tmpfile, $outfile)) 
+		{
+			$xml = ParsHed::PostProcess::wrapHeaderXml($outfile, 0, $is_token_level);
+    	}
+  	} 
+	else 
+	{
+		# Thang 10/11/09: $confLevel to add confidence info
+    	if (ParsHed::Tr2crfpp::decode($tmpfile, $outfile, $conf_level)) 
+		{
+			# Thang 10/11/09: $confLevel to add confidence info
+      		$xml = ParsHed::PostProcess::wrapHeaderXml($outfile, $conf_level);  
+    	}
+  	}
+
+  	unlink($tmpfile);
+  	unlink($outfile);
+  	return ($status, $msg, $xml);
+}
+
+1;

data/parscit/lib/ParsHed/PostProcess.pm

@@ -0,0 +1,322 @@
+package ParsHed::PostProcess;
+
+###
+# Utilities for normalizing the output of CRF++ into standard
+# representations.
+#
+# Luong Minh Thang 25 May, 09. Adopted from Isaac Councill, 07/20/07
+###
+
+use utf8;
+use strict;
+
+use CSXUtil::SafeText qw(cleanXML);
+use ParsCit::Config;
+use ParsCit::PostProcess;
+
+###
+# Main method for processing header data. Specifically, it reads CRF
+# output, performs normalization to individual fields, and outputs to
+# XML
+###
+sub wrapHeaderXml 
+{
+	# Thang Nov 09: $confInfo to add confidence info
+	my ($infile, $conf_info, $is_token_level) = @_; 
+
+  	my $status		= 1;
+  	my $msg			= "";
+  	my $xml			= "";
+  	my $last_tag	= "";
+  	my $variant		= "";
+
+	# Thang Nov 09: rename $confidence -> $overallConfidence
+  	my $overall_confidence = "1.0"; 
+
+  	# Output XML file for display
+  	$xml .= "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
+
+	# Array of hash: each element of fields correspond to a pairs of (tag, content) 
+	# accessible through $fields[$i]->{"tag"} and $fields[$i]->{"content"}
+  	my @fields	= ();
+  	my $count	= 0;
+
+	my $cur_content		= "";
+	# For lines of the same label
+	my $cur_confidence	= 0; 
+
+	# Input file is the result output of crf++ test
+	open(IN, "<:utf8", $infile) or return (undef, undef, 0, "couldn't open infile: $!");
+
+  	while (<IN>) 
+	{	
+		# Confidence info
+    	if (/^\# ([\.\d]+)/) 
+		{
+      		$overall_confidence = $1;
+      		next;
+    	}
+		# Skip comments
+    	elsif (/^\#/) 
+		{ 
+			next; 
+		}                              
+
+		# End of a header, output (useful to handle multiple header classification
+    	if (/^\s*$/) 
+		{
+			# Add the last field
+      		addFieldInfo(\@fields, $last_tag, $cur_content, $cur_confidence, $count);      
+
+      		if ($variant eq "") 
+			{
+				# Generate XML output
+				my $output = generateOutput(\@fields, $conf_info);
+
+				my $l_alg_name = $ParsHed::Config::algorithmName;
+				my $l_alg_version = $ParsHed::Config::algorithmVersion;
+				$xml .= "<algorithm name=\"$l_alg_name\" version=\"$l_alg_version\">\n". "<variant no=\"0\" confidence=\"$overall_confidence\">\n". $output . "</variant>\n</algorithm>\n";
+      		}
+      
+      		@fields		= (); #reset
+      		$last_tag	= "";
+    	} 
+		# In a middle of a header
+		else 
+		{
+      		chop;
+      		my @tokens		= split (/\t/);
+
+      		my $token		= $tokens[0];
+			
+      		my $sys			= $tokens[-1];
+      		my $gold		= $tokens[-2];
+      		my $confidence	= 0;
+
+      		if (!defined $is_token_level)
+			{
+				# Train at line level, get the original line
+				my @tokens	= split(/\|\|\|/, $token);
+				$token		= join(" ", @tokens);
+				
+				###
+				# Thang Nov 09: process confidence output from crf++
+				###
+				if($conf_info)
+				{
+					# $sys contains probability info of the format "tag/prob"
+	  				if ($sys =~ /^(.+)\/([\d\.]+)$/)
+					{
+	    				$sys		= $1;
+	    				$confidence	+= $2;
+	  				} 
+					else 
+					{
+	    				die "Die in ParsHed::PostProcess::wrapHeaderXml : incorrect format \"tag/prob\" $sys\n";
+	  				}
+				}
+				###
+				# End Thang Nov 09: process confidence output from crf++
+				###
+      		}
+	
+			# Start a new tag, not an initial value, output
+      		if ($sys ne $last_tag && $last_tag ne "") 
+			{ 
+				addFieldInfo(\@fields, $last_tag, $cur_content, $cur_confidence, $count);
+	  
+				# Reset the value
+				$cur_content	= ""; 
+				$cur_confidence	= 0;
+				$count			= 0;
+      		}
+
+	      	if (defined $is_token_level && $token eq "+L+") { next; }
+
+    	 	$cur_content	.= $token . " ";
+      		$cur_confidence	+= $confidence;
+	      	$count++;
+	
+			# Update last tag
+      		$last_tag = $sys;
+		}
+  	}
+
+  	close (IN);
+  	return $xml;
+}
+
+###
+# Thang Mar 10: refactor this part of code into a method, to add per-field info
+###
+sub addFieldInfo 
+{
+  	my ($fields, $last_tag, $cur_content, $cur_confidence, $count) = @_;
+
+  	my %tmp_hash		 = ();
+  	$tmp_hash{"tag"}	 = $last_tag;
+  	$tmp_hash{"content"} = $cur_content;
+  
+  	### Thang Nov 09: compute confidence score
+  	if ($count > 0) { $tmp_hash{"confidence"} = $cur_confidence / $count; }
+
+	# Save
+  	push(@{ $fields }, \%tmp_hash);
+}
+
+###
+# Thang Mar 10: refactor this part of code into a method, wrap all field infos into XML form
+###
+sub generateOutput 
+{
+  	my ($fields, $conf_info) = @_;
+
+  	my $output = "";
+  
+  	foreach (@{ $fields }) 
+	{
+    	my $tag		= $_->{"tag"};
+    	my $content	= $_->{"content"};
+
+		###
+    	# Thang Nov 09: modify to output confidence score
+		###
+    	my $conf_str = "";
+    	if ($conf_info) { $conf_str = " confidence=\"".$_->{"confidence"}."\""; }
+
+		# Blank content
+    	if ($content =~ /^\s*$/) { next; };
+
+    	($tag, $content) = normalizeHeaderField($tag, $content);
+
+		# Handle multiple authors in a line
+    	if ($tag eq "authors")
+		{
+      		foreach my $author (@{ $content })
+			{
+				cleanXML(\$author);
+				$output .= "<author$conf_str>$author</author>\n";
+      		}
+    	}
+		# Handle multiple emails at a time
+		elsif ($tag eq "emails")
+		{
+      		foreach my $email (@{ $content })
+			{
+				$output .= "<email$conf_str>$email</email>\n";
+      		}
+    	} 
+		else 
+		{
+      		$output .= "<$tag$conf_str>$content</$tag>\n";
+    	}
+		###
+    	# End Thang Nov 09: modify to output confidence score
+		###
+  	} # end for each fields
+  
+  	return $output;    
+}
+
+###
+# Header normalization subroutine.  Reads in a tag and its content, 
+# perform normalization based on that tag.
+###
+sub normalizeHeaderField 
+{
+  	my ($tag, $content) = @_;;
+
+	# Remove keyword at the beginning
+	$content =~ s/^\W*$tag\W+//i;
+	# Strip leading spaces
+  	$content =~ s/^\s+//g;
+	# Remove trailing spaces
+  	$content =~ s/\s+$//g;		      
+	# Unhyphenate
+  	$content =~ s/\- ([a-z])/$1/g;
+
+  	# Normalize author and break into multiple authors (if any)
+  	if ($tag eq "author") 
+	{
+    	$tag	 = "authors";
+    	$content =~ s/\d//g; # remove numbers
+    	$content = ParsCit::PostProcess::NormalizeAuthorNames($content);
+  	} 
+	elsif ($tag eq "email") 
+	{
+		# Multiple emails of the form {kanmy,luongmin}@nus.edu.sg
+    	if($content =~ /^\{(.+)\}(.+)$/)
+		{
+      		my $begin	  = $1;
+      		my $end		  = $2;
+      		my $separator = ",";
+
+	      	# Find possible separator of emails, beside ","
+    	  	my @separators = ($begin =~ /\s+(\S)\s+/g); 
+      		if (scalar(@separators) > 1)
+			{	
+				my $cand = $separators[0];
+				my $flag = 1;
+				foreach (@separators) 
+				{
+					# Should be the same
+	  				if($_ ne $cand)
+					{
+		    			$flag = 0;
+	    				last;
+	  				}
+				}
+
+				# All separator are the same, and the number of separator > 1, update separator
+				if($flag == 1) 
+				{
+	  				$separator = $cand;
+				}	
+    	  	}
+
+	      	my @tokens = split(/$separator/, $begin);
+
+			# Remove all white spaces
+      		$end =~ s/\s+//g; 
+
+			# There are actually multiple emails
+    	  	if (scalar(@tokens) > 1) 
+			{
+				my @emails = ();
+
+				foreach my $token (@tokens)
+				{
+					# Remove all white spaces
+	  				$token =~ s/\s+//g; 
+	  				push (@emails, "$token$end");
+				}
+
+				$tag	 = "emails";
+				$content = \@emails;
+	      	}
+    	} 
+		# Only one email
+		else 
+		{
+			# Remove all white spaces
+      		$content =~ s/\s+//g; 
+    	}
+  	} 
+	else 
+	{
+		# Escape XML characters
+    	cleanXML(\$content);
+
+		###
+		# Huydhn: don't understand why need to remove punctuation here
+		# just for the sake of appearance
+		#
+		# 17 jan 2011
+		###
+    	$content = ParsCit::PostProcess::StripPunctuation($content);
+	}
+
+  	return ($tag, $content);
+}
+
+1;