biblicit 1.0

data/perl/ParsCit/lib/ParsCit/CitationContext.pm

@@ -0,0 +1,132 @@
+package ParsCit::CitationContext;
+#
+# Utility functions for finding citation contexts after citation
+# parsing.  The idea is to build regular expressions that could match
+# citation references in the body text and scan the body text for
+# occurrences.  The size of the context returned can be configured
+# in ParsCit::Config::contextRadius.
+#
+# Isaac Councill, 7/20/07
+#
+use ParsCit::Config;
+use utf8;
+
+my $contextRadius = $ParsCit::Config::contextRadius;
+my $maxContexts = $ParsCit::Config::maxContexts;
+my $premark = $ParsCit::Config::context_premark;
+my $postmark = $ParsCit::Config::context_postmark;
+
+##
+# Build a list of potential regular expressions based on the supplied
+# citation marker and applies the expressions to the specified body text.
+# Returns a reference to a list of strings that match the expressions,
+# expanded to radius $contextRadius.
+##
+sub getCitationContext {
+    my ($rBodyText, $marker) = @_;
+
+    my ($prioritize, @markers) = guessPossibleMarkers($marker);
+    my @matches = ();
+
+    my $contextsFound = 0;
+    foreach my $mark (@markers) {
+	while (($contextsFound < $maxContexts) &&
+	       $$rBodyText =~ m/(.{$contextRadius}$mark.{$contextRadius})/gs) {
+	    my $match = $1;
+	    $match =~ s/^(.*?)($mark)(.*)$/$1$premark$2$postmark$+/;
+	    push @matches, $match;
+	    $contextsFound++;
+	}
+	if (($prioritize > 0) && ($#matches >= 0)) {
+	    last;
+	}
+    }
+    return \@matches;
+
+}  # getCitationContext
+
+
+##
+# Builds a list of regular expressions based on the supplied
+# citation marker that may indicate a citation reference in
+# body text.  The first value returned is a parameter indicating
+# whether the list should be prioritized (i.e., if matches to
+# one element are found, don't try to match subsquent expressions),
+# with 0 indicating no prioritization and 1 indicating otherwise.
+##
+sub guessPossibleMarkers {
+    my $marker = shift;
+    if ($marker =~ m/^([\[\(])([\p{IsUpper}\p{IsLower}\-\d]+)([\]\)])/) {
+	my $open = makeSafe($1);
+	my $mark = makeSafe($2);
+	my $close = makeSafe($3);
+	my $refIndicator = "$open([\p{IsUpper}\p{IsLower}\\-\\d]+[;,] *)*$mark([;,] *[\p{IsUpper}\p{IsLower}\\-\\d]+)*$close";
+	return (0, $refIndicator);
+    }
+    if ($marker =~ m/^(\d+)\.?/) {
+	my $square = "\\[(\\d+[,;] *)*$1([;,] *\\d+)*\\]";
+	my $paren = "\\((\\d+[,;] *)*$1([,;] *\\d+)*\\)";
+	return (1, $square, $paren);
+    }
+    if ($marker =~ m/^([\p{IsUpper}\p{IsLower}\-\.\']+(, )*)+\d{4}/) {
+	my @tokens = split ", ", $marker;
+	my $year = $tokens[$#tokens];
+	my @names = @tokens[0..$#tokens-1];
+	my @possibleMarkers = ();
+	if ($#names == 0) {
+	    push @possibleMarkers, $names[0].",? $year";
+	}
+	if ($#names == 1) {
+	    push @possibleMarkers, $names[0]." and ".$names[1].",? $year";
+	    push @possibleMarkers, $names[0]." & ".$names[1].",? $year";
+	}
+	if ($#names > 1) {
+	    map { $_ = makeSafe($_) } $names;
+	    map { $_ = $_."," } @names;
+	    my $lastAuth1 = "and ".$names[$#names];
+	    my $lastAuth2 = "& ".$names[$#names];
+	    push @possibleMarkers,
+	      join " ", @names[0..$#names-1], $lastAuth1, $year;
+	    push @possibleMarkers,
+	      join " ", @names[0..$#names-1], $lastAuth2, $year;
+
+	    push @possibleMarkers, $names[0]."? et al\\\.?,? $year";
+	}
+	for (my $i=0; $i<=$#possibleMarkers; $i++) {
+	    my $safeMarker = $possibleMarkers[$i];
+	    $safeMarker =~ s/\-/\\\-/g;
+	    $possibleMarkers[$i] = $safeMarker;
+	}
+	return (0, @possibleMarkers);
+    }
+    return makeSafe($marker);
+
+}  # guessPossibleMarkers
+
+
+##
+# Prepare strings for safe inclusion within a regular expression,
+# escaping control characters.
+##
+sub makeSafe {
+    my $marker = shift;
+    $marker =~ s/\\/\\\\/g;
+    $marker =~ s/\-/\\\-/g;
+    $marker =~ s/\[/\\\[/g;
+    $marker =~ s/\]/\\\]/g;
+    $marker =~ s/\(/\\\(/g;
+    $marker =~ s/\)/\\\)/g;
+    $marker =~ s/\'/\\\'/g;
+    $marker =~ s/\"/\\\"/g;
+    $marker =~ s/\+//g;
+    $marker =~ s/\?//g;
+    $marker =~ s/\*//g;
+    $marker =~ s/\^//g;
+    $marker =~ s/\$//g;
+    $marker =~ s/\./\\\./g;
+    return $marker;
+
+} # makeSafe
+
+
+1;

data/perl/ParsCit/lib/ParsCit/Config.pm

@@ -0,0 +1,46 @@
+package ParsCit::Config;
+
+
+## Global
+
+$algorithmName = "ParsCit";
+$algorithmVersion = "1.0";
+
+
+## Repository Mappings
+
+%repositories = ('rep1' => '/repositories/rep1',
+                 'example1' => '/',
+		 'example2' => '/home',
+		 );
+
+
+## WS settings
+
+$serverURL = '130.203.133.46';
+$serverPort = 10555;
+$URI = 'http://citeseerx.org/algorithms/parscit/wsdl';
+
+
+## Tr2crfpp
+## Paths relative to ParsCit root dir ($FindBin::Bin/..)
+
+$tmpDir = "tmp";
+$dictFile = "resources/parsCitDict.txt";
+$modelFile = "resources/parsCit.model";
+
+$crf_test = "crf_test"; # assumes crfpp is installed in the system path
+
+## Citation Context
+
+$contextRadius = 200;
+$maxContexts = 5;
+$context_premark = "=-=";
+$context_postmark = "-=-";
+
+## Write citation and body file components
+
+$bWriteSplit = 1;
+
+
+1;

data/perl/ParsCit/lib/ParsCit/Controller.pm

@@ -0,0 +1,306 @@
+package ParsCit::Controller;
+#
+# This package is used to pull together various citation
+# processing modules in the ParsCit distribution, serving
+# as a script for handling the entire citation processing
+# control flow.  The extractCitations subroutine should be
+# the only needed API element if XML output is desired;
+# however, the extractCitationsImpl subroutine can be used
+# to get direct access to the list of citation objects.
+#
+# Isaac Councill, 07/23/07
+#
+use strict;
+use ParsCit::PreProcess;
+use ParsCit::PostProcess;
+use ParsCit::Tr2crfpp;
+use ParsCit::CitationContext;
+use ParsCit::Config;
+use CSXUtil::SafeText qw(cleanXML);
+
+
+##
+# Main API method for generating an XML document including all
+# citation data.  Returns a reference XML document and a
+# reference to the article body text.
+##
+sub extractCitations {
+    my ($textFile) = @_;
+    my ($status, $msg, $rCitations, $rBodyText)
+	= extractCitationsImpl($textFile);
+    if ($status > 0) {
+	return buildXMLResponse($rCitations);
+    } else {
+	my $error = "Error: $msg";
+	return \$error;
+    }
+
+} # extractCitations
+
+
+##
+# Main script for actually walking through the steps of
+# citation processing.  Returns a status code (0 for failure),
+# an error message (may be blank if no error), a reference to
+# an array of citation objects and a reference to the body
+# text of the article being processed.
+##
+sub extractCitationsImpl {
+    my ($textFile, $bWriteSplit) = @_;
+
+    if (!defined $bWriteSplit) {
+	$bWriteSplit = $ParsCit::Config::bWriteSplit;
+    }
+
+    my ($status, $msg) = (1, "");
+
+    if (!open (IN, "<:utf8", "$textFile")) {
+	return (-1, "Could not open text file $textFile: $!");
+    }
+
+    my $text;
+    {
+	local $/ = undef;
+	$text = <IN>;
+    }
+    close IN;
+
+    my ($rCiteText, $rNormBodyText, $rBodyText) =
+	ParsCit::PreProcess::findCitationText(\$text);
+    my ($citeFile, $bodyFile) = ("", "");
+    if ($bWriteSplit > 0) {
+	($citeFile, $bodyFile) =
+	    writeSplit($textFile, $rCiteText, $rBodyText);
+    }
+
+    my $rRawCitations = ParsCit::PreProcess::segmentCitations($rCiteText);
+    my @citations = ();
+    my @validCitations = ();
+
+    my $normalizedCiteText;
+    foreach my $citation (@{$rRawCitations}) {
+	# Tr2cfpp needs an enclosing tag for initial class seed.
+	my $citeString = $citation->getString();
+	if (defined $citeString && $citeString !~ m/^\s*$/) {
+	    $normalizedCiteText .=
+		"<title> ".$citation->getString() . " </title>\n";
+	    push @citations, $citation;
+	}
+    }
+
+    if ($#citations < 0) {
+	# Stop - nothing left to do.
+	return ($status, $msg, \@validCitations, $rNormBodyText);
+    }
+
+    my $tmpFile = ParsCit::Tr2crfpp::prepData(\$normalizedCiteText, $textFile);
+    my $outFile = $tmpFile."_dec";
+
+    if (ParsCit::Tr2crfpp::decode($tmpFile, $outFile)) {
+	my ($rRawXML, $rCiteInfo, $tstatus, $tmsg) =
+	    ParsCit::PostProcess::readAndNormalize($outFile);
+	if ($tstatus <= 0) {
+	    return ($tstatus, $msg, undef, undef);
+	}
+	my @citeInfo = @{$rCiteInfo};
+	if ($#citations == $#citeInfo) {
+	    for (my $i=0; $i<=$#citations; $i++) {
+		my $citation = $citations[$i];
+		my %citeInfo = %{$citeInfo[$i]};
+		foreach my $key (keys %citeInfo) {
+		    $citation->loadDataItem($key, $citeInfo{$key});
+		}
+		my $marker = $citation->getMarker();
+		if (!defined $marker) {
+		    $marker = $citation->buildAuthYearMarker();
+		    $citation->setMarker($marker);
+		}
+		my $rContexts =
+		    ParsCit::CitationContext::getCitationContext($rNormBodyText,
+								 $marker);
+		foreach my $context (@{$rContexts}) {
+		    $citation->addContext($context);
+		}
+		push @validCitations, $citation;
+	    }
+	} else {
+	    $status = -1;
+	    $msg = "Mismatch between expected citations and cite info";
+	}
+    }
+
+    unlink($tmpFile);
+    unlink($outFile);
+
+    return ($status, $msg, \@validCitations,
+	    $rBodyText, $citeFile, $bodyFile);
+
+} # extractCitationsImpl
+
+
+##
+# Quickly hacked copy of extractCitationsImpl used for testing a
+# parscit with pre-segmented reference strings.  This method
+# will be collapsed with the main Impl method in future versions.
+##
+sub extractCitationsAlreadySegmented {
+    my ($textFile) = @_;
+
+    my ($status, $msg) = (1, "");
+
+    if (!open(IN, "<:utf8", $textFile)) {
+	return (-1, "Could not open file $textFile: $!");
+    }
+
+    my @rawCitations = ();
+    my $currentCitation;
+    while(<IN>) {
+	chomp();
+	if (m/^\s*$/ && defined $currentCitation) {
+	    my $cite = new ParsCit::Citation();
+	    $cite->setString($currentCitation);
+	    push @rawCitations, $cite;
+	    $currentCitation = undef;
+	    next;
+	}
+	if (!defined $currentCitation) {
+	    $currentCitation = $_;
+	} else {
+	    $currentCitation .= " ".$_;
+	}
+    }
+    close IN;
+    if (defined $currentCitation) {
+	my $cite = new ParsCit::Citation();
+	push @rawCitations, $cite;
+    }
+
+    my @citations;
+    my @validCitations = ();
+    my $normalizedCiteText;
+    foreach my $citation (@rawCitations) {
+	# Tr2cfpp needs an enclosing tag for initial class seed.
+	my $citeString = $citation->getString();
+	if (defined $citeString && $citeString !~ m/^\s*$/) {
+	    $normalizedCiteText .=
+		"<title> ".$citation->getString() . " </title>\n";
+	    push @citations, $citation;
+	}
+    }
+
+    if ($#citations < 0) {
+	# Stop - nothing left to do.
+	return ($status, $msg, \@validCitations);
+    }
+
+    my $tmpFile = ParsCit::Tr2crfpp::prepData(\$normalizedCiteText, $textFile);
+    my $outFile = $tmpFile."_dec";
+
+    if (ParsCit::Tr2crfpp::decode($tmpFile, $outFile)) {
+	my ($rRawXML, $rCiteInfo, $tstatus, $tmsg) =
+	    ParsCit::PostProcess::readAndNormalize($outFile);
+	if ($tstatus <= 0) {
+	    return ($tstatus, $msg, undef, undef);
+	}
+	my @citeInfo = @{$rCiteInfo};
+	if ($#citations == $#citeInfo) {
+	    for (my $i=0; $i<=$#citations; $i++) {
+		my $citation = $citations[$i];
+		my %citeInfo = %{$citeInfo[$i]};
+		foreach my $key (keys %citeInfo) {
+		    $citation->loadDataItem($key, $citeInfo{$key});
+		}
+#		unless ($citation->isValid()>0) {
+#		    next;
+#		}
+		my $marker = $citation->getMarker();
+		if (!defined $marker) {
+		    $marker = $citation->buildAuthYearMarker();
+		    $citation->setMarker($marker);
+		}
+		push @validCitations, $citation;
+	    }
+	} else {
+	    $status = -1;
+	    $msg = "Mismatch between expected citations and cite info";
+	}
+    }
+
+    unlink($tmpFile);
+    unlink($outFile);
+
+    return buildXMLResponse(\@validCitations);
+
+} # extractCitationsAlreadySegmented
+
+
+sub buildXMLResponse {
+    my ($rCitations) = @_;
+    my $l_algName = $ParsCit::Config::algorithmName;
+    my $l_algVersion = $ParsCit::Config::algorithmVersion;
+    cleanXML(\$l_algName);
+    cleanXML(\$l_algVersion);
+
+    my $xml = "<algorithm name=\"$l_algName\" ".
+	"version=\"$l_algVersion\">\n";
+    $xml .= "<citationList>\n";
+
+    foreach my $citation (@$rCitations) {
+	$xml .= $citation->toXML();
+    }
+
+    $xml .= "</citationList>\n";
+    $xml .= "</algorithm>\n";
+
+    return \$xml;
+
+} # buildXMLResponse
+
+
+##
+# Writes separated citation and body text to disk.
+##
+sub writeSplit {
+    my ($textFile, $rCiteText, $rBodyText) = @_;
+
+    my $citeFile = changeExtension($textFile, "cite");
+    my $bodyFile = changeExtension($textFile, "body");
+
+    if (open(OUT, ">$citeFile")) {
+	binmode OUT, ":utf8";
+	print OUT $$rCiteText;
+	close OUT;
+    } else {
+	print STDERR "Could not open .cite file for writing: $!\n";
+    }
+
+    if (open(OUT, ">$bodyFile")) {
+	binmode OUT, ":utf8";
+	print OUT $$rBodyText;
+	close OUT;
+    } else {
+	print STDERR "Could not open .body file for writing: $!\n";
+    }
+
+    return ($citeFile, $bodyFile);
+
+} # writeSplit
+
+
+##
+# Utility to replace the extension of a file name.  Parameter
+# $ext should be just the alphanumeric extension, without the
+# preceeding ".".  If no extension is present on the file name,
+# $ext will be appended.
+##
+sub changeExtension {
+    my ($fn, $ext) = @_;
+    unless ($fn =~ s/^(.*)\..*$/$1\.$ext/) {
+	$fn .= ".$ext";
+    }
+    return $fn;
+
+} # changeExtension
+
+
+1;