biblicit 1.0

data/perl/DocFilter/lib/DocFilter/Config.pm

@@ -0,0 +1,35 @@
+#
+# Copyright 2007 Penn State University
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+package DocFilter::Config;
+
+
+## Global
+
+$algorithmName = "BasicDocFilter";
+$algorithmVersion = "1.0";
+
+
+## Repository Mappings
+
+%repositories = ('example1' => '/',
+		 'example2' => '/home',
+		 );
+
+
+## WS Settings
+
+$serverURL = '127.0.0.1';
+$serverPort = 10666;
+$URI = 'http://citeseerx.org/algorithms/docfilter/wsdl';
+
+1;

data/perl/DocFilter/lib/DocFilter/Filter.pm

@@ -0,0 +1,51 @@
+#
+# Copyright 2007 Penn State University
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+package DocFilter::Filter;
+##
+##  Isaac Councill, 7/31/07
+##
+use strict;
+
+
+sub filter {
+    my ($filePath) = @_;
+
+    if (!open (IN, "<$filePath")) {
+	return (0, 0, "Could not open file $filePath: $!");
+    }
+    my $text;
+    {
+	local $/ = undef;
+	$text = <IN>;
+    }
+
+    if (hasReferences(\$text) <= 0) {
+	return (1, 0, "No reference section is present");
+    }
+    return (1, 1, "All filters passed");
+
+} # filter
+
+
+sub hasReferences {
+    my $rText = shift;
+    if ($$rText =~ /\b(REFERENCES?|References?|BIBLIOGRAPHY|Bibliography|REFERENCES AND NOTES|References and Notes)\:?\s*\n/sg) {
+	return 1;
+    } else {
+	return 0;
+    }
+
+} # hasReferences
+
+
+1;

data/perl/FileConversionService/README.TXT

@@ -0,0 +1,11 @@
+FileConverter README
+Isaac Councill
+
+Check the FileConverter::Config module to make sure the settings
+are correct.  You will probably need to obtain the PDFBox jar file
+and reference it in Config.  It's better to use TET, but TET is
+expensive.
+
+If using PDFBox, make sure java is on your path.
+
+I know this is sparse, more to come...

data/perl/FileConversionService/lib/CSXUtil/SafeText.pm

@@ -0,0 +1,140 @@
+#
+# Copyright 2007 Penn State University
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+package CSXUtil::SafeText;
+##
+##  Methods for stripping bad (XML unsafe) characters
+##  from strings and performing basic HTML entity
+##  translations.  Also contains a utility (stripArtifacts)
+##  for getting rid of crazy control characters and
+##  other things that probably aren't proper text.
+##
+##  Isaac Councill, 12/06/06
+##
+#######################################################
+##
+use strict;
+use utf8;
+require Exporter;
+
+our ($VERSION, @ISA, @EXPORT, @EXPORT_OK, %EXPORT_TAGS);
+
+$VERSION = 1.00;
+
+@ISA = qw(Exporter);
+@EXPORT_OK = qw(@badChars %htmlSpecialChars
+		%htmlCharEntities &stripBadChars
+		&encodeHTMLSpecialChars
+		&decodeHTMLSpecialChars
+		&cleanXML &cleanAll &stripArtifacts);
+
+
+##
+#######################################################
+##
+##  Sharable encoding data.
+##
+
+##  Hex codes for characters that should never be put into
+##  XML - or else parsers will barf.
+our @badChars = qw(\x00 \x01 \x02 \x03 \x04 \x05 \x06 \x07
+		   \x08 \x0B \x0C \x0E \x0F \x10 \x11 \x12
+		   \x13 \x14 \x15 \x16 \x17 \x18 \x19 \x1A
+		   \x1B \x1C \x1D \x1E \x1F \x7F);
+
+##  Subset of HTML characters that could be problematic
+##  for XML.  This is not a complete list of HTML
+##  special characters, but more mappings can be added
+##  as needed.
+our %htmlSpecialCharEncodings = ("&" => "&",
+				 ">" => ">",
+				 "<" => "&lt;",
+				 "\"" => "&quot;"
+				 );
+
+##  The reverse map.
+our %htmlSpecialCharDecodings;
+foreach my $key (keys %htmlSpecialCharEncodings) {
+    my $val = $htmlSpecialCharEncodings{$key};
+    $htmlSpecialCharDecodings{$val} = $key;
+}
+
+
+##
+#######################################################
+##
+##  Subroutines
+##
+
+##  Delete all occurences of bad characters in text,
+##  returns a new string that is clean.
+sub stripBadChars {
+    my $rtext = shift;
+    foreach my $char (@badChars) {
+	$$rtext =~ s/$char//g;
+    }
+}
+
+
+##  Encodes special characters into HTML equivalents
+##  and returns the encoded string.
+sub encodeHTMLSpecialChars {
+    my $rtext = shift;
+    foreach my $char (keys %htmlSpecialCharEncodings) {
+	my $code = $htmlSpecialCharEncodings{$char};
+	$$rtext =~ s/$char/$code/g;
+    }
+}
+
+
+##  Decodes a HTML entities in the supplied string
+##  into non-HTML character equivalents and returns
+##  the decoded string.
+sub decodeHTMLSpecialChars {
+    my $rtext = shift;
+    foreach my $code (keys %htmlSpecialCharDecodings) {
+	my $char = $htmlSpecialCharDecodings{$code};
+	$$rtext =~ s/$code/$char/g;
+    }
+}
+
+
+##  Strip out any characters that don't look like they
+##  belong in a proper, readable text string.
+##
+sub stripArtifacts {
+    my $rtext = shift;
+    $$rtext =~ s/[^\p{IsAlnum}\p{IsPunct}\p{IsSpace}\p{IsS}]//g;
+}
+
+
+##  Convenience routine for executing both XML safety
+##  routines in a single call.
+##
+sub cleanXML {
+    my $rtext = shift;
+    stripBadChars($rtext);
+    encodeHTMLSpecialChars($rtext);
+}
+
+
+##  Clean for XML and also strip out strange characters.
+##
+sub cleanAll {
+    my $rtext = shift;
+    stripBadChars($rtext);
+    stripArtifacts($rtext);
+    encodeHTMLSpecialChars($rtext);
+}
+
+
+1;

data/perl/FileConversionService/lib/FileConverter/CheckSum.pm

@@ -0,0 +1,77 @@
+#
+# Copyright 2007 Penn State University
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+package FileConverter::CheckSum;
+#
+# Container for checksum info and utilities for individual files.
+#
+# Isaac Councill
+#
+use strict;
+use Digest::SHA1;
+use FileConverter::Utils;
+
+
+sub new {
+    my ($class) = @_;
+    my $self = {
+	'_fileType' => undef,
+	'_sha1'     => undef,
+    };
+    bless $self, $class;
+    return $self;
+
+}  # new
+
+
+sub getFileType {
+    my $self = shift;
+    return $self->{'_fileType'};
+}
+
+
+sub setFileType {
+    my ($self, $fileType) = @_;
+    $self->{'_fileType'} = $fileType;
+}
+
+
+sub getSHA1 {
+    my $self = shift;
+    return $self->{'_sha1'};
+}
+
+
+sub setSHA1 {
+    my ($self, $sha1) = @_;
+    $self->{'_sha1'} = $sha1;
+}
+
+
+sub digest {
+    my ($self, $filePath) = @_;
+
+    open(FILE, "<$filePath") or die ("Could not open for reading: $filePath");
+    my $digester = Digest::SHA1->new;
+    $digester->addfile(*FILE);
+
+    my $ext = FileConverter::Utils::getExtension($filePath);
+    my $sha1 = $digester->hexdigest;
+    close FILE;
+
+    $self->setFileType($ext);
+    $self->setSHA1($sha1);
+
+}  # digest
+
+1;
+

data/perl/FileConversionService/lib/FileConverter/Compression.pm

@@ -0,0 +1,137 @@
+#
+# Copyright 2007 Penn State University
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+package FileConverter::Compression;
+#
+# Utilities for handling various compression formats.
+#
+# Isaac Councill, 09/06/07
+#
+use strict;
+use FileConverter::Config;
+use FileConverter::Utils;
+
+# Should all be lower case.
+my %supportedCompressionExt = ("gz" => 1,
+			       "zip" => 1,
+			       "z" => 1,
+			       );
+
+my $gunzip = $FileConverter::Config::gunzip;
+my $uncompress = $FileConverter::Config::uncompress;
+my $unzip = $FileConverter::Config::unzip;
+
+
+sub decompress {
+    my ($fn, $rTrace) = @_;
+    my $ext = FileConverter::Utils::getExtension($fn);
+    if ($ext =~ m/^gz$/i) {
+	return gunzip($fn, $rTrace);
+    }
+    if ($ext =~ m/^z$/i) {
+	return uncompress($fn, $rTrace);
+    }
+    if ($ext =~ m/^zip$/i) {
+	return unzip($fn, $rTrace);
+    }
+    return (0, "Unsupported compression extension: $ext");
+
+} # decompress
+
+
+sub canDecompress {
+    my ($fn) = @_;
+    my $ext = FileConverter::Utils::getExtension($fn);
+    if (defined $supportedCompressionExt{lc($ext)}) {
+	return 1;
+    } else {
+	return 0;
+    }
+
+} # canDecompress
+
+
+sub gunzip {
+    my ($fn, $rTrace) = @_;
+    my @commandArgs = ($gunzip, "-f", $fn);
+
+    system(@commandArgs);
+
+    if ($? == -1) {
+	return (0, "Failed to execute gunzip: $!");
+    } elsif ($? & 127) {
+	return (0, "gunzip died with signal ".($? & 127));
+    };
+    my $code = $?>>8;
+    if ($code == 1) {
+	return (0, "Error executing gunzip (code $code): $!");
+    }
+
+    push @$rTrace, "gunzip";
+
+    my $newFile = FileConverter::Utils::stripExtension($fn);
+    return (1, "", $newFile, $rTrace);
+
+} # gunzip
+
+
+sub uncompress {
+    my ($fn, $rTrace) = @_;
+    my @commandArgs = ($uncompress, "-f", $fn);
+
+    system(@commandArgs);
+
+    if ($? == -1) {
+	return (0, "Failed to execute uncompress: $!");
+    } elsif ($? & 127) {
+	return (0, "uncompress died with signal ".($? & 127));
+    };
+    my $code = $?>>8;
+    if ($code == 1) {
+	return (0, "Error executing uncompress (code $code): $!");
+    }
+
+    push @$rTrace, "uncompress";
+
+    my $newFile = FileConverter::Utils::stripExtension($fn);
+    return (1, "", $newFile, $rTrace);
+
+} # uncompress
+
+
+sub unzip {
+    my ($fn, $rTrace) = @_;
+
+    my $dir = FileConverter::Utils::getDirectory($fn);
+    my @commandArgs = ($unzip, "-qqo", $fn, "-d", $dir);
+
+    system(@commandArgs);
+
+    if ($? == -1) {
+	return (0, "Failed to execute unzip: $!");
+    } elsif ($? & 127) {
+	return (0, "unzip died with signal ".($? & 127));
+    };
+    my $code = $?>>8;
+    if ($code > 2) {
+	return (0, "Error executing unzip (code $code): $!");
+    }
+
+    push @$rTrace, "unzip";
+
+    my $newFile = FileConverter::Utils::stripExtension($fn);
+    return (1, "", $newFile, $rTrace);
+
+} # unzip
+
+
+1;

data/perl/FileConversionService/lib/FileConverter/Config.pm

@@ -0,0 +1,57 @@
+#
+# Copyright 2007 Penn State University
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+package FileConverter::Config;
+
+use FindBin;
+
+## Conversion utilities
+
+# valid options are TET or PDFBOX
+$PDFTOTEXT = "PDFBOX";
+
+# valid options are TEXT or PDF
+$PSConversion = "TEXT";
+
+$TETPath = "$FindBin::Bin/../converters/TET-2.2-Linux/bin/tet";
+
+$TETLicensePath =
+    "$FindBin::Bin/../converters/TET-2.2-Linux/licensekeys.txt";
+
+$PDFBoxLocation = "$FindBin::Bin/FileConversionService/converters/PDFBox/pdfbox-app-1.7.1.jar";
+
+$JODConverterPath = 
+    "$FindBin::Bin/../converters/jodconverter-2.2.0/jodconverter-cli-2.2.0.jar";
+
+$PrescriptPath = "/usr/local/bin/prescript";
+
+## Compression utilities
+
+$gunzip = "/usr/bin/gunzip";
+$uncompress = "/usr/bin/uncompress";
+$unzip = "/usr/bin/unzip";
+
+
+## Repository Mappings
+
+%repositories = ('example1' => '/',
+        		 'example2' => '/home',
+                );
+
+
+## WS settings
+
+$serverURL = '127.0.0.1';
+$serverPort = 10888;
+$URI = 'http://citeseerx.org/fileConversion/wsdl';
+
+1;

data/perl/FileConversionService/lib/FileConverter/Controller.pm

@@ -0,0 +1,191 @@
+#
+# Copyright 2007 Penn State University
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+package FileConverter::Controller;
+#
+# Main interface to FileConverter.  This is where all calls
+# should start, and where all conversion sequences should
+# be managed.
+#
+# Isaac Councill, 09/06/07
+#
+use strict;
+use FileConverter::Utils;
+use FileConverter::Compression;
+use FileConverter::TET;
+use FileConverter::PDFBox;
+use FileConverter::JODConverter;
+use FileConverter::PSConverter;
+use FileConverter::PSToText;
+use FileConverter::Prescript;
+use FileConverter::CheckSum;
+
+
+##
+# Convert the given file to text.  Decompression will occur
+# first, and then further conversion and/or text extraction
+# will proceed according to the file type.
+#
+# Supports zip, gz, .Z compression, but only supports the
+# PDF file type for now.
+##
+sub extractText {
+    my $fn = shift;
+
+    if (! -e $fn) {
+    	return (0, "File does not exist: $fn");
+    }
+
+    my ($status, $msg) = (1, "");
+    my @trace = ();
+    my @checkSums = ();
+    my ($tstatus, $tmsg, $tfn, $rTrace, $rCheckSums);
+
+    while(FileConverter::Compression::canDecompress($fn) > 0) {
+    	($tstatus, $tmsg, $tfn, $rTrace) =
+            FileConverter::Compression::decompress($fn, \@trace);
+        if ($tstatus <= 0) {
+            return ($tstatus, $tmsg);
+        }
+        $fn = $tfn;
+    }
+
+    my $extension = FileConverter::Utils::getExtension($fn);
+
+    if (!defined $extension) {
+        return (0, "File $fn has no extension");
+    }
+
+    if ($extension =~ m/^ps$/i) {
+        # convert poscript file according config.
+        if ($FileConverter::Config::PSConversion eq "TEXT") {
+            # go from ps to text directly
+	    _convert2pdf($fn, $extension, [], \@checkSums);
+            return ps2text($fn, \@trace, \@checkSums);
+        }
+        if ($FileConverter::Config::PSConversion eq "PDF") {
+            # convert to PDF first then to text
+            ($tstatus, $tmsg, $tfn, $rTrace, $rCheckSums) = 
+                _convert2pdf($fn, $extension, \@trace, \@checkSums);
+            if ($tstatus <= 0) {
+                return ($tstatus, $tmsg);
+            }
+            $fn = $tfn;
+            $extension = FileConverter::Utils::getExtension($fn);
+        }
+    }
+    elsif (($extension !~ m/^pdf$/i) && ($extension !~ m/^ps$/i)) {
+        # first, we need to convert the file to PDF.
+        ($tstatus, $tmsg, $tfn, $rTrace, $rCheckSums) =
+            _convert2pdf($fn, $extension, \@trace, \@checkSums);
+        if ($tstatus <= 0) {
+            return ($tstatus, $tmsg);
+        }
+        $fn = $tfn;
+        $extension = FileConverter::Utils::getExtension($fn);
+    }
+    
+    if ($extension =~ m/^pdf$/i) {
+        return pdf2text($fn, \@trace, \@checkSums);
+    } 
+    return (0, "Unsupported file type: $extension");
+
+} # extractText
+
+
+sub convert2pdf {
+    my $fn = shift;
+
+    if (! -e $fn) {
+    	return (0, "File does not exist: $fn");
+    }
+
+    my ($status, $msg) = (1, "");
+    my @trace = ();
+    my @checkSums = ();
+    my ($tstatus, $tmsg, $tfn, $rTrace, $rCheckSums);
+
+    while(FileConverter::Compression::canDecompress($fn) > 0) {
+    	($tstatus, $tmsg, $tfn, $rTrace) =
+            FileConverter::Compression::decompress($fn, \@trace);
+        if ($tstatus <= 0) {
+            return ($tstatus, $tmsg);
+        }
+        $fn = $tfn;
+    }
+
+    my $extension = FileConverter::Utils::getExtension($fn);
+
+    if (!defined $extension) {
+        return (0, "File $fn has no extension");
+    }
+
+    if ($extension =~ m/^pdf$/i) {
+
+	my $sha1 = FileConverter::CheckSum->new();
+	$sha1->digest($fn);
+	push @checkSums, $sha1;
+
+	return (1, "", $fn, \@trace, \@checkSums);
+    }
+
+    if ($extension =~ m/^ps$/i || $extension =~ m/^rtf$/i) {
+	($tstatus, $tmsg, $tfn, $rTrace, $rCheckSums) =
+	    _convert2pdf($fn, $extension, \@trace, \@checkSums);
+	if ($tstatus <= 0) {
+	    return ($tstatus, $tmsg);
+	}
+	
+	my $sha1 = FileConverter::CheckSum->new();
+	$sha1->digest($tfn);
+	push @$rCheckSums, $sha1;
+
+	return ($tstatus, $tmsg, $tfn, $rTrace, $rCheckSums);
+    }
+
+    return (0, "Unsupported file type: $extension");
+
+}  # convert2pdf
+
+
+sub pdf2text {
+    my ($fn, $rTrace, $rCheckSums) = @_;
+    if ($FileConverter::Config::PDFTOTEXT eq "TET") {
+        return FileConverter::TET::extractText($fn, $rTrace, $rCheckSums);
+    }
+    if ($FileConverter::Config::PDFTOTEXT eq "PDFBOX") {
+        return FileConverter::PDFBox::extractText($fn, $rTrace, $rCheckSums);
+    }
+
+} # pdf2text
+
+sub _convert2pdf {
+    my ($fn, $extension, $rTrace, $rCheckSums) = @_;
+    
+    if (($extension =~ m/^rtf$/i) || ($extension =~ m/^doc$/i))  {
+        return FileConverter::JODConverter::convertFile($fn, $rTrace,
+							$rCheckSums);
+    }
+    elsif (($extension =~ m/^ps$/i) || ($extension =~ m/^eps$/i)) {
+        return FileConverter::PSConverter::convertFile($fn, $rTrace,
+						       $rCheckSums);
+    }
+
+} # _convert2pdf
+
+sub ps2text {
+    my($fn, $rTrace, $rCheckSums) = @_;
+    return FileConverter::PSToText::extractText($fn, $rTrace, $rCheckSums);
+
+} # ps2text
+
+1;