biblicit 1.0 → 2.0.3

data/perl/DocFilter/lib/DocFilter/Config.pm

@@ -1,35 +0,0 @@
-#
-# Copyright 2007 Penn State University
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#     http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-package DocFilter::Config;
-
-
-## Global
-
-$algorithmName = "BasicDocFilter";
-$algorithmVersion = "1.0";
-
-
-## Repository Mappings
-
-%repositories = ('example1' => '/',
-		 'example2' => '/home',
-		 );
-
-
-## WS Settings
-
-$serverURL = '127.0.0.1';
-$serverPort = 10666;
-$URI = 'http://citeseerx.org/algorithms/docfilter/wsdl';
-
-1;

data/perl/DocFilter/lib/DocFilter/Filter.pm

@@ -1,51 +0,0 @@
-#
-# Copyright 2007 Penn State University
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#     http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-package DocFilter::Filter;
-##
-##  Isaac Councill, 7/31/07
-##
-use strict;
-
-
-sub filter {
-    my ($filePath) = @_;
-
-    if (!open (IN, "<$filePath")) {
-	return (0, 0, "Could not open file $filePath: $!");
-    }
-    my $text;
-    {
-	local $/ = undef;
-	$text = <IN>;
-    }
-
-    if (hasReferences(\$text) <= 0) {
-	return (1, 0, "No reference section is present");
-    }
-    return (1, 1, "All filters passed");
-
-} # filter
-
-
-sub hasReferences {
-    my $rText = shift;
-    if ($$rText =~ /\b(REFERENCES?|References?|BIBLIOGRAPHY|Bibliography|REFERENCES AND NOTES|References and Notes)\:?\s*\n/sg) {
-	return 1;
-    } else {
-	return 0;
-    }
-
-} # hasReferences
-
-
-1;

data/perl/FileConversionService/README.TXT

@@ -1,11 +0,0 @@
-FileConverter README
-Isaac Councill
-
-Check the FileConverter::Config module to make sure the settings
-are correct.  You will probably need to obtain the PDFBox jar file
-and reference it in Config.  It's better to use TET, but TET is
-expensive.
-
-If using PDFBox, make sure java is on your path.
-
-I know this is sparse, more to come...

data/perl/FileConversionService/lib/CSXUtil/SafeText.pm

@@ -1,140 +0,0 @@
-#
-# Copyright 2007 Penn State University
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#     http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-package CSXUtil::SafeText;
-##
-##  Methods for stripping bad (XML unsafe) characters
-##  from strings and performing basic HTML entity
-##  translations.  Also contains a utility (stripArtifacts)
-##  for getting rid of crazy control characters and
-##  other things that probably aren't proper text.
-##
-##  Isaac Councill, 12/06/06
-##
-#######################################################
-##
-use strict;
-use utf8;
-require Exporter;
-
-our ($VERSION, @ISA, @EXPORT, @EXPORT_OK, %EXPORT_TAGS);
-
-$VERSION = 1.00;
-
-@ISA = qw(Exporter);
-@EXPORT_OK = qw(@badChars %htmlSpecialChars
-		%htmlCharEntities &stripBadChars
-		&encodeHTMLSpecialChars
-		&decodeHTMLSpecialChars
-		&cleanXML &cleanAll &stripArtifacts);
-
-
-##
-#######################################################
-##
-##  Sharable encoding data.
-##
-
-##  Hex codes for characters that should never be put into
-##  XML - or else parsers will barf.
-our @badChars = qw(\x00 \x01 \x02 \x03 \x04 \x05 \x06 \x07
-		   \x08 \x0B \x0C \x0E \x0F \x10 \x11 \x12
-		   \x13 \x14 \x15 \x16 \x17 \x18 \x19 \x1A
-		   \x1B \x1C \x1D \x1E \x1F \x7F);
-
-##  Subset of HTML characters that could be problematic
-##  for XML.  This is not a complete list of HTML
-##  special characters, but more mappings can be added
-##  as needed.
-our %htmlSpecialCharEncodings = ("&" => "&",
-				 ">" => ">",
-				 "<" => "&lt;",
-				 "\"" => "&quot;"
-				 );
-
-##  The reverse map.
-our %htmlSpecialCharDecodings;
-foreach my $key (keys %htmlSpecialCharEncodings) {
-    my $val = $htmlSpecialCharEncodings{$key};
-    $htmlSpecialCharDecodings{$val} = $key;
-}
-
-
-##
-#######################################################
-##
-##  Subroutines
-##
-
-##  Delete all occurences of bad characters in text,
-##  returns a new string that is clean.
-sub stripBadChars {
-    my $rtext = shift;
-    foreach my $char (@badChars) {
-	$$rtext =~ s/$char//g;
-    }
-}
-
-
-##  Encodes special characters into HTML equivalents
-##  and returns the encoded string.
-sub encodeHTMLSpecialChars {
-    my $rtext = shift;
-    foreach my $char (keys %htmlSpecialCharEncodings) {
-	my $code = $htmlSpecialCharEncodings{$char};
-	$$rtext =~ s/$char/$code/g;
-    }
-}
-
-
-##  Decodes a HTML entities in the supplied string
-##  into non-HTML character equivalents and returns
-##  the decoded string.
-sub decodeHTMLSpecialChars {
-    my $rtext = shift;
-    foreach my $code (keys %htmlSpecialCharDecodings) {
-	my $char = $htmlSpecialCharDecodings{$code};
-	$$rtext =~ s/$code/$char/g;
-    }
-}
-
-
-##  Strip out any characters that don't look like they
-##  belong in a proper, readable text string.
-##
-sub stripArtifacts {
-    my $rtext = shift;
-    $$rtext =~ s/[^\p{IsAlnum}\p{IsPunct}\p{IsSpace}\p{IsS}]//g;
-}
-
-
-##  Convenience routine for executing both XML safety
-##  routines in a single call.
-##
-sub cleanXML {
-    my $rtext = shift;
-    stripBadChars($rtext);
-    encodeHTMLSpecialChars($rtext);
-}
-
-
-##  Clean for XML and also strip out strange characters.
-##
-sub cleanAll {
-    my $rtext = shift;
-    stripBadChars($rtext);
-    stripArtifacts($rtext);
-    encodeHTMLSpecialChars($rtext);
-}
-
-
-1;

data/perl/FileConversionService/lib/FileConverter/CheckSum.pm

@@ -1,77 +0,0 @@
-#
-# Copyright 2007 Penn State University
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#     http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-package FileConverter::CheckSum;
-#
-# Container for checksum info and utilities for individual files.
-#
-# Isaac Councill
-#
-use strict;
-use Digest::SHA1;
-use FileConverter::Utils;
-
-
-sub new {
-    my ($class) = @_;
-    my $self = {
-	'_fileType' => undef,
-	'_sha1'     => undef,
-    };
-    bless $self, $class;
-    return $self;
-
-}  # new
-
-
-sub getFileType {
-    my $self = shift;
-    return $self->{'_fileType'};
-}
-
-
-sub setFileType {
-    my ($self, $fileType) = @_;
-    $self->{'_fileType'} = $fileType;
-}
-
-
-sub getSHA1 {
-    my $self = shift;
-    return $self->{'_sha1'};
-}
-
-
-sub setSHA1 {
-    my ($self, $sha1) = @_;
-    $self->{'_sha1'} = $sha1;
-}
-
-
-sub digest {
-    my ($self, $filePath) = @_;
-
-    open(FILE, "<$filePath") or die ("Could not open for reading: $filePath");
-    my $digester = Digest::SHA1->new;
-    $digester->addfile(*FILE);
-
-    my $ext = FileConverter::Utils::getExtension($filePath);
-    my $sha1 = $digester->hexdigest;
-    close FILE;
-
-    $self->setFileType($ext);
-    $self->setSHA1($sha1);
-
-}  # digest
-
-1;
-

data/perl/FileConversionService/lib/FileConverter/Compression.pm

@@ -1,137 +0,0 @@
-#
-# Copyright 2007 Penn State University
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#     http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-package FileConverter::Compression;
-#
-# Utilities for handling various compression formats.
-#
-# Isaac Councill, 09/06/07
-#
-use strict;
-use FileConverter::Config;
-use FileConverter::Utils;
-
-# Should all be lower case.
-my %supportedCompressionExt = ("gz" => 1,
-			       "zip" => 1,
-			       "z" => 1,
-			       );
-
-my $gunzip = $FileConverter::Config::gunzip;
-my $uncompress = $FileConverter::Config::uncompress;
-my $unzip = $FileConverter::Config::unzip;
-
-
-sub decompress {
-    my ($fn, $rTrace) = @_;
-    my $ext = FileConverter::Utils::getExtension($fn);
-    if ($ext =~ m/^gz$/i) {
-	return gunzip($fn, $rTrace);
-    }
-    if ($ext =~ m/^z$/i) {
-	return uncompress($fn, $rTrace);
-    }
-    if ($ext =~ m/^zip$/i) {
-	return unzip($fn, $rTrace);
-    }
-    return (0, "Unsupported compression extension: $ext");
-
-} # decompress
-
-
-sub canDecompress {
-    my ($fn) = @_;
-    my $ext = FileConverter::Utils::getExtension($fn);
-    if (defined $supportedCompressionExt{lc($ext)}) {
-	return 1;
-    } else {
-	return 0;
-    }
-
-} # canDecompress
-
-
-sub gunzip {
-    my ($fn, $rTrace) = @_;
-    my @commandArgs = ($gunzip, "-f", $fn);
-
-    system(@commandArgs);
-
-    if ($? == -1) {
-	return (0, "Failed to execute gunzip: $!");
-    } elsif ($? & 127) {
-	return (0, "gunzip died with signal ".($? & 127));
-    };
-    my $code = $?>>8;
-    if ($code == 1) {
-	return (0, "Error executing gunzip (code $code): $!");
-    }
-
-    push @$rTrace, "gunzip";
-
-    my $newFile = FileConverter::Utils::stripExtension($fn);
-    return (1, "", $newFile, $rTrace);
-
-} # gunzip
-
-
-sub uncompress {
-    my ($fn, $rTrace) = @_;
-    my @commandArgs = ($uncompress, "-f", $fn);
-
-    system(@commandArgs);
-
-    if ($? == -1) {
-	return (0, "Failed to execute uncompress: $!");
-    } elsif ($? & 127) {
-	return (0, "uncompress died with signal ".($? & 127));
-    };
-    my $code = $?>>8;
-    if ($code == 1) {
-	return (0, "Error executing uncompress (code $code): $!");
-    }
-
-    push @$rTrace, "uncompress";
-
-    my $newFile = FileConverter::Utils::stripExtension($fn);
-    return (1, "", $newFile, $rTrace);
-
-} # uncompress
-
-
-sub unzip {
-    my ($fn, $rTrace) = @_;
-
-    my $dir = FileConverter::Utils::getDirectory($fn);
-    my @commandArgs = ($unzip, "-qqo", $fn, "-d", $dir);
-
-    system(@commandArgs);
-
-    if ($? == -1) {
-	return (0, "Failed to execute unzip: $!");
-    } elsif ($? & 127) {
-	return (0, "unzip died with signal ".($? & 127));
-    };
-    my $code = $?>>8;
-    if ($code > 2) {
-	return (0, "Error executing unzip (code $code): $!");
-    }
-
-    push @$rTrace, "unzip";
-
-    my $newFile = FileConverter::Utils::stripExtension($fn);
-    return (1, "", $newFile, $rTrace);
-
-} # unzip
-
-
-1;

data/perl/FileConversionService/lib/FileConverter/Config.pm

@@ -1,57 +0,0 @@
-#
-# Copyright 2007 Penn State University
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#     http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-package FileConverter::Config;
-
-use FindBin;
-
-## Conversion utilities
-
-# valid options are TET or PDFBOX
-$PDFTOTEXT = "PDFBOX";
-
-# valid options are TEXT or PDF
-$PSConversion = "TEXT";
-
-$TETPath = "$FindBin::Bin/../converters/TET-2.2-Linux/bin/tet";
-
-$TETLicensePath =
-    "$FindBin::Bin/../converters/TET-2.2-Linux/licensekeys.txt";
-
-$PDFBoxLocation = "$FindBin::Bin/FileConversionService/converters/PDFBox/pdfbox-app-1.7.1.jar";
-
-$JODConverterPath = 
-    "$FindBin::Bin/../converters/jodconverter-2.2.0/jodconverter-cli-2.2.0.jar";
-
-$PrescriptPath = "/usr/local/bin/prescript";
-
-## Compression utilities
-
-$gunzip = "/usr/bin/gunzip";
-$uncompress = "/usr/bin/uncompress";
-$unzip = "/usr/bin/unzip";
-
-
-## Repository Mappings
-
-%repositories = ('example1' => '/',
-        		 'example2' => '/home',
-                );
-
-
-## WS settings
-
-$serverURL = '127.0.0.1';
-$serverPort = 10888;
-$URI = 'http://citeseerx.org/fileConversion/wsdl';
-
-1;

data/perl/FileConversionService/lib/FileConverter/Controller.pm

@@ -1,191 +0,0 @@
-#
-# Copyright 2007 Penn State University
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#     http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-package FileConverter::Controller;
-#
-# Main interface to FileConverter.  This is where all calls
-# should start, and where all conversion sequences should
-# be managed.
-#
-# Isaac Councill, 09/06/07
-#
-use strict;
-use FileConverter::Utils;
-use FileConverter::Compression;
-use FileConverter::TET;
-use FileConverter::PDFBox;
-use FileConverter::JODConverter;
-use FileConverter::PSConverter;
-use FileConverter::PSToText;
-use FileConverter::Prescript;
-use FileConverter::CheckSum;
-
-
-##
-# Convert the given file to text.  Decompression will occur
-# first, and then further conversion and/or text extraction
-# will proceed according to the file type.
-#
-# Supports zip, gz, .Z compression, but only supports the
-# PDF file type for now.
-##
-sub extractText {
-    my $fn = shift;
-
-    if (! -e $fn) {
-    	return (0, "File does not exist: $fn");
-    }
-
-    my ($status, $msg) = (1, "");
-    my @trace = ();
-    my @checkSums = ();
-    my ($tstatus, $tmsg, $tfn, $rTrace, $rCheckSums);
-
-    while(FileConverter::Compression::canDecompress($fn) > 0) {
-    	($tstatus, $tmsg, $tfn, $rTrace) =
-            FileConverter::Compression::decompress($fn, \@trace);
-        if ($tstatus <= 0) {
-            return ($tstatus, $tmsg);
-        }
-        $fn = $tfn;
-    }
-
-    my $extension = FileConverter::Utils::getExtension($fn);
-
-    if (!defined $extension) {
-        return (0, "File $fn has no extension");
-    }
-
-    if ($extension =~ m/^ps$/i) {
-        # convert poscript file according config.
-        if ($FileConverter::Config::PSConversion eq "TEXT") {
-            # go from ps to text directly
-	    _convert2pdf($fn, $extension, [], \@checkSums);
-            return ps2text($fn, \@trace, \@checkSums);
-        }
-        if ($FileConverter::Config::PSConversion eq "PDF") {
-            # convert to PDF first then to text
-            ($tstatus, $tmsg, $tfn, $rTrace, $rCheckSums) = 
-                _convert2pdf($fn, $extension, \@trace, \@checkSums);
-            if ($tstatus <= 0) {
-                return ($tstatus, $tmsg);
-            }
-            $fn = $tfn;
-            $extension = FileConverter::Utils::getExtension($fn);
-        }
-    }
-    elsif (($extension !~ m/^pdf$/i) && ($extension !~ m/^ps$/i)) {
-        # first, we need to convert the file to PDF.
-        ($tstatus, $tmsg, $tfn, $rTrace, $rCheckSums) =
-            _convert2pdf($fn, $extension, \@trace, \@checkSums);
-        if ($tstatus <= 0) {
-            return ($tstatus, $tmsg);
-        }
-        $fn = $tfn;
-        $extension = FileConverter::Utils::getExtension($fn);
-    }
-    
-    if ($extension =~ m/^pdf$/i) {
-        return pdf2text($fn, \@trace, \@checkSums);
-    } 
-    return (0, "Unsupported file type: $extension");
-
-} # extractText
-
-
-sub convert2pdf {
-    my $fn = shift;
-
-    if (! -e $fn) {
-    	return (0, "File does not exist: $fn");
-    }
-
-    my ($status, $msg) = (1, "");
-    my @trace = ();
-    my @checkSums = ();
-    my ($tstatus, $tmsg, $tfn, $rTrace, $rCheckSums);
-
-    while(FileConverter::Compression::canDecompress($fn) > 0) {
-    	($tstatus, $tmsg, $tfn, $rTrace) =
-            FileConverter::Compression::decompress($fn, \@trace);
-        if ($tstatus <= 0) {
-            return ($tstatus, $tmsg);
-        }
-        $fn = $tfn;
-    }
-
-    my $extension = FileConverter::Utils::getExtension($fn);
-
-    if (!defined $extension) {
-        return (0, "File $fn has no extension");
-    }
-
-    if ($extension =~ m/^pdf$/i) {
-
-	my $sha1 = FileConverter::CheckSum->new();
-	$sha1->digest($fn);
-	push @checkSums, $sha1;
-
-	return (1, "", $fn, \@trace, \@checkSums);
-    }
-
-    if ($extension =~ m/^ps$/i || $extension =~ m/^rtf$/i) {
-	($tstatus, $tmsg, $tfn, $rTrace, $rCheckSums) =
-	    _convert2pdf($fn, $extension, \@trace, \@checkSums);
-	if ($tstatus <= 0) {
-	    return ($tstatus, $tmsg);
-	}
-	
-	my $sha1 = FileConverter::CheckSum->new();
-	$sha1->digest($tfn);
-	push @$rCheckSums, $sha1;
-
-	return ($tstatus, $tmsg, $tfn, $rTrace, $rCheckSums);
-    }
-
-    return (0, "Unsupported file type: $extension");
-
-}  # convert2pdf
-
-
-sub pdf2text {
-    my ($fn, $rTrace, $rCheckSums) = @_;
-    if ($FileConverter::Config::PDFTOTEXT eq "TET") {
-        return FileConverter::TET::extractText($fn, $rTrace, $rCheckSums);
-    }
-    if ($FileConverter::Config::PDFTOTEXT eq "PDFBOX") {
-        return FileConverter::PDFBox::extractText($fn, $rTrace, $rCheckSums);
-    }
-
-} # pdf2text
-
-sub _convert2pdf {
-    my ($fn, $extension, $rTrace, $rCheckSums) = @_;
-    
-    if (($extension =~ m/^rtf$/i) || ($extension =~ m/^doc$/i))  {
-        return FileConverter::JODConverter::convertFile($fn, $rTrace,
-							$rCheckSums);
-    }
-    elsif (($extension =~ m/^ps$/i) || ($extension =~ m/^eps$/i)) {
-        return FileConverter::PSConverter::convertFile($fn, $rTrace,
-						       $rCheckSums);
-    }
-
-} # _convert2pdf
-
-sub ps2text {
-    my($fn, $rTrace, $rCheckSums) = @_;
-    return FileConverter::PSToText::extractText($fn, $rTrace, $rCheckSums);
-
-} # ps2text
-
-1;