biblicit 1.0

data/perl/FileConversionService/lib/FileConverter/JODConverter.pm

@@ -0,0 +1,61 @@
+#
+# Copyright 2007 Penn State University
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+package FileConverter::JODConverter;
+#
+# Wrapper to execute the JODConverter command-line tool for converting
+# doc, rtf to PDF files.
+#
+# Juan Pablo Fernandez Ramirez, 10/05/07
+#
+use strict;
+use FileConverter::Config;
+use FileConverter::Utils;
+
+my $JODConverterLoc = $FileConverter::Config::JODConverterPath;
+
+##
+# Execute the JODConverter utility.
+##
+sub convertFile {
+    my ($filePath, $rTrace, $rCheckSums) = @_;
+    my ($status, $msg) = (1, "");
+    
+    if (FileConverter::Utils::checkProcess("soffice") == 0) {
+    	return (0, "Open Office Service is not running");
+    }
+    
+    my $pdfFilePath = FileConverter::Utils::changeExtension($filePath, "pdf");
+    my @commandArgs = ("java", "-jar", $JODConverterLoc, $filePath, 
+        $pdfFilePath);
+    system(@commandArgs);
+    
+    if ($? == -1) {
+        return (0, "Failed to execute JODConverter: $!");
+    } elsif ($? & 127) {
+        return (0, "Java died with signal ".($? & 127));
+    }
+
+    my $code = $?>>8;
+    if ($code == 0) {
+        push @$rTrace, "JODConverter";
+
+	my $sha1 = FileConverter::CheckSum->new();
+	$sha1->digest($filePath);
+	push @$rCheckSums, $sha1;
+
+        return ($status, $msg, $pdfFilePath, $rTrace, $rCheckSums);
+    } else {
+        return (0, "Error executing JODConverter (code $code): $!");
+    }
+} # convertFile
+1;

data/perl/FileConversionService/lib/FileConverter/PDFBox.pm

@@ -0,0 +1,69 @@
+#
+# Copyright 2007 Penn State University
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+package FileConverter::PDFBox;
+#
+# Wrapper to call the PDFBox ExtractText command-line tool
+# for extracting text from PDF files.  It's recommended to
+# use TET instead, if TET is available.
+#
+# Isaac Councill, 09/06/07
+#
+use strict;
+use FileConverter::Config;
+use FileConverter::Utils;
+
+my $PDFBoxLoc = $FileConverter::Config::PDFBoxLocation;
+
+##
+# Execute the PDFBox utility.
+##
+sub extractText {
+    my ($filePath, $rTrace, $rCheckSums) = @_;
+    my ($status, $msg) = (1, "");
+
+    if (FileConverter::Utils::checkExtension($filePath, "pdf") <= 0) {
+	return (0, "Unexpected file extension at ".
+		__FILE__." line ".__LINE__);
+    }
+
+    my $textFilePath =
+	FileConverter::Utils::changeExtension($filePath, "txt");
+    my @commandArgs = ("java", "-jar", $PDFBoxLoc,
+		       "ExtractText", "-encoding",
+		       "utf8", $filePath, $textFilePath);
+
+    system(@commandArgs);
+
+    if ($? == -1) {
+	return (0, "Failed to execute PDFBox: $!");
+    } elsif ($? & 127) {
+	return (0, "Java died with signal ".($? & 127));
+    }
+
+    my $code = $?>>8;
+    if ($code == 0) {
+	push @$rTrace, "PDFBox";
+
+	my $sha1 = FileConverter::CheckSum->new();
+	$sha1->digest($filePath);
+	push @$rCheckSums, $sha1;
+
+	return ($status, $msg, $textFilePath, $rTrace, $rCheckSums);
+    } else {
+	return (0, "Error executing PDFBox (code $code): $!");
+    }
+
+} # extractText
+
+
+1;

data/perl/FileConversionService/lib/FileConverter/PSConverter.pm

@@ -0,0 +1,69 @@
+#
+# Copyright 2007 Penn State University
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+package FileConverter::PSConverter;
+#
+# Wrapper to execute the ps2pdf command-line tool for converting
+# ps to PDF files.
+#
+# Juan Pablo Fernandez Ramirez, 10/08/07
+#
+use strict;
+use FileConverter::Config;
+use FileConverter::Utils;
+
+my $timeout = 20;
+
+##
+# Execute the converter utility.
+##
+sub convertFile {
+    my ($filePath, $rTrace, $rCheckSums) = @_;
+    my ($status, $msg) = (1, "");
+    
+    my $pdfFilePath = FileConverter::Utils::changeExtension($filePath, "pdf");
+    my @commandArgs = ("ps2pdf13", $filePath, $pdfFilePath);
+    my $child;
+    eval {
+	local $SIG{'ALRM'} = sub { die "alarm\n" };
+	alarm $timeout;
+	$child = system(@commandArgs);
+	alarm 0;
+    };
+
+    if ($@) {
+	if ($@ eq "alarm\n") {
+	    if (defined $child) { kill 9, $child; }
+	    return (0, "ps2pdf timeout");
+	}
+    }
+    
+    if ($? == -1) {
+        return (0, "Failed to execute ps2pdf: $!");
+    } elsif ($? & 127) {
+        return (0, "ps2pdf died with signal ".($? & 127));
+    }
+
+    my $code = $?>>8;
+    if ($code == 0) {
+        push @$rTrace, "ps2pdf";
+
+	my $sha1 = FileConverter::CheckSum->new();
+	$sha1->digest($filePath);
+	push @$rCheckSums, $sha1;
+
+        return ($status, $msg, $pdfFilePath, $rTrace, $rCheckSums);
+    } else {
+        return (0, "Error executing ps2pdf (code $code): $!");
+    }
+} # convertFile
+1;

data/perl/FileConversionService/lib/FileConverter/PSToText.pm

@@ -0,0 +1,88 @@
+#
+# Copyright 2007 Penn State University
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+package FileConverter::PSToText;
+#
+# Wrapper to execute the ps2ascii command-line tool for converting
+# ps to text files.
+#
+# Isaac, 10/08/07
+#
+use strict;
+use FileConverter::Config;
+use FileConverter::Utils;
+use Encode;
+
+my $timeout = 20;
+
+##
+# Execute the converter utility.
+##
+sub extractText {
+    my ($filePath, $rTrace, $rCheckSums) = @_;
+    my ($status, $msg) = (1, "");
+    
+    my $txtFilePath = FileConverter::Utils::changeExtension($filePath, "txt");
+
+    my @commandArgs = ("ps2ascii", $filePath, $txtFilePath);
+    my $child;
+    eval {
+	local $SIG{'ALRM'} = sub { die "alarm\n" };
+	alarm $timeout;
+	$child = system(@commandArgs);
+	alarm 0;
+    };
+
+    if ($@) {
+	if ($@ eq "alarm\n") {
+	    if (defined $child) { kill 9, $child; }
+	    return (0, "ps2ascii timeout");
+	}
+    }
+    
+    if ($? == -1) {
+        return (0, "Failed to execute ps2ascii: $!");
+    } elsif ($? & 127) {
+        return (0, "ps2ascii died with signal ".($? & 127));
+    }
+
+    my $code = $?>>8;
+    if ($code == 0) {
+        push @$rTrace, "ps2ascii";
+	ascii2utf8($txtFilePath);
+
+	my $sha1 = FileConverter::CheckSum->new();
+	$sha1->digest($filePath);
+	push @$rCheckSums, $sha1;
+
+        return ($status, $msg, $txtFilePath, $rTrace, $rCheckSums);
+    } else {
+        return (0, "Error executing ps2ascii (code $code): $!");
+    }
+} # convertFile
+
+sub ascii2utf8 {
+    my $fn = shift;
+
+    open(IN, "<$fn") or die $!;
+    my $text;
+    {
+	local $/ = undef;
+	$text = <IN>;
+    }
+    close IN;
+    $text = Encode::decode_utf8($text);
+    open(OUT, ">:utf8", $fn) or die $!;
+    print OUT $text;
+    close OUT;
+}
+1;

data/perl/FileConversionService/lib/FileConverter/Prescript.pm

@@ -0,0 +1,68 @@
+#
+# Copyright 2007 Penn State University
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+package FileConverter::Prescript;
+#
+# Wrapper to execute the Prescript command-line tool for extracting
+# text from PS files.
+#
+# Juan Pablo Fernandez R., 10/31/07
+#
+use strict;
+use FileConverter::Config;
+use FileConverter::Utils;
+use FileConverter::CheckSum;
+
+my $PrescriptPath = $FileConverter::Config::PrescriptPath;
+
+##
+# Execute the Prescript utility.
+##
+sub extractText {
+    my ($filePath, $rTrace, $rCheckSums) = @_;
+    my ($status, $msg) = (1, "");
+
+    if (FileConverter::Utils::checkExtension($filePath, "ps") <= 0) {
+        return (0, "Unexpected file extension at ". __FILE__." line ".__LINE__);
+    }
+
+    my $textFilePath = FileConverter::Utils::changeExtension($filePath, "txt");
+    my @commandArgs = ($PrescriptPath, "plain", $filePath, $textFilePath);
+
+    system(@commandArgs);
+
+    if ($? == -1) {
+        return (0, "Failed to execute Prescript: $!");
+    } elsif ($? & 127) {
+        return (0, "Prescript died with signal ".($? & 127));
+    }
+
+    my $code = $?>>8;
+    if (($code == 0) || ($code == 1)) {
+        if ($code == 1) {
+            print STDERR "Prescript completed with errors: $filePath\n";
+        }
+
+        push @$rTrace, "PSLIB Prescript";
+
+        my $sha1 = new FileConverter::CheckSum();
+        $sha1->digest($filePath);
+        push @$rCheckSums, $sha1;
+
+        return ($status, $msg, $textFilePath, $rTrace, $rCheckSums);
+
+    } else {
+        return (0, "Error executing Prescript (code $code): $!");
+    }
+} # extractText
+
+1;

data/perl/FileConversionService/lib/FileConverter/TET.pm

@@ -0,0 +1,75 @@
+#
+# Copyright 2007 Penn State University
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+package FileConverter::TET;
+#
+# Wrapper to execute the TET command-line tool for extracting
+# text from PDF files.
+#
+# Isaac Councill, 09/06/07
+#
+use strict;
+use FileConverter::Config;
+use FileConverter::Utils;
+use FileConverter::CheckSum;
+
+my $TETPath = $FileConverter::Config::TETPath;
+my $TETLicensePath = $FileConverter::Config::TETLicensePath;
+
+$ENV{'PDFLIBLICENSEFILE'} = $TETLicensePath;
+
+##
+# Execute the TET utility.
+##
+sub extractText {
+    my ($filePath, $rTrace, $rCheckSums) = @_;
+    my ($status, $msg) = (1, "");
+
+    if (FileConverter::Utils::checkExtension($filePath, "pdf") <= 0) {
+	return (0, "Unexpected file extension at ".
+		__FILE__." line ".__LINE__);
+    }
+
+    my $textFilePath =
+	FileConverter::Utils::changeExtension($filePath, "txt");
+    my @commandArgs = ($TETPath, "-o", $textFilePath, $filePath);
+
+    system(@commandArgs);
+
+    if ($? == -1) {
+	return (0, "Failed to execute TET: $!");
+    } elsif ($? & 127) {
+	return (0, "TET died with signal ".($? & 127));
+    }
+
+    my $code = $?>>8;
+    if (($code == 0) || ($code == 1)) {
+	if ($code == 1) {
+	    print STDERR "TET completed with errors: $filePath\n";
+	}
+
+	push @$rTrace, "PDFLib TET";
+
+	my $sha1 = new FileConverter::CheckSum();
+	$sha1->digest($filePath);
+	push @$rCheckSums, $sha1;
+
+	return ($status, $msg, $textFilePath, $rTrace, $rCheckSums);
+
+    } else {
+	return (0, "Error executing TET (code $code): $!");
+    }
+
+} # extractText
+
+
+1;

data/perl/FileConversionService/lib/FileConverter/Utils.pm

@@ -0,0 +1,130 @@
+#
+# Copyright 2007 Penn State University
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+package FileConverter::Utils;
+#
+# Container for subroutines that may be shared across multiple
+# FileConverter modules.
+#
+# Isaac Councill, 09/06/07
+#
+use strict;
+use Encode;
+
+##
+# Returns the file extension of a file name, if there is one.
+##
+sub getExtension {
+    my ($fn) = @_;
+    if ($fn =~ m/^.*\.(.*)$/) {
+	return $1;
+    }
+    return undef;
+
+} # getExtension
+
+
+##
+# Strips off the last extension of the file name.
+##
+sub stripExtension {
+    my ($fn) = @_;
+    $fn =~ s/^(.*)\..*$/$1/;
+    return $fn;
+
+} # stripExtension
+
+
+##
+##
+# Routine for checking that a filename ends with an expected
+# extension.  Returns 1 if it does, 0 if not.
+##
+sub checkExtension {
+    my ($fn, $ext) = @_;
+    if ($fn =~ m/^.*\.(.*)$/) {
+	if ($1 =~ m/$ext/i) {
+	    return 1;
+	}
+    }
+    return 0;
+
+} # checkExtension
+
+
+##
+# Simple routine for changing the extension of a file.
+# Example: $newFileName = changeExtension($oldFileName, "txt");
+##
+sub changeExtension {
+    my ($fn, $ext) = @_;
+    unless ($fn =~ s/^(.*)\..*$/$1\.$ext/) {
+	$fn .= ".$ext";
+    }
+    return $fn;
+
+} # changeExtension
+
+
+##
+# Returns the directory part of a file path.
+##
+sub getDirectory {
+    my ($filePath) = @_;
+    if ($filePath =~ m/^(.*)\/.*$/) {
+	return $1;
+    } else {
+	return $filePath;
+    }
+
+} # getDirectory
+
+##
+##
+# Routine for checking if a process is running or not
+# Returns 1 if it is runnig, 0 if not.
+##
+sub checkProcess {
+	my ($process) = @_;
+	my $cmd = "ps -ef | grep " . $process . " | grep -v grep";
+	my $result = `$cmd`;
+	if ($result eq '') {
+		return 0;
+	}
+	else {
+		return 1;
+	}
+} # checkProcess
+
+
+##
+# Convert an file of the specified encoding to UTF-8
+##
+sub convertToUTF8 {
+    my ($fn, $encoding) = @_;
+    my $octets;
+    open (FILE, "<$fn") or die "could not open file $fn: $!";
+    binmode FILE, ":bytes";
+    {
+	local $/ = undef;
+	$octets = <FILE>;
+    }
+    close FILE;
+    
+    Encode::from_to($octets, $encoding, "utf8");
+    open (FILE, ">:utf8", "$fn") or die "could not open file $fn: $!";
+    print FILE Encode::decode_utf8($octets);
+    close FILE;
+
+}
+
+1;

data/perl/HeaderParseService/README.TXT

@@ -0,0 +1,80 @@
+SVMHeaderParse README
+IGC
+
+SVMHeaderParse is a utility for extracting header information
+from research papers based on Support Vector Machines and
+heuristic regularization.  For details on the algorithm,
+please see the following paper:
+
+Hui Han, C. Lee Giles, Eren Manavoglu, Hongyuan Zha,
+Zhenyue Zhang, Edward A. Fox. "Automatic Document Metadata
+Extraction Using Support Vector Machines", in Proceedings
+of ACM/IEEE Joint Conference on Digital Libraries
+(JCDL 2003): 37-48, 2003.
+
+Installation:
+
+In order to use the SVMHeaderParse web service you will need
+the following modules in your perl library:
+
+Log::Log4perl
+Log::Dispatch
+
+Edit lib/HeaderParse/Config/API_Config.pm to provide values
+appropriate for your environment.  Also edit wsdl/SVMHeaderParse.wsdl
+to reflect any changes to your service URL.
+
+Command Line Usage:
+
+Command line utilities are provided for extracting header information
+from text documents.  By default, text files are expected to be encoded
+in UTF-8, but the expected encoding can be adjusted using perl
+command line switches.  To run SVMHeaderParse on a single document,
+execute the following command:
+
+extractHeader.pl textfile [outfile]
+
+If "outfile" is specified, the XML output will be written to that
+file; otherwise, the XML will be printed to STDOUT.
+
+There is also a web service interface available, using the
+SOAP::Lite perl module.  To start the service, just execute:
+
+headerparse-service.pl
+
+By default, the service will start on port 40000, but this is
+configurable in the HeaderParse::Config::API_Config library module.
+A WSDL file is provided with the distribution that outlines the
+message details expected by the SVMHeaderParse service.  If the
+service port is changed, the WSDL file must also be modified to
+reflect that change.  Expected parameters in the input message
+are "filePath" (a path to the text file to parse) and "repositoryID".
+The SVMHeaderParse service is designed for deployment in an
+environment where text files may be located on file systems mounted
+from arbitrary machines on the network.  Thus, "repositoryID" provides
+a means to map a given shared file system to it's mount point.
+Repository mappings are configurable in the API_Config module.  The
+"filePath" parameter provides a path to the text file relative to
+the repository mount point.  The local file system may be specified
+using the reserved repository ID "LOCAL".  In that case, an absolute
+path to the text file may be specified.
+
+A perl client is also provided that demonstrates how to use the
+service.  Execute the client with the following command:
+
+headerparse-client.pl filePath repositoryID
+
+If the call is successful, the XML output will be printed to STDOUT.
+
+API:
+
+The SVMHeaderParse libraries may be used directly from external perl
+applications.  The interface module is HeaderParse::API::Parser.  If
+XML output is desired, use the
+
+HeaderParse::API::Parser::_parseHeader($filePath, $jobID)
+
+subroutine.  If the SVMHeaderParse library is used from external
+Perl applications, remember to use the "-CSD" perl option for
+global unicode stream support (or otherwise handle encoding) or
+risk string corruption.