biblicit 1.0 → 2.0.3

data/parscit/bin/xml2train.pl

@@ -0,0 +1,193 @@
+#!/usr/bin/perl
+# Author: Do Hoang Nhat Huy <dcsdhnh@nus.edu.sg>, generated at Fri, 3 Dec 2010 14:36:00
+# Modified from template by Min-Yen Kan <kanmy@comp.nus.edu.sg>
+require 5.0;
+use strict;
+
+use FindBin;
+use Getopt::Long;
+
+# I do not know a better solution to find a lib path in -T mode.
+# So if you know a better solution, I'd be glad to hear.
+# See this http://www.perlmonks.org/?node_id=585299 for why I used the below code
+my $path;	# Path to Parscit binary directory
+BEGIN 
+{
+	if ($FindBin::Bin =~ /(.*)/) { $path  = $1; }
+}
+
+use lib "$path/../lib";
+
+# Local libraries
+use Omni::Omnidoc;
+use ParsCit::Tr2crfpp;
+use ParsCit::PreProcess;
+# Dependencies
+
+
+### USER customizable section
+my $version = "1.0";
+$0 =~ /([^\/]+)$/; my $progname = $1;
+### END user customizable section
+
+sub License 
+{
+	print STDERR "# Copyright 2011 \251 by Do Hoang Nhat Huy\n";
+}
+
+### HELP Sub-procedure
+sub Help 
+{
+	print STDERR "Process Omnipage XML output (Reference Section Only) and extract text lines together with other XML information\n";
+	print STDERR "usage: $progname -h\t[invokes help]\n";
+  	print STDERR "       $progname -in xmlfile -out outfile -opt option [-codec -app]\n";
+ 	print STDERR "Options:\n";
+  	print STDERR "\t-q		\tQuiet Mode (don't echo license)\n";
+  	print STDERR "\t-in		\tXML input from Omnipage\n";
+  	print STDERR "\t-out	\tOutput file\n";
+  	print STDERR "\t-codec	\tCodec of the input XML: utf-16 or utf-8. Default is utf-8\n";
+  	print STDERR "\t-opt	\tOptio: train (output is train file for crf++) or xml (output is xml features). Default is train\n";
+}
+
+my $help	= 0;
+my $quite	= 0;
+my $infile	= undef;
+my $outfile	= undef;
+my $option	= "train";
+my $codec	= "utf-8";
+
+$help = 1 unless GetOptions('in=s'		=> \$infile,
+			 				'out=s'		=> \$outfile,
+							'opt=s'		=> \$option,
+							'codec=s'	=> \$codec,
+			 				'h' 		=> \$help,
+							'q' 		=> \$quite);
+
+if ($help || !defined $infile || !defined $outfile) 
+{
+	Help();
+	exit(0);
+}
+
+if (!$quite) 
+{
+	License();
+}
+
+# Sanity check
+if (($option ne "train") && ($option ne "xml"))
+{
+	die "Die: -opt must equal \"train\" or \"xml\".\n";
+}
+
+if (($codec ne "utf-8") && ($codec ne "utf-16"))
+{
+	die "Die: -codec must equal \"utf-8\" or \"utf-16\".\n";
+}
+
+# Untaint check
+$infile		= UntaintPath($infile);
+$outfile 	= UntaintPath($outfile);
+
+$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
+# End untaint check
+
+# MAIN
+my $infile_utf8 = $infile . "-utf8";
+if ($codec eq "utf-16") { Convert($infile, "UTF16", $infile_utf8, "UTF8"); }
+
+if (! open(IN, "<:utf8", $infile)) { return (-1, "Could not open xml file " . $infile . ": " . $!); }
+my $xml = do { local $/; <IN> };
+close IN;
+
+# Cleanup
+CleanUp(\$xml);
+
+# New document
+my $doc = new Omni::Omnidoc();
+$doc->set_raw($xml);
+
+# Extract the reference portion from the XML
+my ($start_ref, $end_ref, $rcite_text_from_xml) = ParsCit::PreProcess::findCitationTextXML($doc);
+
+if ($option eq "train")
+{
+	# Prepare to split unmarked reference portion
+	my $tmp_file = ParsCit::Tr2crfpp::prepDataUnmarked($doc, $start_ref, $end_ref);
+
+	# Save the temporary file
+	my $cmd = "mv " . $tmp_file . " " . $outfile;
+
+	Execute($cmd);
+}
+else
+{
+	
+}
+
+# END
+
+# Convert the input XML
+sub Convert
+{
+	my ($from_file, $from_encode, $to_file, $to_encode, $log) = @_;
+	
+	# Call iconv program
+	my $cmd = "iconv" . " -f " . $from_encode . " -t " . $to_encode . " " . $from_file . " -o " . $to_file;
+	
+	# Transformation
+	Execute($cmd);
+}
+
+# Clean up the input XML
+sub CleanUp
+{
+	my ($ref_xml) = @_;
+	
+	# Remove <?xml version="1.0" encoding="UTF-8"?>
+	$$ref_xml =~ s/<\?xml.+?>\n//g;
+	# Remove <!--XML document generated using OCR technology from ScanSoft, Inc.-->
+	$$ref_xml =~ s/<\!\-\-XML.+?>\n//g; 
+	# Add the root tag
+	$$ref_xml = "<root>" . "\n" . $$ref_xml . "\n" . "</root>";	
+}
+
+sub UntaintPath 
+{
+	my ($path) = @_;
+
+	if ($path =~ /^([-_:" \/\w\.%\p{C}\p{P}]+)$/ ) 
+	{
+		$path = $1;
+	} 
+	else 
+	{
+		die "Bad path \"$path\"\n";
+	}
+
+	return $path;
+}
+
+sub Untaint 
+{
+	my ($s) = @_;
+	if ($s =~ /^([\w \-\@\(\),\.\/>\p{C}\p{P}]+)$/) 
+	{
+		$s = $1;               # $data now untainted
+	} 
+	else 
+	{
+		die "Bad data in $s";  # log this somewhere
+	}
+	return $s;
+}
+
+sub Execute 
+{
+	my ($cmd) = @_;
+	print STDERR "Executing: $cmd\n";
+	system($cmd);
+}
+
+
+

data/parscit/lib/CSXUtil/SafeText.pm

@@ -0,0 +1,130 @@
+package CSXUtil::SafeText;
+
+#######################################################
+##  Methods for stripping bad (XML unsafe) characters
+##  from strings and performing basic HTML entity
+##  translations.  Also contains a utility (stripArtifacts)
+##  for getting rid of crazy control characters and
+##  other things that probably aren't proper text.
+##
+##  Isaac Councill, 12/06/06
+##
+#######################################################
+
+use utf8;
+use strict;
+require Exporter;
+
+our ($VERSION, @ISA, @EXPORT, @EXPORT_OK, %EXPORT_TAGS);
+
+$VERSION	= 1.00;
+@ISA		= qw(Exporter);
+@EXPORT_OK	= qw(@badChars %htmlSpecialChars %htmlCharEntities &stripBadChars &encodeHTMLSpecialChars &decodeHTMLSpecialChars &cleanXML &cleanAll &stripArtifacts);
+
+##
+#######################################################
+##
+##  Sharable encoding data.
+##
+##  Hex codes for characters that should never be put into
+##  XML - or else parsers will barf.
+#######################################################
+our @badChars = qw(	\x00 \x01 \x02 \x03 \x04 \x05 \x06 \x07
+                  	\x08 \x0B \x0C \x0E \x0F \x10 \x11 \x12
+                  	\x13 \x14 \x15 \x16 \x17 \x18 \x19 \x1A
+                  	\x1B \x1C \x1D \x1E \x1F \x7F	);
+
+###
+##  Subset of HTML characters that could be problematic
+##  for XML.  This is not a complete list of HTML
+##  special characters, but more mappings can be added
+##  as needed.
+###
+our %htmlSpecialCharEncodings = (	"&"  => "&",
+                                	">"  => ">",
+                                	"<"  => "&lt;",
+                                	"\"" => "&quot;",
+				 					"\'" => "&apos;" # Added by Thang (v090625)
+								);
+
+##  The reverse map.
+our %htmlSpecialCharDecodings;
+
+foreach my $key (keys %htmlSpecialCharEncodings) 
+{
+	my $val = $htmlSpecialCharEncodings{$key};
+   	$htmlSpecialCharDecodings{$val} = $key;
+}
+
+
+##
+#######################################################
+##
+##  Subroutines
+##
+
+##  Delete all occurences of bad characters in text,
+##  returns a new string that is clean.
+sub stripBadChars 
+{
+	my $rtext = shift;
+	foreach my $char (@badChars) { $$rtext =~ s/$char//g; }
+}
+
+
+##  Encodes special characters into HTML equivalents
+##  and returns the encoded string.
+sub encodeHTMLSpecialChars 
+{
+	my $rtext = shift;
+   	foreach my $char (keys %htmlSpecialCharEncodings) 
+	{
+		my $code = $htmlSpecialCharEncodings{$char};
+       	$$rtext =~ s/$char/$code/g;
+	}
+}
+
+
+##  Decodes a HTML entities in the supplied string
+##  into non-HTML character equivalents and returns
+##  the decoded string.
+sub decodeHTMLSpecialChars 
+{
+	my $rtext = shift;
+	foreach my $code (keys %htmlSpecialCharDecodings) 
+	{
+		my $char = $htmlSpecialCharDecodings{$code};
+       	$$rtext =~ s/$code/$char/g;
+   	}
+}
+
+
+##  Strip out any characters that don't look like they
+##  belong in a proper, readable text string.
+sub stripArtifacts 
+{
+   	my $rtext = shift;
+	$$rtext =~ s/[^\p{IsAlnum}\p{IsPunct}\p{IsSpace}\p{IsS}]//g;
+}
+
+
+##  Convenience routine for executing both XML safety
+##  routines in a single call.
+sub cleanXML 
+{
+   	my $rtext = shift;
+   	stripBadChars($rtext);
+   	encodeHTMLSpecialChars($rtext);
+}
+
+
+##  Clean for XML and also strip out strange characters.
+sub cleanAll 
+{
+   	my $rtext = shift;
+   	stripBadChars($rtext);
+   	stripArtifacts($rtext);
+   	encodeHTMLSpecialChars($rtext);
+}
+
+1;

data/parscit/lib/Omni/Config.pm

@@ -0,0 +1,93 @@
+package Omni::Config;
+
+# Global
+# Names of the classes
+$ALG_NAME		= "Omni";
+# Version
+$ALG_VERSION	= "110505";
+
+# All Omnipage XML tags 
+%omni_tag_list	=		(	'DOCUMENT'		=> 'document',
+		 	  				'PAGE'			=> 'page',
+							'COLUMN'		=> 'column',
+		   					'DESC'			=> 'description',
+			   				'SRC'			=> 'source',
+							'LANGUAGE' 		=> 'language',
+							'STYLE'			=> 'style',
+							'STYLE-TABLE'	=> 'styleTable',
+							'THEO-PAGE'		=> 'theoreticalPage',
+							'BODY'			=> 'body',
+							'SECTION'		=> 'section',
+							'COL'			=> 'column',
+							'PARA'			=> 'para',
+							'LINE'			=> 'ln',
+							'WORD'			=> 'wd',
+							'SPACE'			=> 'space',
+							'RUN'			=> 'run',
+							'BULLET'		=> 'bullet',
+							'TABLE'			=> 'table',
+							'GRID'			=> 'gridTable',
+							'GRID-COL'		=> 'gridCol',
+							'GRID-ROW'		=> 'gridRow',
+							'CELL'			=> 'cell',
+							'BOTTOM-CELL'	=> 'bottomBorder',
+							'TOP-CELL'		=> 'topBorder',
+							'LEFT-CELL'		=> 'leftBorder',
+							'RIGHT-CELL'	=> 'rightBorder',
+							'NEWLINE'		=> 'nl',
+							'TAB'			=> 'tab',
+							'DD'			=> 'dd',
+							'PICTURE'		=> 'picture',
+							'FRAME'			=> 'frame'
+						);
+$tag_list = \%omni_tag_list;
+
+# All Omnipage XML attributes 
+%omni_att_list	=		(	'ALIGN'			=> 'alignment',
+							'FONTFACE'		=> 'fontFace',
+							'FONTFAMILY'	=> 'fontFamily',
+							'FONTPITCH'		=> 'fontPitch',
+							'FONTSIZE'		=> 'fontSize',
+							'UNDERLINE'		=> 'underline',
+							'SPACING'		=> 'spacing',
+							'SCALE'			=> 'scale',
+							'BOTTOM'		=> 'b',
+							'TOP'			=> 't',
+							'LEFT'			=> 'l',
+							'RIGHT'			=> 'r',
+							'LANGUAGE'		=> 'language',
+							'SUSCRIPT'		=> 'subsuperscript',
+							'BASELINE'		=> 'baseline',
+							'BOLD'			=> 'bold',
+							'ITALIC'		=> 'italic',
+							'SPACEB'		=> 'spaceBefore',
+							# These attribute usually go with <dd> tag
+							'BOTTOMDIST'	=> 'bottomDistance',
+							'TOPDIST'		=> 'topDistance',
+							'LEFTDIST'		=> 'leftDistance',
+							'RIGHTDIST'		=> 'rightDistance',
+							# These attribute usually fo with <cell> tag
+							'GROWFROM'		=> 'gridRowFrom',
+							'GROWTO'		=> 'gridRowTill',
+							'GCOLFROM'		=> 'gridColFrom',
+							'GCOLTO'		=> 'gridColTill',
+							'VALIGN'		=> 'verticalAlignment',
+						);
+$att_list = \%omni_att_list;
+
+# All object type in Omni library
+%omni_obj_list	=		(	'OMNIDOC'		=> 'document',
+							'OMNIPAGE'		=> 'page',
+							'OMNICOL'		=> 'column',
+							'OMNIDD'		=> 'dd',
+							'OMNITABLE'		=> 'table',
+							'OMNIIMG'		=> 'image',
+							'OMNIPARA'		=> 'paragraph',
+							'OMNILINE'		=> 'line',
+							'OMNIRUN'		=> 'run',
+							'OMNIWORD'		=> 'word',
+							'OMNIFRAME'		=> 'frame',
+						);
+$obj_list = \%omni_obj_list;
+
+1;

data/parscit/lib/Omni/Omnicell.pm

@@ -0,0 +1,263 @@
+package Omni::Omnicell;
+
+# Configuration
+use strict;
+
+# Local libraries
+use Omni::Config;
+use Omni::Omnipara;
+
+# Extern libraries
+use XML::Twig;
+use XML::Parser;
+
+# Global variables
+my $tag_list = $Omni::Config::tag_list;
+my $att_list = $Omni::Config::att_list;
+my $obj_list = $Omni::Config::obj_list;
+
+# Temporary variables
+my $tmp_content 		= undef;
+my $tmp_alignment		= undef;
+my $tmp_grid_col_from	= undef;
+my $tmp_grid_col_to		= undef;
+my $tmp_grid_row_from	= undef;
+my $tmp_grid_row_to		= undef;
+my $tmp_vertical_align	= undef;
+
+# My observation is that <table> contains <gridTable> and <cell>
+# <gridTable> contain the base grid's coordinates
+# <cell> contain the cell's position based on <gridTable> coordinates
+# and various types of objects: <picture>, <para>, may be even <dd> but
+# I'm not quite sure about this
+my @tmp_objs	= ();
+
+###
+# A cell object in Omnipage xml: a cell is an essential member of <table> object 
+#
+# Do Hoang Nhat Huy, 14 Feb 2011
+###
+# Initialization
+sub new
+{
+	my ($class) = @_;
+
+	# Objs: a paragraph can have many cells
+	my @objs	= ();
+
+	# Class members
+	my $self = {	'_self'			=> $obj_list->{ 'OMNICELL' },
+					'_raw'			=> undef,
+					'_content'		=> undef,
+					'_alignment'	=> undef,
+					'_row_from'		=> undef,
+					'_row_to'		=> undef,
+					'_col_from'		=> undef,
+					'_col_to'		=> undef,
+					'_v_alignment'	=> undef,
+					'_objs'			=> \@objs	};
+
+	bless $self, $class;
+	return $self;
+}
+
+# 
+sub set_raw
+{
+	my ($self, $raw) = @_;
+
+	# Save the raw xml <cell> ... </cell>
+	$self->{ '_raw' }	= $raw;
+
+	# Parse the raw string
+	my $twig_roots		= { $tag_list->{ 'CELL' }	=> 1 };
+	my $twig_handlers 	= { $tag_list->{ 'CELL' }	=> \&parse};
+
+	# XML::Twig 
+	my $twig = new XML::Twig(	twig_roots 		=> $twig_roots,
+						 	 	twig_handlers	=> $twig_handlers,
+						 	 	pretty_print 	=> 'indented'	);
+
+	# Start the XML parsing
+	$twig->parse($raw);
+	$twig->purge;
+
+	# Copy information from temporary variables to class members
+	$self->{ '_alignment' }		= $tmp_alignment;
+	$self->{ '_row_from' }		= $tmp_grid_row_from;
+	$self->{ '_row_to' }		= $tmp_grid_row_to;
+	$self->{ '_col_from' }		= $tmp_grid_col_from;
+	$self->{ '_col_to' }		= $tmp_grid_col_to;
+	$self->{ '_v_alignment' }	= $tmp_vertical_align;
+
+	# Copy all objects 
+	@{$self->{ '_objs' } }	= @tmp_objs;
+	
+	# Copy content
+	$self->{ '_content' }	= $tmp_content;
+}
+
+sub get_raw
+{
+	my ($self) = @_;
+	return $self->{ '_raw' };
+}
+
+sub parse
+{
+	my ($twig, $node) = @_;
+
+	# At first, content is blank
+	$tmp_content 	= "";
+	# because there's no line
+	@tmp_objs		= ();
+
+	# Get <cell> node attributes
+	$tmp_alignment		= GetNodeAttr($node, $att_list->{ 'ALIGN' });
+
+	$tmp_grid_row_from	= GetNodeAttr($node, $att_list->{ 'GROWFROM' });
+	$tmp_grid_row_to	= GetNodeAttr($node, $att_list->{ 'GROWTO' });
+	$tmp_grid_col_from	= GetNodeAttr($node, $att_list->{ 'GCOLFROM' });
+	$tmp_grid_col_to	= GetNodeAttr($node, $att_list->{ 'GCOLTO' });
+	
+	# TODO: don't understand, attribute with value = 0 will be returned as undef by twig
+	$tmp_grid_row_from	= ($tmp_grid_row_from ne "")	? $tmp_grid_row_from	: 0;
+	$tmp_grid_row_to	= ($tmp_grid_row_to ne "")		? $tmp_grid_row_to		: 0;
+	$tmp_grid_col_from	= ($tmp_grid_col_from ne "")	? $tmp_grid_col_from 	: 0;
+	$tmp_grid_col_to	= ($tmp_grid_col_to ne "")		? $tmp_grid_col_to 		: 0;
+
+	$tmp_vertical_align	= GetNodeAttr($node, $att_list->{ 'VALIGN' });
+
+	# Check if there's any object: <para> and <picture object: <para> and <picture>
+	my $img_tag		= $tag_list->{ 'PICTURE' };
+	my $para_tag	= $tag_list->{ 'PARA' };
+
+	my $child = undef;
+	# Get the first child in the body text
+	$child = $node->first_child();
+
+	while (defined $child)
+	{
+		my $xpath = $child->path();
+
+		# if this child is a <para> tag
+		if ($xpath =~ m/\/$para_tag$/)
+		{
+			my $para = new Omni::Omnipara();
+
+			# Set raw content
+			$para->set_raw($child->sprint());
+
+			# Update paragraph list
+			push @tmp_objs, $para;
+
+			# Update content
+			$tmp_content = $tmp_content . $para->get_content() . "\n";
+		}
+		# if this child is a <picture> tag
+		elsif ($xpath =~ m/\/$img_tag$/)		
+		{
+			#my $img = new Omni::Omniimg();
+
+			# Set raw content
+			#$img->set_raw($child->sprint());
+
+			# Update paragraph list
+			#push @tmp_objs, $img;
+
+			# Update content
+			#$tmp_content = $tmp_content . $img->get_content() . "\n";
+		}
+
+		# Little brother
+		if ($child->is_last_child) 
+		{ 
+			last; 
+		}
+		else
+		{
+			$child = $child->next_sibling();
+		}
+	}
+}
+
+sub get_name
+{
+	my ($self) = @_;
+	return $self->{ '_self' };
+}
+
+sub get_objs_ref
+{
+	my ($self) = @_;
+	return $self->{ '_objs' };
+}
+
+sub get_content
+{
+	my ($self) = @_;
+	return $self->{ '_content' };
+}
+
+sub get_alignment
+{
+	my ($self) = @_;
+	return $self->{ '_alignment' };
+}
+
+sub get_grid_row_from
+{
+	my ($self) = @_;
+	return $self->{ '_row_from' };
+}
+
+sub get_grid_row_to
+{
+	my ($self) = @_;
+	return $self->{ '_row_to' };
+}
+
+sub get_grid_col_from
+{
+	my ($self) = @_;
+	return $self->{ '_col_from' };
+}
+
+sub get_grid_col_to
+{
+	my ($self) = @_;
+	return $self->{ '_col_to' };
+}
+
+sub get_vertical_alignment
+{
+	my ($self) = @_;
+	return $self->{ '_v_alignment' };
+}
+
+# Support functions
+sub GetNodeAttr 
+{
+	my ($node, $attr) = @_;
+	return ($node->att($attr) ? $node->att($attr) : "");
+}
+
+sub SetNodeAttr 
+{
+	my ($node, $attr, $value) = @_;
+	$node->set_att($attr, $value);
+}
+
+sub GetNodeText
+{
+	my ($node) = @_;
+	return $node->text;
+}
+
+sub SetNodeText
+{
+	my ($node, $value) = @_;
+	$node->set_text($value);
+}
+
+1;