biblicit 1.0 → 2.0.3

data/parscit/bin/sectLabel/processOmniXML.pl

@@ -0,0 +1,1427 @@
+#!/usr/bin/perl -wT
+
+# Author: Luong Minh Thang <luongmin@comp.nus.edu.sg>, generated at Tue, 02 Jun 2009 01:30:42
+# Modified from template by Min-Yen Kan <kanmy@comp.nus.edu.sg>
+
+require 5.0;
+use strict;
+use Getopt::Long;
+use HTML::Entities;
+
+# I do not know a better solution to find a lib path in -T mode.
+# So if you know a better solution, I'd be glad to hear.
+# See this http://www.perlmonks.org/?node_id=585299 for why I used the below code
+use FindBin;
+FindBin::again(); # to get correct path in case 2 scripts in different directories use FindBin
+
+my $path;
+BEGIN 
+{
+	if ($FindBin::Bin =~ /(.*)/) 
+	{
+		$path = $1;
+	}
+}
+use lib "$path/../../lib";
+
+use lib "/home/wing.nus/tools/languages/programming/perl-5.10.0/lib/5.10.0";
+use lib "/home/wing.nus/tools/languages/programming/perl-5.10.0/lib/site_perl/5.10.0";
+
+use SectLabel::PreProcess;
+
+# USER customizable section
+$0 =~ /([^\/]+)$/; my $progname = $1;
+my $outputVersion = "1.0";
+# END user customizable section
+
+sub License 
+{
+	print STDERR "# Copyright 2009 \251 by Luong Minh Thang\n";
+}
+
+# HELP sub-procedure
+sub Help 
+{
+	print STDERR "Process Omnipage XML output (concatenated results fromm all pages of a PDF file), and extract text lines together with other XML infos\n";
+	print STDERR "usage: $progname -h\t[invokes help]\n";
+	print STDERR "       $progname -in xmlFile -out outFile [-xmlFeature -decode -markup -para] [-tag tagFile -allowEmptyLine -log]\n";
+	print STDERR "Options:\n";
+	print STDERR "\t-q\tQuiet Mode (don't echo license)\n";
+	print STDERR "\t-xmlFeature: append XML feature together with text extracted\n";
+	print STDERR "\t-decode: decode HTML entities and then output, to avoid double entity encoding later\n";
+	print STDERR "\t-para: marking in the output each paragraph with # Para lineId numLines\n";
+	print STDERR "\t-markup: marking in the output detailed word-level info ### Page w h\\n## Para l t r b\\n# Line l t r b\\nword l t r b\n";
+	print STDERR "\t-tag tagFile: count XML tags/values for statistics purpose\n";
+}
+
+my $QUIET 			= 0;
+my $HELP 			= 0;
+my $outFile 		= undef;
+my $inFile 			= undef;
+
+my $isXmlFeature	= 0;
+my $isDecode 		= 0;
+
+my $isMarkup 		= 0;
+my $isParaDelimiter = 0;
+
+my $tagFile 		= "";
+my $isAllowEmpty 	= 0;
+my $isDebug 		= 1;
+
+$HELP = 1 unless GetOptions('in=s' 				=> \$inFile,
+			 				'out=s' 			=> \$outFile,
+			 				'decode' 			=> \$isDecode,
+			 				'xmlFeature' 		=> \$isXmlFeature,
+
+			 				'tag=s' 			=> \$tagFile,
+			 				'allowEmptyLine'	=> \$isAllowEmpty,
+			 				'markup' 			=> \$isMarkup,
+
+			 				'para' 				=> \$isParaDelimiter,
+			 				'log' 				=> \$isDebug,
+			 				'h' 				=> \$HELP,
+			 				'q' 				=> \$QUIET);
+
+if ($HELP || !defined $inFile || !defined $outFile) 
+{
+	Help();
+	exit(0);
+}
+
+if (!$QUIET) 
+{
+	License();
+}
+
+# Untaint
+$inFile		 = untaintPath($inFile);
+$outFile 	 = untaintPath($outFile);
+$tagFile 	 = untaintPath($tagFile);
+$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
+# End untaint
+
+# Mark page, para, line, word
+my %gPageHash	= ();
+
+# Mark paragraph
+my @gPara		= ();
+
+# XML features
+# locFeature
+my @gPosHash	= (); 
+my $gMinPos 	= 1000000; 
+my $gMaxPos 	= 0;
+
+# alignFeature
+my @gAlign 		= (); 
+
+# boldFeature
+my @gBold 		= ();
+
+# italicFeature
+my @gItalic 	= (); 
+
+# fontSizeFeature
+my %gFontSizeHash	= (); 
+my @gFontSize 		= ();
+
+# fontFaceFeature
+my %gFontFaceHash	= (); 
+my @gFontFace 		= ();
+
+# picFeature
+my @gPic			= (); 
+
+# tableFeature
+my @gTable			= ();
+
+# bulletFeature
+my @gBullet 		= (); 
+
+# spaceFeature
+#my %gSpaceHash		= (); 
+#my @gSpace			= ();
+# End XML features
+
+my %tags = ();
+
+if ($isDebug)
+{
+	print STDERR "\n# Processing file $inFile & output to $outFile\n";
+}
+
+my $markupOutput	= "";
+my $allText 		= processFile($inFile, $outFile, \%tags);
+
+# Find header part
+my @lines	 = split(/\n/, $allText);
+my $numLines = scalar(@lines);
+my ($headerLength, $bodyLength, $bodyStartId) =	SectLabel::PreProcess::FindHeaderText(\@lines, 0, $numLines);
+
+# Output
+if ($isMarkup)
+{
+	open(OF, ">:utf8", "$outFile") || die"#Can't open file \"$outFile\"\n";
+	print OF "$markupOutput";
+	close OF;
+} 
+else 
+{
+	output(\@lines, $outFile);
+}
+
+if ($tagFile ne "")
+{
+	printTagInfo(\%tags, $tagFile);
+}
+
+sub processFile 
+{
+	my ($inFile, $tags) = @_;
+	
+	if (!(-e $inFile)) { die "# $progname crash\t\tFile \"$inFile\" doesn't exist"; }
+	open (IF, "<:utf8", $inFile) || die "# $progname crash\t\tCan't open \"$inFile\"";
+	
+	my $isPic	= 0;
+	my $isPara	= 0;
+	my $isTable = 0;
+	my $isSpace = 0;
+
+	my $text	= undef; 
+	my $allText	= undef;
+
+	my $lineId = 0;
+
+	# Each line contains a header
+	while (<IF>) 
+	{
+		if (/^\#/) { next; } # skip comments
+
+		chomp;		
+		s/\cM$//; # remove ^M character at the end of the file if any
+
+		my $line = $_;
+		if($tagFile ne "")
+		{
+			processTagInfo($line, $tags);
+		}
+
+		# if ($line =~ /<\?xml version.+>/) { } ### Xml ###
+		# if ($line =~ /^<\/column>$/) { }		### Column ###
+		if ($isMarkup && $line =~ /<theoreticalPage (.*)\/>/ && $isMarkup)
+		{
+			$markupOutput .= "### Page $1\n";
+		}
+
+		### pic ###
+		if ($line =~ /^<dd (.*)>$/)
+		{
+			$isPic = 1;
+			if($isMarkup)
+			{
+				$markupOutput .= "### Dd $1\n";
+			}
+		}
+		elsif ($line =~ /^<\/dd>$/)
+		{
+			$isPic = 0;
+		}
+
+		### Table ###
+		if ($line =~ /^<table .*>$/)
+		{
+			$text .= $line."\n"; # we need the header
+			$isTable = 1;
+		}
+		elsif ($line =~ /^<\/table>$/)
+		{
+			my $tableText = processTable($text, $isPic);
+			$allText .= $tableText;
+
+			my @tmpLines = split(/\n/, $tableText);
+			$lineId += scalar(@tmpLines);
+			
+			$isTable = 0;
+			$text = "";
+		}
+		elsif($isTable)
+		{
+			$text .= $line."\n";
+			next;
+		}
+		### Paragraph ###
+		# Note: table processing should have higher priority than paragraph, i.e. the priority does matter
+		elsif ($line =~ /^<para (.*)>$/)
+		{
+			$text .= $line."\n"; # we need the header
+			$isPara = 1;
+
+			if($isMarkup)
+			{
+				$markupOutput .= "## Para $1\n";
+			}
+		}
+		elsif ($line =~ /^<\/para>$/)
+		{
+			my ($paraText, $l, $t, $r, $b);
+			($paraText, $l, $t, $r, $b, $isSpace) = processPara($text, 0, $isPic);
+			$allText .= $paraText;
+
+			my @tmpLines = split(/\n/, $paraText);
+			$lineId += scalar(@tmpLines);
+			$isPara = 0;
+			$text = "";
+		}
+		elsif($isPara)
+		{
+			$text .= $line."\n";
+			next;
+		}
+	}  
+	
+	close IF;
+	return $allText;
+}
+
+sub output 
+{
+	my ($lines, $outFile) = @_;
+
+	open(OF, ">:utf8", "$outFile") || die"#Can't open file \"$outFile\"\n";
+
+	####### Final output ############
+	# xml feature label
+	my %gFontSizeLabels = (); 
+	# my %gSpaceLabels = (); # yes, no
+
+	if($isXmlFeature)
+	{
+		getFontSizeLabels(\%gFontSizeHash, \%gFontSizeLabels);
+		# getSpaceLabels(\%gSpaceHash, \%gSpaceLabels);
+	}
+
+	my $id = -1;
+	my $output = "";
+	my $paraLineId = -1;
+	my $paraLineCount = 0;
+	foreach my $line (@{$lines}) 
+	{
+		$id++;
+
+		$line =~ s/\cM$//; # remove ^M character at the end of each line if any
+
+		# empty lines
+		if ($line =~ /^\s*$/)
+		{ 
+			if (!$isAllowEmpty)
+			{
+				next; 
+			} 
+			else 
+			{
+				if ($isDebug)
+				{
+	 				print STDERR "#! Line $id empty!\n";
+				}
+			}
+		} 
+
+		if ($gPara[$id] eq "yes")		
+		{
+			# mark para
+			if ($output ne "")
+			{       
+				if ($isParaDelimiter)
+				{
+	 				print OF "# Para $paraLineId $paraLineCount\n$output";
+	 				$paraLineCount = 0;
+				} 
+				else 
+				{
+	 				if($isDecode)
+					{
+		 				$output = decode_entities($output);
+	 				}
+	 				print OF $output;
+				}
+
+				$output = "";
+			}
+			$paraLineId = $id;
+		}
+		
+		$output .= $line;
+		$paraLineCount++;
+
+		## Output XML features
+		if ($isXmlFeature)
+		{
+			# loc feature
+			my $locFeature;
+
+			if($gPosHash[$id] != -1)
+			{
+				$locFeature = "xmlLoc_".int(($gPosHash[$id] - $gMinPos) * 8.0 / ($gMaxPos - $gMinPos + 1));
+			}
+			
+			# align feature
+			my $alignFeature = "xmlAlign_".$gAlign[$id];
+
+			# fontSize feature
+			my $fontSizeFeature;
+			if($gFontSize[$id] == -1)
+			{
+				$fontSizeFeature = "xmlFontSize_none";
+			} 
+			else 
+			{
+				$fontSizeFeature = "xmlFontSize_".$gFontSizeLabels{$gFontSize[$id]};
+			}
+
+			my $boldFeature = "xmlBold_".$gBold[$id]; # bold feature
+			my $italicFeature = "xmlItalic_".$gItalic[$id]; # italic feature
+			my $picFeature = "xmlPic_".$gPic[$id]; # pic feature
+			my $tableFeature = "xmlTable_".$gTable[$id]; # table feature
+			my $bulletFeature = "xmlBullet_".$gBullet[$id]; # bullet feature
+
+			# space feature
+			#my $spaceFeature;
+			#if($gSpace[$id] eq "none")
+			#{
+			#	$spaceFeature = "xmlSpace_none";
+			#} 
+			#else 
+			#{
+			#	$spaceFeature = "xmlSpace_".$gSpaceLabels{$gSpace[$id]};
+			#}
+
+			# differential features
+			my ($alignDiff, $fontSizeDiff, $fontFaceDiff, $fontSFDiff, $fontSFBIDiff, $fontSFBIADiff, $paraDiff) = getDifferentialFeatures($id);
+
+			$output .= " |XML| $locFeature $boldFeature $italicFeature $fontSizeFeature $picFeature $tableFeature $bulletFeature $fontSFBIADiff $paraDiff\n"; 
+			# $alignFeature $alignDiff $fontSizeDiff $fontFaceDiff $fontSFDiff $fontSFBIDiff 
+		} 
+		else 
+		{
+			$output .= "\n";
+		}
+	}
+
+	# mark para
+	if ($output ne "")
+	{
+		if ($isParaDelimiter)
+		{
+			print OF "# Para $paraLineId $paraLineCount\n$output";
+			$paraLineCount = 0;
+		} 
+		else 
+		{
+			if ($isDecode)
+			{
+				$output = decode_entities($output);
+			}
+
+			print OF $output;
+		}
+		$output = ""
+	}
+	close OF;
+}
+
+sub getDifferentialFeatures 
+{
+	my ($id) = @_;
+
+	# alignChange feature
+	my $alignDiff = "bi_xmlA_";
+	if ($id == 0)
+	{
+		$alignDiff .= $gAlign[$id];
+	} 
+	elsif ($gAlign[$id] eq $gAlign[$id-1])
+	{
+		$alignDiff .= "continue";
+	} 
+	else 
+	{
+		$alignDiff .= $gAlign[$id];
+	}
+	
+	# fontFaceChange feature
+	my $fontFaceDiff = "bi_xmlF_";
+	if ($id == 0)
+	{
+		$fontFaceDiff .= "new";
+	} 
+	elsif ($gFontFace[$id] eq $gFontFace[$id-1])
+	{
+		$fontFaceDiff .= "continue";
+	} 
+	else 
+	{
+		$fontFaceDiff .= "new";
+	}
+
+	# fontSizeChange feature
+	my $fontSizeDiff = "bi_xmlS_";
+	if ($id == 0)
+	{
+		$fontSizeDiff .= "new";
+	} 
+	elsif ($gFontSize[$id] == $gFontSize[$id-1])
+	{
+		$fontSizeDiff .= "continue";
+	} 
+	else 
+	{
+		$fontSizeDiff .= "new";
+	}  
+	
+	# fontSFChange feature
+	my $fontSFDiff = "bi_xmlSF_";
+	if($id == 0){
+		$fontSFDiff .= "new";
+	} elsif($gFontSize[$id] == $gFontSize[$id-1] && $gFontFace[$id] eq $gFontFace[$id-1]){
+		$fontSFDiff .= "continue";
+	} else {
+		$fontSFDiff .= "new";
+	}
+	
+	# fontSFBIChange feature
+	my $fontSFBIDiff = "bi_xmlSFBI_";
+	if($id == 0){
+		$fontSFBIDiff .= "new";
+	} elsif($gFontSize[$id] == $gFontSize[$id-1] && $gFontFace[$id] eq $gFontFace[$id-1] && $gBold[$id] eq $gBold[$id-1] && $gItalic[$id] eq $gItalic[$id-1]){
+		$fontSFBIDiff .= "continue";
+	} else {
+		$fontSFBIDiff .= "new";
+	}
+	
+	# fontSFBIAChange feature
+	my $fontSFBIADiff = "bi_xmlSFBIA_";
+	if($id == 0){
+		$fontSFBIADiff .= "new";
+	} elsif($gFontSize[$id] == $gFontSize[$id-1] && $gFontFace[$id] eq $gFontFace[$id-1] && $gBold[$id] eq $gBold[$id-1] && $gItalic[$id] eq $gItalic[$id-1] && $gAlign[$id] eq $gAlign[$id-1]){
+		$fontSFBIADiff .= "continue";
+	} else {
+		$fontSFBIADiff .= "new";
+	}
+
+	# para change feature
+	my $paraDiff = "bi_xmlPara_";
+	if($id < $bodyStartId){ # header part, consider each line as a separate paragraph
+		$paraDiff .= "header";
+	} else {
+		if($gPara[$id] eq "yes"){
+			$paraDiff .= "new";
+		} else {
+			$paraDiff .= "continue";
+		}
+	}
+
+	return ($alignDiff, $fontSizeDiff, $fontFaceDiff, $fontSFDiff, $fontSFBIDiff, $fontSFBIADiff, $paraDiff);
+}
+
+sub getFontSizeLabels {
+	my ($gFontSizeHash, $gFontSizeLabels) = @_;
+
+	if($isDebug){ print STDERR "# Map fonts\n"; }
+	my @sortedFonts = sort { $gFontSizeHash->{$b} <=> $gFontSizeHash->{$a} } keys %{$gFontSizeHash}; # sort by values, obtain keys
+	
+	my $commonSize = $sortedFonts[0];
+	@sortedFonts = sort { $a <=> $b } keys %{$gFontSizeHash}; # sort by keys, obtain keys
+	my $commonIndex = 0; # index of common font size
+	foreach(@sortedFonts){
+		if($commonSize == $_) { # found
+			last;
+		}
+		$commonIndex++;
+	}
+	
+	# small fonts
+	for(my $i = 0; $i<$commonIndex; $i++){ # smallIndex $largeIndex 
+		$gFontSizeLabels->{$sortedFonts[$i]} = "smaller";
+
+		if($isDebug){
+			print STDERR "$sortedFonts[$i] --> $gFontSizeLabels->{$sortedFonts[$i]}, freq = $gFontSizeHash->{$sortedFonts[$i]}\n";
+		}
+	}
+
+	# common fonts
+	$gFontSizeLabels->{$commonSize} = "common";
+	if($isDebug){
+		print STDERR "$sortedFonts[$commonIndex] --> $gFontSizeLabels->{$sortedFonts[$commonIndex]}, freq = $gFontSizeHash->{$sortedFonts[$commonIndex]}\n";
+	}
+
+	# large fonts
+	for(my $i = ($commonIndex+1); $i<scalar(@sortedFonts); $i++){ # ($largeIndex+1) (scalar(@sortedFonts)-1)
+		if((scalar(@sortedFonts)-$i) <= 3){
+			$gFontSizeLabels->{$sortedFonts[$i]} = "largest".($i+1-scalar(@sortedFonts));
+		} else {
+			$gFontSizeLabels->{$sortedFonts[$i]} = "larger";
+		}
+
+		if($isDebug){	  
+			print STDERR "$sortedFonts[$i] --> $gFontSizeLabels->{$sortedFonts[$i]}, freq = $gFontSizeHash->{$sortedFonts[$i]}\n";
+		}
+	}
+}
+
+sub getSpaceLabels {
+	my ($gSpaceHash, $gSpaceLabels) = @_;
+
+	if($isDebug){
+		print STDERR "\n# Map space\n";
+	}
+	my @sortedSpaces = sort { $gSpaceHash->{$b} <=> $gSpaceHash->{$a} } keys %{$gSpaceHash}; # sort by freqs, obtain space faces
+	
+	my $commonSpace = $sortedSpaces[0];
+	my $commonFreq = $gSpaceHash->{$commonSpace};
+	# find similar common freq with larger spaces
+	for(my $i = 0; $i<scalar(@sortedSpaces); $i++){ # 0 ($smallIndex-1)
+		my $freq = $gSpaceHash->{$sortedSpaces[$i]};
+		if($freq/$commonFreq > 0.8){
+			if($sortedSpaces[$i] > $commonSpace){
+	$commonSpace = $sortedSpaces[$i];
+			}
+		} else {
+			last;
+		}
+	}
+
+	for(my $i = 0; $i<scalar(@sortedSpaces); $i++){ # 0 ($smallIndex-1)
+		if($sortedSpaces[$i] > $commonSpace){
+			$gSpaceLabels->{$sortedSpaces[$i]} = "yes";
+		} else {
+			$gSpaceLabels->{$sortedSpaces[$i]} = "no";
+		}
+
+		if($isDebug){
+			print STDERR "$sortedSpaces[$i] --> $gSpaceLabels->{$sortedSpaces[$i]}, freq = $gSpaceHash->{$sortedSpaces[$i]}\n";
+		}
+	}
+}
+
+sub processTable 
+{
+	my ($inputText, $isPic) = @_;
+
+	my $isCell = 0; # for table cell object
+
+	my $allText = "";
+	my $text = ""; 
+
+	my @lines = split(/\n/, $inputText);
+	my %tablePos = (); # $tablePos{$cellText} = "$l-$t-$r-$bottom"
+	my %table = (); # $table{$row}->{$col} = \@paraTexts
+	my $rowFrom;   
+	my $colFrom;
+	my $rowTill;   
+	my $colTill;
+
+	# xml feature
+	my $align = "none"; 
+	my $pos = -1;
+	foreach my $line (@lines) 
+	{
+		if ($line =~ /^<table (.+?)>$/)
+		{
+			my $attr = $1;
+			
+			if($isMarkup) {	$markupOutput .= "### Table $attr\n"; }
+
+			# Fix: wrong regex sequence, huydhn
+			#if ($attr =~ /^.*l=\"(\d+)\" t=\"(\d+)\" r=\"(\d+)\" b=\"(\d+)\".*alignment=\"(.+?)\".*$/)
+			#{
+			#	my ($l, $t, $r, $bottom) = ($1, $2, $3, $4);
+			#	$align = $5;
+
+			#	# pos feature
+			#	$pos = ($t+$bottom)/2.0;
+				
+			#	if($pos < $gMinPos) { $gMinPos = $pos; }
+			#	if($pos > $gMaxPos) { $gMaxPos = $pos; }
+			#} 
+			#else 
+			#{
+			#	print STDERR "# no table alignment or location \"$line\"\n";
+			#	$align = "";
+			#}
+
+			my ($l, $t, $r, $bottom) = undef;
+			if ($attr =~ /^.*l=\"(\d+)\".*$/) { $l = $1; }
+			if ($attr =~ /^.*t=\"(\d+)\".*$/) { $t = $1; }
+			if ($attr =~ /^.*r=\"(\d+)\".*$/) { $r = $1; }
+			if ($attr =~ /^.*b=\"(\d+)\".*$/) { $bottom = $1; }
+
+			if ($t && $bottom)
+			{
+				# pos feature
+				$pos = ($t + $bottom) / 2.0;
+				
+				if($pos < $gMinPos) { $gMinPos = $pos; }
+				if($pos > $gMaxPos) { $gMaxPos = $pos; }
+			}
+			else
+			{
+				die "# Undefined table location \"$line\"\n";
+			}
+
+			if ($attr =~ /^.*alignment=\"(\d+)\".*$/) 
+			{ 
+				$align = $1; 
+			} 
+			else 
+			{
+				print STDERR "# no table alignment \"$line\"\n";
+				$align = "";
+			}
+			# End.
+		}
+		elsif ($line =~ /^<cell .*gridColFrom=\"(\d+)\" gridColTill=\"(\d+)\" gridRowFrom=\"(\d+)\" gridRowTill=\"(\d+)\".*>$/) # new cell
+		{ 
+			$colFrom = $1;
+			$colTill = $2;
+			$rowFrom = $3;
+			$rowTill = $4;
+			#print STDERR "$rowFrom $rowTill $colFrom $colTill\n";
+			$isCell = 1;
+		}
+		elsif ($line =~ /^<\/cell>$/) # end cell
+		{ 
+			my @paraTexts = ();
+			processCell($text, \@paraTexts, \%tablePos, $isPic);
+			
+			for(my $i = $rowFrom; $i<=$rowTill; $i++)
+			{
+				for(my $j = $colFrom; $j<=$colTill; $j++)
+				{
+					if(!$table{$i}) { $table{$i} = (); }
+					if(!$table{$i}->{$j}) {	$table{$i}->{$j} = (); }
+
+	 				if($i == $rowFrom && $j == $colFrom)
+					{
+		 				push(@{$table{$i}->{$j}}, @paraTexts);
+						if(scalar(@paraTexts) > 1) { last; }
+	 				} 
+					else 
+					{
+		 				push(@{$table{$i}->{$j}}, ""); #add stub "" for spanning rows or cols
+	 				}
+				}
+			}
+				
+			$isCell = 0;
+			$text = "";
+		}    
+		elsif($isCell)
+		{
+			$text .= $line."\n";
+			next;
+		}
+	}
+
+	# note: such a complicated code is because in the normal node, Omnipage doesn't seem to strictly print column by column given a row is fixed.
+	# E.g if col1: paraText1, col2: paraText21\n$paraText22, and col3: paraText31\n$paraText32
+	# It will print  paraText1\tparaText21\tparaText31\n\t$paraText22\t$paraText32
+	my @sortedRows = sort {$a <=> $b} keys %table;
+	my $isFirstLinePara = 1;
+	foreach my $row (@sortedRows)
+	{
+		my %tableR = %{$table{$row}};
+		my @sortedCols = sort {$a <=> $b} keys %tableR;
+		while(1)
+		{
+			my $isStop = 1;
+			my $rowText = "";
+
+			foreach my $col (@sortedCols)
+			{
+				# there's still some thing to process
+				if(scalar(@{$tableR{$col}}) > 0)
+				{ 
+	 				$isStop = 0;
+	 				$rowText .= shift(@{$tableR{$col}});
+				}
+				$rowText .= "\t";
+			}
+
+			if ((!$isAllowEmpty && $rowText =~ /^\s*$/) || ($isAllowEmpty && $rowText eq ""))
+			{
+				$isStop = 1;
+			}
+
+			if($isStop) 
+			{
+				last;
+			} 
+			else 
+			{
+				$rowText =~ s/\t$/\n/;
+				$allText .= $rowText;
+				# print STDERR "$rowText";
+				
+				# para
+				if($isFirstLinePara)
+				{
+	 				push(@gPara, "yes");
+	 				$isFirstLinePara = 0;
+				} 
+				else 
+				{
+	 				push(@gPara, "no");
+				}
+
+				if($isXmlFeature)
+				{
+	 				# table feature
+	 				push(@gTable, "yes");
+
+	 				# pic feature
+	 				if($isPic)
+					{
+		 				push(@gPic, "yes");
+	 				} 
+					else 
+					{
+		 				push(@gPic, "no");
+	 				}
+
+	 				push(@gPosHash, $pos); # update xml pos value
+	 				push(@gAlign, $align); # update xml alignment value
+
+	 				### Not assign value ###
+					push(@gFontSize, -1); # fontSize feature	  
+					push(@gFontFace, "none"); # fontFace feature	  
+	 				push(@gBold, "no"); # bold feature	  
+	 				push(@gItalic, "no"); # italic feature
+	 				push(@gBullet, "no"); # bullet feature
+					# push(@gSpace, "none"); # space feature
+				} # end if xml feature
+			}
+		}
+	}
+
+	return $allText;
+}
+
+sub processCell 
+{
+	my ($inputText, $paraTexts, $tablePos, $isPic) = @_;
+
+	my $text = ""; 
+	my @lines = split(/\n/, $inputText);
+	my $isPara = 0;
+	my $flag = 0;
+	foreach my $line (@lines) 
+	{    
+		if ($line =~ /^<para (.*)>$/)
+		{
+			$text .= $line."\n"; # we need the header
+			$isPara = 1;
+
+			if($isMarkup)
+			{
+				$markupOutput .= "## ParaTable $1\n";
+			}
+		}
+		elsif ($line =~ /^<\/para>$/)
+		{
+			my ($paraText, $l, $t, $r, $b) = processPara($text, 1, $isPic);
+			my @tokens = split(/\n/, $paraText);
+
+			foreach my $token (@tokens)
+			{
+				if($token ne "")
+				{
+	 				push(@{$paraTexts}, $token);
+	 				$flag = 1;	
+				}
+			}
+
+			if(!$tablePos->{$paraText})
+			{
+				$tablePos->{$paraText} = "$l-$t-$r-$b";
+			} 
+			else 
+			{
+				#print STDERR "#! Warning: in method processCell, encounter the same paraText $paraText\n";
+			}
+
+			$isPara = 0;
+			$text = "";
+		}
+		elsif ($isPara)
+		{
+			$text .= $line."\n";
+			next;
+		}
+	}
+	
+	# at least one value should be added for cell which is ""
+	if($flag == 0) 
+	{
+		push(@{$paraTexts}, "");
+	}
+}
+
+sub getAttrValue {
+	my ($attrText, $attr) = @_;
+
+	my $value = "none";
+	if($attrText =~ /^.*$attr=\"(.+?)\".*$/){
+		$value = $1;
+	}
+	
+	return $value;
+}
+
+sub checkFontAttr {
+	my ($attrText, $attr, $attrHash, $count) = @_;
+
+	if($attrText =~ /^.*$attr=\"(.+?)\".*$/){
+		my $attrValue = $1;
+		
+		$attrHash->{$attrValue} = $attrHash->{$attrValue} ? ($attrHash->{$attrValue}+$count) : $count;
+	}
+}
+
+sub processPara 
+{
+	my ($inputText, $isCell, $isPic) = @_;
+	
+	my $isSpace = 0;
+	my $isSpecialSpace = 0;
+	my $isTab = 0;
+	my $isBullet = 0;
+
+	my $isForcedEOF = "none";  # 3 signals for end of L: forcedEOF=\"true\" in attribute of <ln> or || <nl orig=\"true\"\/> || end of </para> without encountering any of the above signal in the para plus $isSpace = 0
+	# xml feature
+	my $align = "none"; 
+	my ($l, $t, $r, $bottom);
+	my %fontSizeHash = ();
+	my %fontFaceHash = ();
+	my @boldArray = ();
+	my @italicArray = ();
+	my $space = "none";
+
+	my $lnAttr; my $isLn = 0; my $lnBold = "none"; my $lnItalic = "none";
+	my $runAttr;  my $runText = ""; my $isRun = 0; my $runBold = "none"; my $runItalic = "none";
+	my $wdAttr; my $wdText = ""; my $isWd = 0;
+
+	my $wdIndex = 0; # word index in a line. When encountering </ln>, this parameter indicates the number of words in a line
+	my $lnBoldCount = 0;
+	my $lnItalicCount = 0;
+
+	my $allText = "";
+	my $text = ""; #invariant: when never enter a new line, $text will be copied into $allText, and $text is cleared
+
+	binmode(STDERR, ":utf8");
+
+	my $isFirstLinePara = 1;
+	my @lines = split(/\n/, $inputText);
+	for (my $i=0; $i<scalar(@lines); $i++)
+	{
+		my $line = $lines[$i];
+
+		## new para
+		if ($line =~ /^<para (.+?)>$/)
+		{
+			my $attr = $1;
+			$align = getAttrValue($attr, "alignment");
+			# $indent = getAttrValue($attr, "li");
+			$space = getAttrValue($attr, "spaceBefore");
+		}
+		## new ln
+		elsif ($line =~ /^<ln (.+)>$/)
+		{ 
+			$lnAttr = $1;
+			$isLn = 1;
+
+			if ($isMarkup)
+			{
+				$markupOutput .= "# Line $lnAttr\n";
+			}
+
+			# Fix: wrong regex sequence, huydhn
+			#if ($lnAttr =~ /^.*l=\"(\d+)\" t=\"(\d+)\" r=\"(\d+)\" b=\"(\d+)\".*$/)
+			#{
+			#	($l, $t, $r, $bottom) = ($1, $2, $3, $4);
+			#}
+
+			if ($lnAttr =~ /^.*l=\"(\d+)\".*$/)	{ $l = $1; } else {	$l = undef; }
+			if ($lnAttr =~ /^.*t=\"(\d+)\".*$/)	{ $t = $1; } else {	$t = undef; }
+			if ($lnAttr =~ /^.*r=\"(\d+)\".*$/)	{ $r = $1; } else {	$r = undef; }
+			if ($lnAttr =~ /^.*b=\"(\d+)\".*$/)	{ $bottom = $1; } else { $bottom = undef; }
+			# End.
+
+			$isForcedEOF = getAttrValue($lnAttr, "forcedEOF");
+			
+			# Bold & Italic
+			if($isXmlFeature)
+			{ 
+				$lnBold = getAttrValue($lnAttr, "bold");
+				$lnItalic = getAttrValue($lnAttr, "italic");
+			}
+		}
+		## new run
+		elsif ($line =~ /<run (.*)>$/)
+		{
+			$runAttr = $1;
+
+			$isSpace = 0;
+			$isTab = 0;
+			$isRun = 1;
+			
+			# new wd, that consists of many runs
+			if($line =~ /^<wd (.*?)>/)
+			{
+				$isWd = 1;
+				$wdAttr = $1;
+			}
+
+			# Bold & Italic
+			if ($isXmlFeature)
+			{ 
+				$runBold = getAttrValue($runAttr, "bold");
+				$runItalic = getAttrValue($runAttr, "italic");
+			}
+		}
+		## wd
+		elsif ($line =~ /^<wd (.+)?>(.+)<\/wd>$/)
+		{
+			$wdAttr = $1;
+			my $word = $2;
+			$isSpace = 0;
+			$isTab = 0;
+
+			if ($isMarkup)
+			{
+				$markupOutput .= "$word $wdAttr\n";
+			}
+
+			# FontSize & FontFace
+			if($isXmlFeature)
+			{ 
+				checkFontAttr($wdAttr, "fontSize", \%fontSizeHash, 1);
+				checkFontAttr($wdAttr, "fontFace", \%fontFaceHash, 1);
+			}
+			
+			# Bold & Italic
+			if($isXmlFeature)
+			{
+				my $wdBold = getAttrValue($wdAttr, "bold");
+				my $wdItalic = getAttrValue($wdAttr, "italic");
+
+				if($wdBold eq "true" || $runBold eq "true" || $lnBold eq "true")
+				{
+	 				$boldArray[$wdIndex] = 1;
+	 				$lnBoldCount++;
+				}
+		
+				if($wdItalic eq "true" || $runItalic eq "true" || $lnItalic eq "true")
+				{
+	 				$italicArray[$wdIndex] = 1;
+	 				$lnItalicCount++;
+				}
+			} # if($isXmlFeature)
+
+			## add text
+			$text .= "$word";
+
+			if($isRun) { $runText .= "$word "; }
+			$wdIndex++;
+		}
+		## end wd
+		elsif ($line =~ /^<\/wd>$/)
+		{
+			$isWd = 0;
+			
+			if($isMarkup)
+			{
+				$markupOutput .= "$wdText $wdAttr\n";
+				$wdAttr = "";
+			}
+		}
+		## end run
+		elsif ($line =~ /^(.*)<\/run>$/)
+		{ 
+			my $word = $1;
+
+			## add text
+			if($word ne "")			
+			{
+				# Bold & Italic
+				if($isXmlFeature)
+				{ 
+	 				if($runBold eq "true" || $lnBold eq "true")
+					{
+		 				$boldArray[$wdIndex] = 1;
+		 				$lnBoldCount++;
+	 				}
+
+			 		if($runItalic eq "true" || $lnItalic eq "true")
+					{
+		 				$italicArray[$wdIndex] = 1;
+		 				$lnItalicCount++;
+	 				}
+				}
+
+				# appear in the final result
+				if($isLn) { $text .= "$word"; }
+
+				# for internal record
+				if($isRun) { $runText .= "$word "; }	
+				if($isWd) { $wdText .= "$word"; }	
+
+				$wdIndex++;
+			}
+
+			# xml feature
+			# not a space, tab or new-line run
+			if($isXmlFeature && $runText ne "") 
+			{ 
+				my @words = split(/\s+/, $runText);
+				my $numWords = scalar(@words);
+				checkFontAttr($runAttr, "fontSize", \%fontSizeHash, $numWords);
+				checkFontAttr($runAttr, "fontFace", \%fontFaceHash, $numWords);
+			}
+			
+			# reset run
+			# <run> not enclosed within <ln>
+			if (!$isLn) { $wdIndex = 0;	}
+			$runText = "";
+			$isRun = 0;      
+			$isSpecialSpace = 0;
+
+			# Bold & Italic
+			if($isXmlFeature)
+			{
+				$runBold = "none";
+				$runItalic = "none";
+
+				# <run> not enclosed within <ln>
+				if (!$isLn)
+				{ 
+	 				$lnBoldCount = 0;
+	 				$lnItalicCount = 0;
+				}
+			}
+		}
+		## end ln
+		elsif ($line =~ /^<\/ln>$/)
+		{
+			if((!$isAllowEmpty && $text !~ /^\s*$/) || ($isAllowEmpty && $text ne ""))
+			{
+				# there's a forced EOL?
+				# not an emply line with space character
+				if($isForcedEOF eq "true" || (!$isCell && !$isSpecialSpace) )
+				{ 
+	 				$text .= "\n";
+					
+					# update allText
+	 				$allText .= $text;
+	 				$text = "";
+				}
+
+				my $numWords = $wdIndex;
+
+				if (!$isCell)
+				{
+	 				if ($isFirstLinePara)
+					{
+		 				push(@gPara, "yes");
+		 				$isFirstLinePara = 0;
+	 				} 
+					else 
+					{
+		 				push(@gPara, "no");
+	 				}
+				}
+
+				if ($isXmlFeature && $numWords >= 1)
+				{
+	 				# xml feature
+					# assumtion that: fontSize is either occur in <ln>, or within multiple <run> under <ln>, but not both
+					checkFontAttr($lnAttr, "fontSize", \%fontSizeHash, $numWords);
+					checkFontAttr($lnAttr, "fontFace", \%fontFaceHash, $numWords);
+				}
+		
+				if ($isXmlFeature && !$isCell && !$isSpecialSpace)
+				{
+	 				my $pos = ($t+$bottom)/2.0;
+					if ($pos < $gMinPos) { $gMinPos = $pos;	}
+					if ($pos > $gMaxPos) { $gMaxPos = $pos; }
+
+					push(@gPosHash, $pos); # pos feature
+	 				push(@gAlign, $align); # alignment feature
+	 				push(@gTable, "no"); # table feature
+
+	 				# pic feature
+	 				if($isPic)
+					{
+		 				push(@gPic, "yes");
+
+		 				### Not assign value ###
+		 				push(@gFontSize, -1); # bold feature	  
+		 				push(@gFontFace, "none"); # bold feature	  
+		 				push(@gBold, "no"); # bold feature	  
+		 				push(@gItalic, "no"); # italic feature
+		 				push(@gBullet, "no"); # bullet feature
+	 				} 
+					else 
+					{
+		 				push(@gPic, "no");
+
+		 				updateXMLFontFeature(\%fontSizeHash, \%fontFaceHash);
+		 				%fontSizeHash = (); %fontFaceHash = ();
+
+		 				updateXMLFeatures($lnBoldCount, $lnItalicCount, $numWords, $isBullet, $space);
+	 				} # end if pic
+				} # end if($isXmlFeature && !$isCell && !$isSpecialSpace)
+			}
+
+			## reset ln
+			$isLn = 0;      
+			$isForcedEOF = "none";
+			$isSpecialSpace = 0;
+			$wdIndex = 0;
+
+			# Bold & Italic
+			if ($isXmlFeature)
+			{ 
+				$lnBold = "none";
+				$lnItalic = "none";
+
+				$lnBoldCount = 0;
+				$lnItalicCount = 0;
+			}
+		} # end else </ln>
+		## nl newline signal
+		elsif ($line =~ /^<nl orig=\"true\"\/>$/)
+		{
+			if($isLn)
+			{
+				$isSpace = 0;
+			} 
+			else 
+			{
+				if($isDebug)
+				{
+	 				print STDERR "#!!! Warning: found <nl orig=\"true\"\/> while not in tag <ln>: $line\n";
+				}
+			}
+		}
+		## space
+		elsif ($line =~ /^<space\/>$/)
+		{
+			my $startTag = "";
+			my $endTag = "";
+			if($i>0 && $lines[$i-1] =~ /^<(.+?)\b.*/)
+			{
+				$startTag = $1;
+			}
+				
+			if($i < (scalar(@lines) -1) && $lines[$i+1] =~ /^<\/(.+)>/)
+			{
+				$endTag = $1;
+			}
+			
+			if($startTag eq $endTag && $startTag ne "")
+			{
+				# print STDERR "# Special space after \"$text\"\n";
+				$isSpecialSpace = 1;
+			}
+
+			## addText
+			$text .= " ";
+			$isSpace = 1;
+		}
+		## tab
+		elsif ($line =~ /^<tab .*\/>$/)
+		{
+			## add Text
+			$text .= "\t";
+			$isTab = 1;
+		}
+		## bullet
+		elsif ($line =~ /^<bullet .*>$/)
+		{
+			$isBullet = 1;
+		}
+	}
+
+	$allText .= $text;
+	return ($allText, $l, $t, $r, $bottom, $isSpace);
+}
+
+sub updateXMLFontFeature {
+	my ($fontSizeHash, $fontFaceHash) = @_;
+
+	# font size feature
+	if(scalar(keys %{$fontSizeHash}) == 0){
+		push(@gFontSize, -1);
+	} else {
+		my @sortedFonts = sort { $fontSizeHash->{$b} <=> $fontSizeHash->{$a} } keys %{$fontSizeHash};
+		
+		my $fontSize = $sortedFonts[0];
+		push(@gFontSize, $fontSize);
+		
+		$gFontSizeHash{$fontSize} = $gFontSizeHash{$fontSize} ? ($gFontSizeHash{$fontSize}+1) : 1;
+	}
+	
+	# font face feature
+	if(scalar(keys %{$fontFaceHash}) == 0){
+		push(@gFontFace, "none");
+	} else {
+		my @sortedFonts = sort { $fontFaceHash->{$b} <=> $fontFaceHash->{$a} } keys %{$fontFaceHash};
+		my $fontFace = $sortedFonts[0];
+		push(@gFontFace, $fontFace);
+		
+		$gFontFaceHash{$fontFace} = $gFontFaceHash{$fontFace} ? ($gFontFaceHash{$fontFace}+1) : 1;
+	}
+}
+
+sub updateXMLFeatures {
+	my ($lnBoldCount, $lnItalicCount, $numWords, $isBullet, $space) = @_;
+	# bold feature
+	my $boldFeature;
+	if ($lnBoldCount/$numWords >= 0.667){
+		$boldFeature = "yes";
+	} else {
+		$boldFeature = "no";
+	}
+	push(@gBold, $boldFeature);
+	
+	# italic feature
+	my $italicFeature;
+	if ($lnItalicCount/$numWords >= 0.667){
+		$italicFeature = "yes";
+	} else {
+		$italicFeature = "no";
+	}
+	push(@gItalic, $italicFeature);  
+	
+	# bullet feature
+	if($isBullet){
+		push(@gBullet, "yes");
+	} else {
+		push(@gBullet, "no");
+	}
+	
+	# space feature
+#  push(@gSpace, $space);
+}
+
+## Find the positions of header, body, and citation
+sub getStructureInfo {
+	my ($lines, $numLines) = @_;
+
+	my ($bodyLength, $citationLength, $bodyEndId) =
+		SectLabel::PreProcess::findCitationText($lines, 0, $numLines);
+	
+	my ($headerLength, $bodyStartId);
+	($headerLength, $bodyLength, $bodyStartId) =
+		SectLabel::PreProcess::findHeaderText($lines, 0, $bodyLength);
+	
+	# sanity check
+	my $totalLength = $headerLength + $bodyLength + $citationLength;
+	if($numLines != $totalLength){
+		print STDOUT "Die in getStructureInfo(): different num lines $numLines != $totalLength\n"; # to display in Web
+		die "Die in getStructureInfo(): different num lines $numLines != $totalLength\n";
+	}
+	return ($headerLength, $bodyLength, $citationLength, $bodyStartId, $bodyEndId);
+}
+
+## Count XML tags/values for statistics purpose
+sub processTagInfo {
+	my ($line, $tags) = @_;
+
+	my $tag;
+	my $attr;
+	if($line =~ /^<(.+?)\b(.*)/){
+		$tag = $1;
+		$attr = $2;
+		if(!$tags->{$tag}){
+			$tags->{$tag} = ();
+		}
+		if($attr =~ /^\s*(.+?)\s*\/?>/){
+			$attr = $1;
+		}
+		
+		my @tokens = split(/\s+/, $attr);
+		foreach my $token (@tokens){
+			if($token =~ /^(.+)=(.+)$/){
+	my $attrName = $1;
+	my $value = $2;
+	if(!$tags->{$tag}->{$attrName}){
+	 $tags->{$tag}->{$attrName} = ();
+	}
+	if(!$tags->{$tag}->{$attrName}->{$value}){
+	 $tags->{$tag}->{$attrName}->{$value} = 0;
+	}
+	$tags->{$tag}->{$attrName}->{$value}++;
+			}
+		}
+	}
+}
+
+## Print tag info to file
+sub printTagInfo {
+	my ($tags, $tagFile) = @_;
+
+	open(TAG, ">:utf8", "$tagFile") || die"#Can't open file \"$tagFile\"\n";
+	my @sortedTags = sort {$a cmp $b} keys %{$tags};
+	foreach(@sortedTags){
+		my @attrs = sort {$a cmp $b} keys %{$tags->{$_}};
+		print TAG "# Tag = $_\n";
+		foreach my $attr (@attrs) {
+			print TAG "$attr:";
+			my @values = sort {$a cmp $b} keys %{$tags->{$_}->{$attr}};
+			foreach my $value (@values){
+	print TAG " $value-$tags->{$_}->{$attr}->{$value}";
+			}
+			print TAG "\n";
+		}
+	}
+	close TAG;
+}
+
+sub untaintPath {
+	my ($path) = @_;
+
+	if ( $path =~ /^([-_\/\w\.]*)$/ ) {
+		$path = $1;
+	} else {
+		die "Bad path \"$path\"\n";
+	}
+
+	return $path;
+}
+
+sub untaint {
+	my ($s) = @_;
+	if ($s =~ /^([\w \-\@\(\),\.\/]+)$/) {
+		$s = $1;               # $data now untainted
+	} else {
+		die "Bad data in $s";  # log this somewhere
+	}
+	return $s;
+}
+
+sub execute {
+	my ($cmd) = @_;
+	if($isDebug){
+		print STDERR "Executing: $cmd\n";
+	}
+	$cmd = untaint($cmd);
+	system($cmd);
+}
+
+sub newTmpFile {
+	my $tmpFile = `date '+%Y%m%d-%H%M%S-$$'`;
+	chomp($tmpFile);
+	return $tmpFile;
+}