biblicit 2.0.3 → 2.0.4

data/parscit/bin/sectLabel/processOmniXMLv3.pl

@@ -1,964 +0,0 @@
-#!/usr/bin/perl
-
-# Author: Do Hoang Nhat Huy <huydo@comp.nus.edu.sg>
-# Modified from template by Min-Yen Kan <kanmy@comp.nus.edu.sg>
-
-require 5.0;
-use strict;
-
-# Dependencies
-use FindBin;
-use Getopt::Long;
-use HTML::Entities;
-
-# I do not know a better solution to find a lib path in -T mode.
-# So if you know a better solution, I'd be glad to hear.
-# See this http://www.perlmonks.org/?node_id=585299 for why I used the below code
-
-# To get correct path in case 2 scripts in different directories use FindBin
-FindBin::again(); 
-my $path = undef;
-BEGIN 
-{
-	if ($FindBin::Bin =~ /(.*)/) { $path = $1; }
-}
-use lib "$path/../../lib";
-
-use lib "/home/wing.nus/tools/languages/programming/perl-5.10.0/lib/5.10.0";
-use lib "/home/wing.nus/tools/languages/programming/perl-5.10.0/lib/site_perl/5.10.0";
-
-# Local libraries
-use Omni::Config;
-use Omni::Omnidoc;
-use SectLabel::PreProcess;
-
-# Omnilib configuration: object name
-my $obj_list = $Omni::Config::obj_list;
-
-### USER customizable section
-$0 =~ /([^\/]+)$/; my $progname = $1;
-my $version = "1.0";
-### END user customizable section
-
-sub License 
-{
-	print STDERR "# Copyright 2011 \251 by Do Hoang Nhat Huy\n";
-}
-
-sub Help 
-{
-	print STDERR "Process Omnipage XML output (concatenated results fromm all pages of a PDF file), and extract text lines together with other XML infos\n";
-	print STDERR "usage: $progname -h\t[invokes help]\n";
-	print STDERR "       $progname -in xmlfile -out outfile [-decode] [-log]\n";
-	print STDERR "Options:\n";
-	print STDERR "\t-q      \tQuiet Mode (don't echo license)\n";
-	print STDERR "\t-decode \tDecode HTML entities and then output, to avoid double entity encoding later\n";
-}
-
-my $quite			= 0;
-my $help			= 0;
-my $out_file		= undef;
-my $in_file			= undef;
-my $is_decode		= 0;
-my $is_debug		= 0;
-my $address			= 1;
-
-$help = 1 unless GetOptions(	'in=s' 		=> \$in_file,
-								'out=s' 	=> \$out_file,
-								'decode' 	=> \$is_decode,
-								'log'		=> \$is_debug,
-								'h'			=> \$help,
-								'q'			=> \$quite	);
-
-if ($help || ! defined $in_file || ! defined $out_file) 
-{
-	Help();
-  	exit(0);
-}
-
-if (!$quite) 
-{
-	License();
-}
-
-### Untaint ###
-$in_file	 = UntaintPath($in_file);
-$out_file	 = UntaintPath($out_file);
-$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
-### End untaint ###
-
-# Mark page, para, line, word
-my %g_page_hash = ();
-
-# Mark paragraph
-my @g_para = ();
-
-# XML features
-# Location feature
-my @g_pos_hash	= (); 
-my $g_maxpos	= 0;
-my $g_minpos	= 1000000; 
-# Align feature
-my @g_align		= (); 
-# Bold feature
-my @g_bold		= ();
-# Italic feature
-my @g_italic	= ();
-# Pic feature
-my @g_pic		= (); 
-# Table feature
-my @g_table		= ();
-# Bullet feature
-my @g_bullet	= ();
-# Font size feature
-my %g_font_size_hash	= (); 
-my @g_font_size			= ();
-# Font face feature
-my %g_font_face_hash	= (); 
-my @g_font_face 		= ();
-
-# All lines
-my @lines		= ();
-# and their address
-my @lines_addr	= ();
-
-# BEGIN
-ProcessFile($in_file);
-# Find header part
-my $num_lines	= scalar(@lines);
-my ($header_length, $body_length, $body_start_id) = SectLabel::PreProcess::FindHeaderText(\@lines, 0, $num_lines);
-# Done
-Output(\@lines, $out_file);
-
-if ($address == 1)
-{
-	my $address_handle = undef;
-	# Save the line address for further use
-	open($address_handle, ">:utf8", $out_file . ".address") || die"#Can't open file \"$out_file.address\"\n";
-	foreach my $addr (@lines_addr)
-	{
-		print $address_handle $addr->{ 'L1' }, " ", $addr->{ 'L2' }, " ", $addr->{ 'L3' }, " ", $addr->{ 'L4' }, "\n";
-	}
-	# Done
-	close $address_handle;
-}
-# END
-
-sub ProcessFile 
-{
-	my ($in_file) = @_;
-
-	my $input_handle = undef;
-	if (! open($input_handle, "<:utf8", $in_file)) { die "Could not open xml file " . $in_file; }
-	my $xml = do { local $/; <$input_handle> };
-	close $input_handle;
-
-	###
-	# Huydhn
-	# NOTE: the omnipage xml is not well constructed (concatenated multiple xml files).
-	# This merged xml need to be fixed first before pass it to xml processing libraries, e.g. xml::twig
-	###
-	# Convert to Unix format
-	$xml =~ s/\r//g;
-	# Remove <?xml version="1.0" encoding="UTF-8"?>
-	$xml =~ s/<\?xml.+?>\n//g;
-	# Remove <!--XML document generated using OCR technology from ScanSoft, Inc.-->
-	$xml =~ s/<\!\-\-XML.+?>\n//g;
-	# Declaration and root
-	$xml = "<?xml version=\"1.0\"?>" . "\n" . "<root>" . "\n" . $xml . "\n" . "</root>";
-
-	# New document
-	my $doc = new Omni::Omnidoc();
-	$doc->set_raw($xml);
-
-	# Current position	
-	my %current		 = ();
-
-	# All pages in the document
-	my $pages = $doc->get_objs_ref();
-
-	# From page, To page
-	my $start_page	= 0;
-	my $end_page	= scalar(@{ $pages }) - 1;
-
-	# Image area flag
-	my $is_pic = 0;
-
-	# Tree traveling is 'not' fun. Seriously.
-	# This is like a dungeon seige.
-	for (my $x = $start_page; $x <= $end_page; $x++)	
-	{
-		# Current position
-		$current{ 'L1' } = $x;
-
-		# Column or dd
-		my $level_2	 =	$pages->[ $x ]->get_objs_ref();
-		my $start_l2 =	0;
-		my $end_l2	 =	scalar(@{ $level_2 }) - 1;
-
-		for (my $y = $start_l2; $y <= $end_l2; $y++)
-		{
-			# Thang's code
-			# Thang considers <dd> tag as image, I just follow that
-			if ($level_2->[ $y ]->get_name() eq $obj_list->{ 'OMNIDD' })
-			{
-				$is_pic = 1;	
-			}
-			else
-			{
-				$is_pic = 0;				
-			}
-			# End Thang's code
-
-			# Current position
-			$current{ 'L2' } = $y;
-
-			# Table or paragraph
-			my $level_3	 = 	$level_2->[ $y ]->get_objs_ref();
-			my $start_l3 =	0;
-			my $end_l3	 =	scalar(@{ $level_3 }) - 1;
-
-			for (my $z = $start_l3; $z <= $end_l3; $z++)
-			{
-				# Current position
-				$current{ 'L3' } = $z;
-
-				# Is a paragraph
-				if ($level_3->[ $z ]->get_name() eq $obj_list->{ 'OMNIPARA' })
-				{
-					# Thang's code
-					ProcessPara($level_3->[ $z ], $is_pic, \%current);
-					# End Thang's code
-				}
-				# or a table
-				elsif ($level_3->[ $z ]->get_name() eq $obj_list->{ 'OMNITABLE' })
-				{
-					# Thang's code
-					ProcessTable($level_3->[ $z ], $is_pic, \%current, 0);
-					# End Thangs's code
-				}
-				# or a frame
-				elsif ($level_3->[ $z ]->get_name() eq $obj_list->{ 'OMNIFRAME' })
-				{
-					# Frame contains multiple paragraph ?
-					ProcessFrame($level_3->[ $z ], $is_pic, \%current);
-				}
-			}
-		}
-	}
-}
-
-sub Output 
-{
-	my ($lines, $out_file) = @_;
-
-	my $output_handle = undef;
-	# This is the output
-	open($output_handle, ">:utf8", $out_file) || die"#Can't open file \"$out_file\"\n";
-
-  	# XML feature label
-	my %g_font_size_labels = (); 
-	GetFontSizeLabels(\%g_font_size_hash, \%g_font_size_labels);
-  	
-  	my $output			= "";
- 	my $para_line_id	= -1;
-  	my $para_line_count	= 0;
-
-	# This is the index of the line
-	my $id = 0;
-	# For each line in the whole document
-	foreach my $line (@{ $lines })
-	{
-		# Remove empty line 
-		$line =~ s/^\s+|\s+$//g;
-
-		# New paragraph
-    	if (($g_para[ $id ] eq "yes") && ($output ne ""))
-		{
-			if ($is_decode) { $output = decode_entities($output); }
-			# Write output to file
-			print $output_handle $output;
-			# Clean output for new paragraph
-			$output = "";
-      	}
-    
-    	$output .= $line;
-
-		my $loc_feature = undef;
-      	# XML location feature
-   		if ($g_pos_hash[ $id ] != (-1)) { $loc_feature = "xmlLoc_".int(($g_pos_hash[$id] - $g_minpos) * 8.0 / ($g_maxpos - $g_minpos + 1)); }
-
-	 	# Align feature
-   		my $align_feature = "xmlAlign_" . $g_align[ $id ];
-
-      	my $font_size_feature = undef;
-		# Font_size feature
-		if (($g_font_size[$id] eq "") || ($g_font_size[$id] == -1))
-		{
-			$font_size_feature = "xmlFontSize_none";
-		} 
-		else 
-		{
-			$font_size_feature = "xmlFontSize_" . $g_font_size_labels{ $g_font_size[ $id ] };
-		}
-
-		# Bold feature
-      	my $bold_feature	= "xmlBold_"	. $g_bold[ $id ];
-		# Italic feature
-		my $italic_feature	= "xmlItalic_"	. $g_italic[ $id ];
-		# Image feature
-   		my $pic_feature		= "xmlPic_"		. $g_pic[ $id ];
-		# Table feature
-   		my $table_feature	= "xmlTable_"	. $g_table[ $id ];
-		# Bullet feature
-		my $bullet_feature	= "xmlBullet_"	. $g_bullet[ $id ];
-		# Differential features
-		my ($align_diff, $font_size_diff, $font_face_diff, $font_sf_diff, $font_sfbi_diff, $font_sfbia_diff, $para_diff) = GetDifferentialFeatures($id);
-
-   		# Each line and its XML features
-		$output .= " |XML| $loc_feature $bold_feature $italic_feature $font_size_feature $pic_feature $table_feature $bullet_feature $font_sfbia_diff $para_diff" . "\n"; 
-
-		# Update line index
-    	$id++;
-  	}
-
-	# New paragraph
-	if ($output ne "")
-	{
-		if ($is_decode) { $output = decode_entities($output); }
-		# Write output to file
-		print $output_handle $output;
-		# Clean output for new paragraph
-		$output = "";
-  	}
- 
- 	# Done
-  	close $output_handle;
-}
-
-sub GetDifferentialFeatures 
-{
-	my ($id) = @_;
-
-  	my $align_diff = "bi_xmlA_";
-	# AlignChange feature
-	if ($id == 0)
-	{
-    	$align_diff .= $g_align[ $id ];
-  	} 
-	elsif ($g_align[ $id ] eq $g_align[ $id - 1 ])
-	{
-    	$align_diff .= "continue";
-  	} 
-	else 
-	{
-    	$align_diff .= $g_align[$id];
-  	}
-  
-	my $font_face_diff = "bi_xmlF_";
-  	# FontFaceChange feature
-  	if ($id == 0)
-	{
-    	$font_face_diff .= "new";
-  	} 
-	elsif ($g_font_face[ $id ] eq $g_font_face[ $id - 1 ])
-	{
-    	$font_face_diff .= "continue";
-  	} 
-	else 
-	{
-    	$font_face_diff .= "new";
-  	}
-
-  	my $font_size_diff = "bi_xmlS_";
-	# FontSizeChange feature
-	if ($id == 0)
-	{
-    	$font_size_diff .= "new";
-	} 
-	elsif ($g_font_size[ $id ] == $g_font_size[ $id - 1 ])
-	{
-    	$font_size_diff .= "continue";
-  	} 
-	else 
-	{
-    	$font_size_diff .= "new";
-  	}  
-  
-  	my $font_sf_diff = "bi_xmlSF_";
-  	# FontSFChange feature
-  	if ($id == 0)
-	{
-    	$font_sf_diff .= "new";
-	} 
-	elsif ($g_font_size[ $id ] == $g_font_size[ $id - 1 ] && $g_font_face[ $id ] eq $g_font_face[ $id - 1 ])
-	{
-    	$font_sf_diff .= "continue";
-  	} 
-	else 
-	{
-    	$font_sf_diff .= "new";
-  	}
-  
-  	my $font_sfbi_diff = "bi_xmlSFBI_";
-  	# FontSFBIChange feature
-	if ($id == 0)
-	{
-    	$font_sfbi_diff .= "new";
-  	} 
-	elsif ($g_font_size[ $id ] == $g_font_size[ $id - 1 ] && $g_font_face[ $id ] eq $g_font_face[ $id - 1 ] && $g_bold[ $id ] eq $g_bold[ $id - 1 ] && $g_italic[ $id ] eq $g_italic[ $id - 1 ])
-	{
-    	$font_sfbi_diff .= "continue";
-  	} 
-	else 
-	{
-    	$font_sfbi_diff .= "new";
-  	}
-  
-  	my $font_sfbia_diff = "bi_xmlSFBIA_";
-  	# FontSFBIAChange feature
-  	if ($id == 0)
-	{
-    	$font_sfbia_diff .= "new";
-  	} 
-	elsif ($g_font_size[ $id ] == $g_font_size[ $id - 1 ] && $g_font_face[ $id ] eq $g_font_face[ $id - 1 ] && $g_bold[ $id ] eq $g_bold[ $id - 1 ] && $g_italic[ $id ] eq $g_italic[$id - 1] && $g_align[ $id ] eq $g_align[ $id - 1 ])
-	{
-    	$font_sfbia_diff .= "continue";
-  	} 
-	else 
-	{
-    	$font_sfbia_diff .= "new";
-  	}
-
-  	# ParaChange feature
-  	my $para_diff = "bi_xmlPara_";
-	# Header part, consider each line as a separate paragraph
-  	if ($id < $body_start_id)
-	{ 
-    	$para_diff .= "header";
-  	} 
-	else 
-	{
-    	if($g_para[$id] eq "yes")
-		{
-      		$para_diff .= "new";
-    	} 
-		else 
-		{
-      		$para_diff .= "continue";
-    	}
-  	}
-
-  	return ($align_diff, $font_size_diff, $font_face_diff, $font_sf_diff, $font_sfbi_diff, $font_sfbia_diff, $para_diff);
-}
-
-sub GetFontSizeLabels 
-{
- 	my ($g_font_size_hash, $g_font_size_labels) = @_;
-
-	# Sort by value in desccending order
-  	my @sorted_fonts = sort { $g_font_size_hash->{ $b } <=> $g_font_size_hash->{ $a } } keys %{ $g_font_size_hash };
-	# and get the 
-	my $common_size = $sorted_fonts[ 0 ];
-
-	# Sort by key in ascending order
-	@sorted_fonts = sort { $a <=> $b } keys %{ $g_font_size_hash };
-
-	my $common_index = 0; 
-	# Index of common font size
-	foreach (@sorted_fonts)
-	{
-		# Found
-    	if ($common_size == $_) { last;	}
-    	$common_index++;
-  	}
-  
-  	# Small fonts
-  	for (my $i = 0; $i < $common_index; $i++)
-	{
-    	$g_font_size_labels->{ $sorted_fonts[ $i ] } = "smaller";
-  	}
-
-  	# Common fonts
-  	$g_font_size_labels->{ $common_size } = "common";
-
-  	# Large fonts
-  	for (my $i = ($common_index + 1); $i < scalar(@sorted_fonts); $i++)
-	{ 
-    	if ((scalar(@sorted_fonts) - $i) <= 3)
-		{
-      		$g_font_size_labels->{ $sorted_fonts[$i] } = "largest" . ($i + 1 - scalar(@sorted_fonts));
-    	} 
-		else 
-		{
-      		$g_font_size_labels->{ $sorted_fonts[$i] } = "larger";
-    	}
-  	}
-}
-
-sub ProcessFrame
-{
-	my ($omniframe, $is_pic, $line_addr) = @_;
-
-	# Line index in the whole frame
-	my $lindex	= 0;
-	# All paragraph or table in the frame
-	my $objs	= $omniframe->get_objs_ref();
-	# For each paragraph or table in the frame
-	for (my $i = 0; $i < scalar(@{ $objs }); $i++)
-	{
-		if ($objs->[ $i ]->get_name() eq $obj_list->{ 'OMNIPARA' })
-		{
-			# Paragraph attributes
-			my $align	= $objs->[ $i ]->get_alignment();
-			my $space	= $objs->[ $i ]->get_space_before();
-			# Line attributes
-  			my ($left, $top, $right, $bottom) = undef;
-			# Run attributes	
-			my $bold_count		= 0;
-  			my $italic_count	= 0;
-  			my %font_size_hash	= ();
-		  	my %font_face_hash	= ();
-
-			my $omnilines = $objs->[ $i ]->get_objs_ref();
-			# For each line in the paragraph	
-			for (my $t = 0; $t < scalar(@{ $omnilines }); $t++)
-			{
-				# Save the line
-				push @lines, $omnilines->[ $t ]->get_content();
-				# Save the line's address
-				$line_addr->{ 'L4' } = $lindex;
-				push @lines_addr, { %{ $line_addr } };
-				# Point to the next line in the whole frame
-				$lindex++;
-
-				# Line attributes
-				$left	= $omnilines->[ $t ]->get_left_pos();
-				$right	= $omnilines->[ $t ]->get_right_pos();
-				$top	= $omnilines->[ $t ]->get_top_pos();
-				$bottom	= $omnilines->[ $t ]->get_bottom_pos();
-	
-				# Runs
-				my $runs	= $omnilines->[ $t ]->get_objs_ref();
-				my $start_r	= 0;
-				my $end_r	= scalar(@{ $runs }) - 1;
-
-				# Total number of words in a line
-				my $words_count = 0;
-
-				for (my $u = $start_r; $u <= $end_r; $u++)
-				{			
-					# Thang's compatible code (instead of using get_objs_ref)
-					my $rcontent = undef;
-					# Get run content
-					$rcontent	 = $runs->[ $u ]->get_content();
-					# Trim
-					$rcontent	 =~ s/^\s+|\s+$//g;
-					# Split to words
-					my @words = split(/\s+/, $rcontent);	
-
-					# Update the number of words
-					$words_count += scalar(@words);
-
-					# XML format
-					my $font_size					= $runs->[ $u ]->get_font_size();
-					$font_size_hash{ $font_size }	= $font_size_hash{ $font_size } ? $font_size_hash{ $font_size } + scalar(@words) : scalar(@words); 
-					# XML format
-					my $font_face 					= $runs->[ $u ]->get_font_face();
-					$font_face_hash{ $font_face }	= $font_face_hash{ $font_face } ? $font_face_hash{ $font_face } + scalar(@words) : scalar(@words); 
-					# XML format
-					if ($runs->[ $u ]->get_bold() eq "true") { $bold_count += scalar(@words); } 
-					# XML format
-					if ($runs->[ $u ]->get_italic() eq "true") { $italic_count += scalar(@words); }
-				}
-			
-				# Line attributes - relative position in paragraph
-				if ($t == 0)
-				{
- 					push @g_para, "yes";
-				} 
-				else 
-				{
- 					push @g_para, "no";
-				}
-		
-				# Line attributes - line position
-				my $pos = ($top + $bottom) / 2.0;
-				# Compare to global min and max position
-				if ($pos < $g_minpos) { $g_minpos = $pos; }
-				if ($pos > $g_maxpos) { $g_maxpos = $pos; }
-				# Pos feature
-				push @g_pos_hash, $pos;
-				# Alignment feature
-		  		push @g_align, $align;
-				# Table feature
-				push @g_table, "no";
-	
-				if ($is_pic)
-				{
-		    		push @g_pic, "yes";
-   			 		# Not assign value if line is in image area
-    				push @g_bold, "no";		
-    				push @g_italic, "no";
-	    			push @g_bullet, "no";
-	    			push @g_font_size, -1; 		
-    				push @g_font_face, "none";
-	  			} 
-				else 
-				{
-    				push @g_pic, "no";
-					UpdateXMLFontFeature(\%font_size_hash, \%font_face_hash);			
-					UpdateXMLFeatures($bold_count, $italic_count, $words_count, $omnilines->[ $t ]->get_bullet(), $space);
-				}			
-		
-				# Reset hash
-				%font_size_hash = (); 
-				%font_face_hash = ();
-				# Reset
-				$bold_count		= 0;
-				$italic_count	= 0;
-			}
-		}
-		elsif ($objs->[ $i ]->get_name() eq $obj_list->{ 'OMNITABLE' })
-		{
-			$lindex = ProcessTable($objs->[ $i ], $is_pic, $line_addr, $lindex);
-		}
-	}
-}
-
-sub ProcessTable 
-{
-	my ($omnitable, $is_pic, $line_addr, $lindex) = @_;
-
-	# Table attributes
-	my ($left, $top, $right, $bottom) = undef;
-	$left		= $omnitable->get_left_pos();
-	$right		= $omnitable->get_right_pos();
-	$top		= $omnitable->get_top_pos();
-	$bottom		= $omnitable->get_bottom_pos();
-	# Table attributes
-	my $align	= $omnitable->get_alignment();
-
-	# Thang's code
-	my $pos = ($top + $bottom) / 2.0;
-	# Set new min and max position
-	if ($pos < $g_minpos) { $g_minpos = $pos; }
-	if ($pos > $g_maxpos) { $g_maxpos = $pos; }
-	# End Thangs's code
-
-	# All row in the table
-	my $rows	= $omnitable->get_row_content();
-	# For each row in the table
-	for (my $i = 0; $i < scalar(@{ $rows }); $i++)
-	{
-		my @row_lines = split(/\n/, $rows->[ $i ]);
-		# For each line in the row
-		for (my $j = 0; $j < scalar(@row_lines); $j++)
-		{
-			# Save the line
-			push @lines, $row_lines[ $j ];
-			# Save the line's address
-			$line_addr->{ 'L4' } = $lindex;
-			push @lines_addr, { %{ $line_addr } };
-			# Point to the next line in the whole table
-			$lindex++;
-
-			if (($j == 0) && ($i == 0))
-			{
-				push @g_para, "yes";
-			} 
-			else 
-			{
-	 			push @g_para, "no";
-			}
-
-			# Table feature
-			push @g_table, "yes";
-
-			# Pic feature
-			if ($is_pic)
-			{
- 				push @g_pic, "yes";
-			} 
-			else 
-			{
-				push @g_pic, "no";
-			}
-		
-			# Update xml pos value
-			push @g_pos_hash, $pos;
-			# Update xml alignment value
-	 		push @g_align, $align;
-		
-			# Fontsize feature	  
-			push @g_font_size, -1;
-			# Fontface feature	  
-			push @g_font_face, "none";
-			# Bold feature	  
-			push @g_bold, "no";	 				
- 			# Italic feature
-			push @g_italic, "no";
-			# Bullet feature
-			push @g_bullet, "no";
-		}
-	}
-
-	# Nonsense
-	return $lindex;
-}
-
-sub ProcessPara 
-{
-	my ($paragraph, $is_pic, $line_addr) = @_;
- 
- 	# Paragraph attributes
-	my $align	= $paragraph->get_alignment();
-	my $space	= $paragraph->get_space_before();
-	# Line attributes
-  	my ($left, $top, $right, $bottom) = undef;
-	# Run attributes	
-	my $bold_count		= 0;
-  	my $italic_count	= 0;
-  	my %font_size_hash	= ();
-  	my %font_face_hash	= ();
-
-	# Lines
-	my $omnilines	= $paragraph->get_objs_ref();
-	my $start_l		= 0;
-	my $end_l		= scalar(@{ $omnilines }) - 1;
-
-	# Lines
-	for (my $t = $start_l; $t <= $end_l; $t++)
-	{
-		# Skip blank line
-		my $lcontent = $omnilines->[ $t ]->get_content();
-		$lcontent	 =~ s/^\s+|\s+$//g;
-		# Skip blank line
-		if ($lcontent eq "") { next; }
-
-		# Save the line
-		push @lines, $omnilines->[ $t ]->get_content();
-		# Save the line's address
-		$line_addr->{ 'L4' } = $t;
-		push @lines_addr, { %{ $line_addr } };
-
-		# Line attributes
-		$left	= $omnilines->[ $t ]->get_left_pos();
-		$right	= $omnilines->[ $t ]->get_right_pos();
-		$top	= $omnilines->[ $t ]->get_top_pos();
-		$bottom	= $omnilines->[ $t ]->get_bottom_pos();
-
-		# Runs
-		my $runs	= $omnilines->[ $t ]->get_objs_ref();
-		my $start_r	= 0;
-		my $end_r	= scalar(@{ $runs }) - 1;
-
-		# Total number of words in a line
-		my $words_count = 0;
-
-		for (my $u = $start_r; $u <= $end_r; $u++)
-		{			
-			# Thang's compatible code (instead of using get_objs_ref)
-			my $rcontent = undef;
-			# Get run content
-			$rcontent	 = $runs->[ $u ]->get_content();
-			# Trim
-			$rcontent	 =~ s/^\s+|\s+$//g;
-			# Split to words
-			my @words = split(/\s+/, $rcontent);	
-
-			# Update the number of words
-			$words_count += scalar(@words);
-
-			# XML format
-			my $font_size					= $runs->[ $u ]->get_font_size();
-			$font_size_hash{ $font_size }	= $font_size_hash{ $font_size } ? $font_size_hash{ $font_size } + scalar(@words) : scalar(@words); 
-			# XML format
-			my $font_face 					= $runs->[ $u ]->get_font_face();
-			$font_face_hash{ $font_face }	= $font_face_hash{ $font_face } ? $font_face_hash{ $font_face } + scalar(@words) : scalar(@words); 
-			# XML format
-			if ($runs->[ $u ]->get_bold() eq "true") { $bold_count += scalar(@words); } 
-			# XML format
-			if ($runs->[ $u ]->get_italic() eq "true") { $italic_count += scalar(@words); }
-		}
-			
-		# Line attributes - relative position in paragraph
-		if ($t == $start_l)
-		{
- 			push @g_para, "yes";
-		} 
-		else 
-		{
- 			push @g_para, "no";
-		}
-		
-		# Line attributes - line position
-		my $pos = ($top + $bottom) / 2.0;
-		# Compare to global min and max position
-		if ($pos < $g_minpos) { $g_minpos = $pos; }
-		if ($pos > $g_maxpos) { $g_maxpos = $pos; }
-		# Pos feature
-		push @g_pos_hash, $pos;
-		# Alignment feature
-  		push @g_align, $align;
-		# Table feature
-		push @g_table, "no";
-
-		if ($is_pic)
-		{
-    		push @g_pic, "yes";
-   	 		# Not assign value if line is in image area
-    		push @g_bold, "no";		
-    		push @g_italic, "no";
-    		push @g_bullet, "no";
-    		push @g_font_size, -1; 		
-    		push @g_font_face, "none";
-  		} 
-		else 
-		{
-    		push @g_pic, "no";
-			UpdateXMLFontFeature(\%font_size_hash, \%font_face_hash);			
-			UpdateXMLFeatures($bold_count, $italic_count, $words_count, $omnilines->[ $t ]->get_bullet(), $space);
-		}		
-		
-		# Reset hash
-		%font_size_hash = (); 
-		%font_face_hash = ();
-		# Reset
-		$bold_count		= 0;
-		$italic_count	= 0;
-	}
-}
-
-sub UpdateXMLFontFeature 
-{
-	my ($font_size_hash, $font_face_hash) = @_;
-
-  	# Font size feature
-  	if (scalar(keys %{ $font_size_hash }) == 0)
-	{
-    	push @g_font_size, -1;
-  	} 
-	else 
-	{
-    	my @sorted_fonts = sort { $font_size_hash->{ $b } <=> $font_size_hash->{ $a } } keys %{ $font_size_hash };
-   
-   		my $font_size = undef;
-		# Iw two font sizes are equal in number, get the larger one
-   		if ((scalar(@sorted_fonts) != 1) && ($font_size_hash->{ $sorted_fonts[ 0 ] } == $font_size_hash->{ $sorted_fonts[ 1 ] }))
-		{
-			$font_size = ($sorted_fonts[ 0 ] > $sorted_fonts[ 1 ]) ? $sorted_fonts[ 0 ] : $sorted_fonts[ 1 ];
-		}
-		else
-		{
-    		$font_size = $sorted_fonts[ 0 ];
-		}
-
-		if ($font_size eq "") { $font_size = 0; }
-   
-		push @g_font_size, $font_size;
-    	$g_font_size_hash{ $font_size } = $g_font_size_hash{ $font_size } ? $g_font_size_hash{ $font_size } + 1 : 1;
-  	}
-  
-  	# Font face feature
-  	if (scalar(keys %{ $font_face_hash }) == 0)
-	{
-    	push @g_font_face, "none";
-  	} 
-	else 
-	{
-    	my @sorted_fonts = sort { $font_face_hash->{ $b } <=> $font_face_hash->{ $a } } keys %{ $font_face_hash };		
-
-    	my $font_face = $sorted_fonts[ 0 ];
-    	push @g_font_face, $font_face;
-		
-		$g_font_face_hash{ $font_face } = $g_font_face_hash{ $font_face } ? $g_font_face_hash{ $font_face } + 1 : 1;
-  	}
-}
-
-sub UpdateXMLFeatures 
-{
-	my ($bold_count, $italic_count, $words_count, $is_bullet, $space) = @_;
-
-	# Bold feature
-  	my $bold_feature = undef;
-  	if (($words_count != 0) && ($bold_count / $words_count >= 0.667))
-	{
-    	$bold_feature = "yes";
-  	} 
-	else 
-	{
-    	$bold_feature = "no";
-  	}
-  	push @g_bold, $bold_feature;
-  
-  	# Italic feature
-  	my $italic_feature = undef;
-  	if (($words_count != 0) && ($italic_count / $words_count >= 0.667))
-	{
-    	$italic_feature = "yes";
-  	} 
-	else 
-	{
-    	$italic_feature = "no";
-  	}
-  	push @g_italic, $italic_feature;
-  
-  	# Bullet feature
-  	if ((defined $is_bullet) && ($is_bullet eq "true"))
-	{
-    	push @g_bullet, "yes";
-  	} 
-	else 
-	{
-    	push @g_bullet, "no";
-  	}
-}
-
-sub UntaintPath 
-{
-	my ($path) = @_;
-
-  	if ( $path =~ /^([-_\/\w\.]*)$/ ) 
-	{
-    	$path = $1;
-  	} 
-	else 
-	{
-    	die "Bad path \"$path\"\n";
-  	}
-
-  	return $path;
-}
-
-sub Untaint 
-{
-	my ($s) = @_;
-  	if ($s =~ /^([\w \-\@\(\),\.\/]+)$/) 
-	{
-    	$s = $1;               # $data now untainted
-  	} 
-	else 
-	{
-    	die "Bad data in $s";  # log this somewhere
-  	}
-  	
-	return $s;
-}
-
-sub Execute 
-{
-	my ($cmd) = @_;
-  	$cmd = Untaint($cmd);
-	system($cmd);
-}
-
-sub NewTmpFile 
-{
-	my $tmp_file = `date '+%Y%m%d-%H%M%S-$$'`;
-  	chomp  $tmp_file;
-  	return $tmp_file;
-}
-
-
-