biblicit 1.0 → 2.0.3

data/parscit/lib/SectLabel/Tr2crfpp.pm

@@ -0,0 +1,1246 @@
+package SectLabel::Tr2crfpp;
+
+###
+# Created from templateAppl.pl version 3.4 by Min-Yen Kan <kanmy@comp.nus.edu.sg>.
+# Modified by Isaac Councill on 7/20/07: wrapped the code as a package for use by
+# an external controller.
+#
+# Copyright 2005 \251 by Min-Yen Kan (not sure what this means for IGC edits, but
+# what the hell -IGC)
+###
+
+use utf8;
+use strict 'vars';
+
+# Dependencies
+use FindBin;
+use Encode ();
+
+# Local libraries
+use SectLabel::Config;
+
+### USER customizable section
+my $crf_test	= $SectLabel::Config::crf_test;
+### END user customizable section
+
+my %dict		= ();
+my %func_word	= ();
+
+my %keywords	= (); 
+my %bigrams		= ();
+my %trigrams	= ();
+my %fourthgrams	= ();
+
+# list of tags trained in parsHed
+# those with value 0 do not have frequent keyword features
+my $all_tags = $SectLabel::Config::tags;
+
+my %config = (	'1token'	=> 0,
+	      		'2token'	=> 0,
+	      		'3token'	=> 0,
+	      		'4token'	=> 0,
+
+	      		# Token-level features
+	      		'parscit'		=> 0, # Use all Parscit original features
+	      		'parscit_char'	=> 0, # Parscit char features
+
+	      		'tokenCapital'	=> 0,
+	      		'tokenNumber'	=> 0,
+	      		'tokenName'		=> 0,
+	      		'tokenPunct'	=> 0,
+	      		'tokenKeyword'	=> 0,
+	      
+				'1gram'	=> 0,
+	      		'2gram'	=> 0,
+	      		'3gram'	=> 0,
+	      		'4gram'	=> 0,
+
+				'lineNum'		=> 0,
+	      		'linePunct'		=> 0,
+	      		'linePos'		=> 0,
+	      		'lineLength'	=> 0,
+	      		'lineCapital'	=> 0,
+
+	      		# Pos
+	      		'xmlLoc'	=> 0,
+	      		'xmlAlign'	=> 0,
+	      		'xmlIndent'	=> 0,
+	      
+	      		# Format
+	      		'xmlFontSize'	=> 0,
+	      		'xmlBold'		=> 0,
+	      		'xmlItalic'		=> 0,
+	      
+	      		# Object
+	      		'xmlPic'	=> 0,
+	      		'xmlTable'	=> 0,
+	      		'xmlBullet'	=> 0,
+
+	      		# Bigram differential features
+	      		'bi_xmlA' 		=> 0,
+	      		'bi_xmlS' 		=> 0,
+	      		'bi_xmlF'		=> 0,
+	      		'bi_xmlSF'		=> 0,
+	      		'bi_xmlSFBI'	=> 0,
+	      		'bi_xmlSFBIA'	=> 0,
+	      		'bi_xmlPara'	=> 0,
+
+	      		# Unused
+	      		'xmlSpace'	=> 0,
+			);
+
+my %tag_map = (	"lineLevel"	=> "UL",
+	      		"xml"		=> "UX",
+
+	      		"bi_xml"	=> "B", # Bigram
+	      		"1token"	=> "U1",
+	      		"2token"	=> "U2",
+	      		"3token"	=> "U3",
+	      		"4token"	=> "U4",
+
+	      		"1gram"	=> "U5",
+	      		"2gram"	=> "U6",
+	      		"3gram"	=> "U7",
+	      		"4gram"	=> "U8",
+	      
+	      		"capital"	=> "U9",
+	      		"number"	=> "UA0",
+	      		"punct"		=> "UA1",
+	      		"func"		=> "UA2",
+	      		"binary"	=> "UA3",
+			);
+
+binmode(STDERR, ":utf8");
+binmode(STDOUT, ":utf8");
+
+sub Initialize 
+{
+	my ($dict_file, $func_file, $config_file) = @_;
+
+  	ReadDict($dict_file);
+  	LoadListHash($func_file, \%func_word);
+
+  	if (defined $config_file && $config_file ne "")
+	{
+    	LoadConfigFile($config_file, \%config);
+  	} 
+	else 
+	{
+    	die "!defined $config_file || $config_file eq \"\"\n";
+  	}
+
+	# if ($kFile ne "")		 { ReadKeywordDict($kFile, \%keywords); }
+	# if ($biFile ne "")	 { ReadKeywordDict($biFile, \%bigrams); }
+	# if ($triFile ne "")	 { ReadKeywordDict($triFile, \%trigrams); }
+	# if ($fourthFile ne "") { ReadKeywordDict($fourthFile, \%fourthgrams); }
+}
+
+# Entry point called by sectLabel/tr2crfpp.pl
+sub Tr2crfpp 
+{
+	my ($infile, $outfile, $dict_file, $func_file, $config_file, $is_generate_template) = @_; #$kFile, $biFile, $triFile, $fourthFile
+
+	if(!defined $is_generate_template) { die "Die: Tr2crfpp::tr2crfpp - undefined is_generate_template\n"; }
+
+  	Initialize($dict_file, $func_file, $config_file);
+
+  	# File IOs
+  	open (IF, "<:utf8", $infile) || die "# crash\t\tCan't open \"$infile\"";
+  	
+	my @lines = <IF>;
+  	ProcessData(\@lines, $outfile, $is_generate_template);
+
+	close (IF);
+}
+
+# Entry point called by SectLabel::Controller
+sub ExtractTestFeatures 
+{
+	my ($text_lines, $filename, $dict_file, $func_file, $config_file, $is_debug) = @_;
+  	
+	my $tmpfile = BuildTmpFile($filename);
+  	Initialize($dict_file, $func_file, $config_file);
+
+	my $is_generate_template = 0;
+  	ProcessData($text_lines, $tmpfile, $is_generate_template, $is_debug);
+
+  	return $tmpfile;
+}
+
+sub ProcessData 
+{
+	my ($lines, $outfile, $is_generate_template, $is_debug) = @_;
+
+	open (OF, ">:utf8", $outfile) || die "# crash\t\tCan't open \"$outfile\"";
+
+  	my %count_map	= ();
+  	GetDocLineCounts($lines, \%count_map);
+  	my $num_docs		= scalar(keys %count_map);
+
+  	my $is_abstract	= 0;
+  	my $is_intro	= 0;
+  	my $doc_id		= 0;
+  	my $tag			= "noTag";
+	my $index		= -1;
+  	my $num_lines	= $count_map{$doc_id};
+
+	if ($is_debug) { print STDERR "numLines = $num_lines\n"; }
+
+	my $xml_feature = "";
+	
+	foreach my $line (@{$lines}) 
+	{
+    	chomp($line);
+    	$index++;
+    
+		# if ($line =~ /^\#/) { next; } # skip comments
+
+		# Blank lines, new documents
+    	if ($line =~ /^\s*$/) 
+		{
+      		print OF "\n";
+
+      		# Reset
+      		$is_abstract = 0;
+      		$is_intro	 = 0;
+      		$index		 = -1;
+
+			$doc_id++;
+      		$num_lines = $count_map{$doc_id};
+
+      		next; 
+    	} 
+		else 
+		{
+      		if ($line =~ /^(.+?) \|\|\| (.+)$/) 
+			{
+				$tag	= $1;
+				$line	= $2;
+				
+				if(!defined $all_tags->{$tag})
+				{
+					# print STDERR "#! Warning: tag \"$tag\" not defined - skip \"$line\"\n";
+	  				next;
+				}
+      		}
+
+      		if ($line =~ /^(.+) \|XML\| (.+?)$/) 
+			{
+				$line			= $1;
+				$xml_feature	= $2;
+      		}
+
+      		if ($line =~ /abstract/i)
+			{
+				$is_abstract = 1;
+      		} 
+			elsif ($line =~ /introduction/i)
+			{
+				$is_intro = 1;
+      		} 
+			else 
+			{
+				if($is_abstract == 1)
+				{
+	  				$is_abstract = 2;
+				}
+
+				if($is_intro == 1)
+				{
+	  				$is_intro = 2;
+				}
+      		}
+
+      		my @feats		= ();
+      		my @templates	= CRFFeature($line, $index, $num_lines, $is_abstract, $is_intro, $xml_feature, $tag, \@feats);
+
+			# Generate CRF features
+      		if ($is_generate_template)
+			{
+				$is_generate_template = 0; # Done generate template file
+				print STDOUT join("", @templates);
+      		}
+
+			# if($index == -1) 
+			# {
+			# 	last;
+			# }
+
+			print OF join (" ", @feats);
+      		print OF "\n";
+    	}
+  	}
+
+  	close (OF);
+}
+
+sub GetDocLineCounts 
+{
+	my ($lines, $count_map) = @_;
+
+  	my $count	= 0;
+  	my $doc_id	= 0;
+  	my $flag	= 0;
+
+  	foreach my $line (@{$lines}) 
+	{
+    	chomp($line);
+    	$count++;
+
+		# Blank lines, new documents
+    	if ($line =~ /^\s*$/) 
+		{
+			# More than 1 document
+      		$flag = 1;
+
+      		$count_map->{$doc_id} = $count;
+      
+      		$count = 0;
+      		$doc_id++;
+    	}
+  	}
+
+  	if ($flag == 0)
+	{
+    	$count_map->{$doc_id} = $count;
+  	}
+}
+
+###
+# Main method to extract features
+###
+sub CRFFeature 
+{
+	my ($line, $index, $num_lines, $is_abstract, $is_intro, $xml_feature, $tag, $feats) = @_;
+  	
+	my $token			= "";
+  	my @templates		= ();
+  	my %feature_counts	= (); # To perform feature linking
+
+	my @tmp_tokens	= split(/\s+/, $line);
+  	# Filter out empty token
+  	my @tokens		= ();
+
+  	foreach my $token (@tmp_tokens)
+	{
+    	$token =~ s/^\s+//g; # Strip off leading spaces
+    	$token =~ s/\s+$//g; # Strip off trailing spaces
+
+		if ($token ne "")
+		{
+      		push(@tokens, $token);
+    	}
+  	}
+
+  	# Full form: does not count in crf template file, simply for outputing purpose to get the whole line data
+  	my $lineFull = join("|||", @tokens);
+  	push(@{$feats}, "$lineFull");
+
+  	###
+  	# Line-level features
+  	###
+	GenerateLineFeature($line, \@tokens, $index, $num_lines, $is_abstract, $is_intro, $feats, "# Line-level features\n", $tag_map{"lineLevel"}, \@templates, \%feature_counts);
+
+	###
+  	# XML features
+	###
+  	GenerateXmlFeature($xml_feature, $feats, "# Xml features\n", $tag_map{"xml"}, $tag_map{"bi_xml"}, \@templates, \%feature_counts);
+
+	# GenerateNumberFeature(\@tokens, $feats, "#number. features\n", $tag_map{"number"}, \@templates, \%feature_counts);
+  
+  	# Keyword features
+	for (my $i = 1; $i <= 4; $i++)
+	{
+    	if ($config{"${i}gram"})
+		{
+      		my @top_tokens = ();
+      		GetNgrams($line, $i, \@top_tokens);
+      		GenerateKeywordFeature(\@top_tokens, $feats, \%keywords, "# ${i}gram features\n", $tag_map{"${i}gram"}, \@templates, \%feature_counts);
+    	}
+  	}
+
+  	###
+  	# Token-level features
+  	###
+  	# Apply most of Parscit features
+  	for (my $i = 1; $i <= 4; $i++)
+	{
+    	if($config{"${i}token"})
+		{
+      		GenerateTokenFeature(\@tokens, ($i-1), \%keywords, $feats, "#${i}token general features\n", $tag_map{"${i}token"}, \@templates, \%feature_counts);
+    	}
+  	}
+
+  	###
+  	# Feature linking
+  	###
+  	my $i = undef;
+
+	if ($config{"back1"})
+	{
+    	FeatureLink(\@templates, "UA", "#constraint on first token features at -1 relative position \n", $feature_counts{$tag_map{"1token"}}->{"start"}, $feature_counts{$tag_map{"1token"}}->{"end"}, "-1");
+  	}
+  	push(@templates, "\n");
+
+  	if ($config{"forw1"})
+	{
+    	FeatureLink(\@templates, "UB", "#constraint on first token features at +1 relative position \n", $feature_counts{$tag_map{"1token"}}->{"start"}, $feature_counts{$tag_map{"1token"}}->{"end"}, "1");
+  	}
+  	push(@templates, "\n");
+
+  	# Output tag
+  	push(@{$feats}, $tag);
+
+  	push(@templates, "# Output\nB0\n");
+  	return @templates;
+}
+
+sub GenerateXmlFeature 
+{
+	my ($xml_feature, $feats, $msg, $label, $biLabel, $templates, $feature_counts) = @_;
+
+  	my @features	= split(/\s+/, $xml_feature);  	
+	my $count		= 0;
+ 	my $type		= undef;
+
+  	my %bi_feature_flag = ();
+	foreach my $feature (@features) 
+	{
+    	if ($feature =~ /^bi_xml/) { $bi_feature_flag{$count} = 1; }
+
+    	if ($feature =~ /^((bi_)?xml[a-zA-Z]+)\_.+$/)
+		{
+      		$type = $1;
+      		if ($config{$type}) 
+			{
+				push(@{$feats}, $feature);
+				$count++;
+      		}
+    	} 
+		else 
+		{
+      		die "Die: xml feature doesn't match \"$feature\"\n";
+    	}
+  	}
+  
+  	UpdateTemplate(scalar(@{$feats}), $count, $msg, $label, $templates, $feature_counts, $biLabel, \%bi_feature_flag);
+}
+
+sub UpdateTemplate 
+{
+	my ($cur_size, $num_features, $msg, $label, $templates, $feature_counts, $biLabel, $bi_feature_flag) = @_;
+
+  	# Crfpp template
+  	push(@{$templates}, $msg);
+  	my $prev_size = $cur_size - $num_features;
+  	$feature_counts->{$label}->{"start"} = $prev_size;
+  	$feature_counts->{$label}->{"end"} = $cur_size;
+  
+  	my $i = 0;
+  	for (my $j = $prev_size; $j < $cur_size; $j++)
+	{
+    	if ($bi_feature_flag->{$i})
+		{
+      		push(@{$templates}, "$biLabel".$i++.":%x[0,$j]\n");
+    	} 
+		else 
+		{
+      		push(@{$templates}, "$label".$i++.":%x[0,$j]\n");
+    	}
+  	}
+  	
+	push(@{$templates}, "\n");
+}
+
+sub GenerateLineFeature 
+{
+	my ($line, $tokens, $index, $num_lines, $is_abstract, $is_intro, $feats, $msg, $label, $templates, $feature_counts) = @_;
+
+  	# Crfpp template
+  	push(@{$templates}, $msg);
+  	my $prev_size = scalar(@{$feats});
+  	$feature_counts->{$label}->{"start"} = $prev_size;
+
+  	# Editor
+  	my $has_possible_editor = ($line =~ /[^A-Za-z](ed\.|editor|editors|eds\.)/) ? "possibleEditors" : "noEditors";
+  	push(@{$feats}, $has_possible_editor);
+  
+  	if ($config{"lineNum"})
+	{
+    	my $word	= $tokens->[0];
+    	my $num		= "";
+
+    	if (scalar(@{$tokens}) > 1)
+		{
+      		$num =	($word =~ /^[1-9]\.[1-9]\.?$/)			? "posSubsec"		: 
+					($word =~ /^[1-9]\.[1-9]\.[1-9]\.?$/)	? "posSubsubsec"	: 
+					($word =~ /^\w\.[1-9]\.[1-9]\.?$/)		? "posCategory"		:
+					"";
+    	}
+    
+    	if ($num eq "")
+		{
+      		$num =	($word =~ /^[1-9][A-Za-z]\w*$/)		? "numFootnote" 	:
+					($word =~ /^[1-9]\s*(http|www)/) 	? "numWebfootnote" 	:
+	  				"lineNumOthers";
+    	}
+    
+    	push(@{$feats}, $num);
+  	}
+  
+  	if ($config{"linePunct"})
+	{
+    	my $punct	= "";
+    	$punct		=	($line =~ /@\w+\./)			? "possibleEmail"	: 
+      					($line =~ /(www|http)/)		? "possibleWeb" 	: 
+						($line =~ /\(\d\d?\)\s*$/)	? "endNumbering" 	: 
+	  					"linePunctOthers";
+  
+    	push(@{$feats}, $punct);
+  	}
+  
+  	if ($config{"lineCapital"})
+	{
+    	my $cap = GetCapFeature($tokens);
+    	push(@{$feats}, $cap);
+  	}
+  
+  	if ($config{"linePos"})
+	{
+    	my $position = "POS-".int($index*8.0/$num_lines);
+    	push(@{$feats}, $position);
+  	}
+  
+  	if ($config{"lineLength"})
+	{ 
+    	# Num tokens, words
+    	my @tokens		= split(/\s+/, $line);    
+    	my $num_words	= 0;
+
+    	foreach my $token (@tokens)
+		{
+      		if ($token =~ /^\p{P}*[a-zA-Z]+\p{P}*$/)
+			{
+				$num_words++;
+      		}
+    	}
+    
+		my $word_length = 
+      	($num_words >= 5) ? "5+Words" : "${num_words}Words";
+
+    	push(@{$feats}, $word_length);
+  	}
+
+  	# For crfpp template
+  	my $cur_size = scalar(@{$feats});
+  	$feature_counts->{$label}->{"end"} = $cur_size;
+  
+  	my $i = 0;
+  	for (my $j = $prev_size; $j < $cur_size; $j++)
+	{
+    	push(@{$templates}, "$label".$i++.":%x[0,$j]\n");
+  	}
+  
+  	push(@{$templates}, "\n");
+}
+
+sub GenerateTokenFeature 
+{
+	my ($tokens, $index, $keywords, $feats, $msg, $label, $templates, $feature_counts) = @_;
+
+  	my $num_tokens	= scalar(@{$tokens});
+  	my $token		= "EMPTY";
+  	if ($num_tokens > $index) { $token = $tokens->[$index]; }
+
+  	# Crfpp template
+  	push(@{$templates}, $msg);
+  	my $prev_size = scalar(@{$feats});
+  	$feature_counts->{$label}->{"start"} = $prev_size;
+
+  	# Prep
+  	my $word	= $token;
+  	my $word_lc	= lc($token);
+
+	# No punctuation
+	my $word_np	= $token;
+	$word_np	=~ s/[^\w]//g;  	
+	if ($word_np =~ /^\s*$/) { $word_np = "EMPTY"; }
+
+	# Lowercased word, no punctuation
+  	my $word_lcnp = lc($word_np);    
+  	if ($word_lcnp =~ /^\s*$/) { $word_lcnp = "EMPTY"; }
+  
+  	# Lexical features
+  	push(@{$feats}, "TOKEN-$word"); # Lexical word
+  	push(@{$feats}, "$word_lc");  	# Lowercased word
+  	push(@{$feats}, "$word_lcnp");  # Lowercased word, no punct
+
+	# Parscit char feature
+	if ($config{"parscit"})
+	{
+		# Parscit char feature
+    	if ($config{"parscit_char"})
+		{
+      		my @chars 		= split(//,$word);
+      		my $last_char 	= $chars[-1];
+      	
+			if ($last_char =~ /[\p{IsLower}]/) 
+			{ 
+				$last_char = 'a'; 
+			}
+      		elsif ($last_char =~ /[\p{IsUpper}]/) 
+			{ 
+				$last_char = 'A'; 
+			}
+      		elsif ($last_char =~ /[0-9]/) 
+			{ 
+				$last_char = '0'; 
+			}
+
+			# 1 = last char
+	  		push(@{$feats}, $last_char);		       
+
+      		# Thang added 02-Mar-10 this to avoid uninitialized warnning messages when using -w
+      		for (my $i = scalar(@chars); $i < 4;$i++)
+			{
+				push(@chars, '|');
+      		}
+
+      		push(@{$feats}, $chars[0]);		      		# 2 = first char
+      		push(@{$feats}, join("",@chars[0..1]));  	# 3 = first 2 chars
+      		push(@{$feats}, join("",@chars[0..2]));  	# 4 = first 3 chars
+      		push(@{$feats}, join("",@chars[0..3]));  	# 5 = first 4 chars
+      
+      		push(@{$feats}, $chars[-1]);		       	# 6 = last char
+      		push(@{$feats}, join("",@chars[-2..-1])); 	# 7 = last 2 chars
+      		push(@{$feats}, join("",@chars[-3..-1])); 	# 8 = last 3 chars
+      		push(@{$feats}, join("",@chars[-4..-1]));	# 9 = last 4 chars
+    	}
+  	}
+
+  	# Capitalization features
+  	if ($config{"tokenCapital"})
+	{
+    	my $ortho = ($word_np =~ /^[\p{IsUpper}]$/) 				? "singleCap" 	:
+      				($word_np =~ /^[\p{IsUpper}][\p{IsLower}]+/)	? "InitCap" 	:
+					($word_np =~ /^[\p{IsUpper}]+$/) 				? "AllCap" 		: "others";
+    
+		push(@{$feats}, $ortho);
+  	}
+
+  	# Number features
+  	if ($config{"tokenNumber"})
+	{
+    	my $num = undef;
+    
+    	if ($config{"parscit"})
+		{
+      		$num = 	($word_np =~ /^(19|20)[0-9][0-9]$/) 	? "year" 			:
+					($word =~ /[0-9]\-[0-9]/) 				? "possiblePage" 	:
+	  				($word =~ /[0-9]\([0-9]+\)/)	 		? "possibleVol"		:
+	    			($word_np =~ /^[0-9]$/) 				? "1dig" 			:
+	      			($word_np =~ /^[0-9][0-9]$/) 			? "2dig" 			:
+					($word_np =~ /^[0-9][0-9][0-9]$/) 		? "3dig" 			:
+		  			($word_np =~ /^[0-9]+$/) 				? "4+dig" 			:
+		    		($word_np =~ /^[0-9]+(th|st|nd|rd)$/) 	? "ordinal" 		:
+		      		($word_np =~ /[0-9]/) 					? "hasDig" 			: "nonNum";
+    	} 
+		else 
+		{
+      		$num =	($word =~ /^[1-9]+\.$/)					? "endDot" 			:
+	  				($word =~ /^[1-9]+:$/) 					? "endCol"			:
+	    	    	# Parscit features
+	    			($word_np =~ /^(19|20)[0-9][0-9]$/) 	? "year" 			:
+	      			($word =~ /[0-9]\-[0-9]/) 				? "possiblePage"	:
+					($word =~ /[0-9]\([0-9]+\)/) 			? "possibleVol" 	:
+		  			($word_np =~ /^[0-9]$/) 				? "1dig" 			:
+		    		($word_np =~ /^[0-9][0-9]$/) 			? "2dig" 			:
+		      		($word_np =~ /^[0-9][0-9][0-9]$/) 		? "3dig" 			:
+					($word_np =~ /^[0-9]+$/) 				? "4+dig" 			:
+			  		($word_np =~ /^[0-9]+(th|st|nd|rd)$/) 	? "ordinal" 		:
+			    	($word_np =~ /[0-9]/) 					? "hasDig" 			: "nonNum";
+    	}
+    
+		push(@{$feats}, $num);
+  	}
+
+  	# Gazetteer (names) features
+  	if ($config{"tokenName"})
+	{
+    	my $dict_status = (defined $dict{$word_lcnp}) ? $dict{$word_lcnp} : 0;
+    	# my $is_in_dict = ($dict_status != 0) ? "is_in_dict" : "no";
+    
+		my $is_in_dict = $dict_status;
+    
+		my ($publisher_name, $place_name, $month_name, $last_name, $female_name, $male_name) = undef;
+    
+    	if ($dict_status >= 32) { $dict_status -= 32; 	$publisher_name	= "publisherName"	} else { $publisher_name	= "no"; }
+    	if ($dict_status >= 16)	{ $dict_status -= 16; 	$place_name 	= "placeName" 		} else { $place_name 		= "no"; }
+    	if ($dict_status >= 8)	{ $dict_status -= 8; 	$month_name 	= "monthName" 		} else { $month_name 		= "no"; }
+    	if ($dict_status >= 4)	{ $dict_status -= 4; 	$last_name 		= "lastName" 		} else { $last_name 		= "no"; }
+    	if ($dict_status >= 2) 	{ $dict_status -= 2; 	$female_name 	= "femaleName" 		} else { $female_name 		= "no"; }
+    	if ($dict_status >= 1) 	{ $dict_status -= 1; 	$male_name 		= "maleName" 		} else { $male_name 		= "no"; }
+    
+    	push(@{$feats}, $is_in_dict);		# 13 = name status
+    	push(@{$feats}, $male_name);	    # 14 = male name
+    	push(@{$feats}, $female_name);		# 15 = female name
+    	push(@{$feats}, $last_name);	    # 16 = last name
+    	push(@{$feats}, $month_name);		# 17 = month name
+    	push(@{$feats}, $place_name);		# 18 = place name
+    	push(@{$feats}, $publisher_name);	# 19 = publisher name
+  	}
+  
+  	# Punctuation features
+  	if ($config{"tokenPunct"})
+	{
+    	my $punct = undef;
+
+    	if ($config{"parscit"})
+		{
+      		$punct = 	($word =~ /^[\"\'\`]/) 						? "leadQuote" 	:
+						($word =~ /[\"\'\`][^s]?$/) 				? "endQuote" 	:
+	  					($word =~ /\-.*\-/) 						? "multiHyphen"	:
+	    				($word =~ /[\-\,\:\;]$/) 					? "contPunct" 	:
+	      				($word =~ /[\!\?\.\"\']$/) 					? "stopPunct" 	:
+	        			($word =~ /^[\(\[\{\<].+[\)\]\}\>].?$/) 	? "braces" 		:
+		  				($word =~ /^[0-9]{2-5}\([0-9]{2-5}\).?$/)	? "possibleVol" : "others";
+    	} 
+		else 
+		{
+      		$punct =	($word =~ /^[a-z]\d$/) 						? "possibleVar" : # x1, x2
+	  		# Parscit
+			#			($word =~ /^[\"\'\`]/) 						? "leadQuote" 	:
+			#			($word =~ /[\"\'\`][^s]?$/) 				? "endQuote" 	:
+			#			($word =~ /\-.*\-/) 						? "multiHyphen" :
+						($word =~ /[\-\,\:\;]$/) 					? "contPunct" 	:
+		  				($word =~ /[\!\?\.\"\']$/) 					? "stopPunct" 	:
+		    			($word =~ /^[\(\[\{\<].+[\)\]\}\>].?$/) 	? "braces" 		:
+		     	 		($word =~ /^[0-9]{2-5}\([0-9]{2-5}\).?$/)	? "possibleVol"	: "punctOthers";
+
+			#			($word =~ /^[\*\^\x{0608}\x{0708}\x{07A0}][A-Za-z]\w*$/  && $index == 0)	? "punctFootnote" 	:
+			#			($word =~ /^[\p{P}\p{Math_Symbol}]*\p{Math_Symbol}\p{P}\p{Math_Symbol}]*/)	? "mathSym" 		:
+    	}
+
+    	push(@{$feats}, $punct);
+	}
+
+  	if ($config{"tokenKeyword"})
+	{
+    	my $keyword_fea = "noKeyword";
+    	my $token = $word;
+
+		$token =~ s/^\p{P}+//g; # Strip out leading punctuations
+    	$token =~ s/\p{P}+$//g; # Strip out trailing punctuations
+    	$token =~ s/\d/0/g; 	# Canocalize number into "0"
+    
+		foreach (keys %{$all_tags})
+		{
+      		if ($all_tags->{$_} == 0) { next; }
+      
+      		if ($keywords->{$_}->{$token})
+			{
+				$keyword_fea = "keyword-$_";
+				last;
+      		}
+    	}
+
+    	push(@{$feats}, $keyword_fea);
+  	}
+
+  	# For crfpp template
+  	my $cur_size = scalar(@{$feats});
+  	$feature_counts->{$label}->{"end"} = $cur_size;
+  	
+	my $i = 0;
+  	for(my $j = $prev_size; $j < $cur_size; $j++)
+	{
+    	push(@{$templates}, "$label".$i++.":%x[0,$j]\n");
+  	}
+  
+  	push(@{$templates}, "\n");
+}
+
+sub GetCapFeature 
+{
+	my ($tokens) = @_;
+	
+	my $cap 	= "OthersCaps";
+ 	my $n 		= 0;
+  	my $count 	= 0; # non-word
+  	my $count1 	= 0;
+  	my $line 	= "";
+
+  	# Check capitalization
+  	my $is_skip = 0;
+  	for (my $i = 0; $i < scalar(@{$tokens}); $i++)
+	{
+    	my $token = $tokens->[$i];
+    	if ($token =~ /^\p{P}*$/) { next; }
+	
+		my @chars = split(//, $token);    
+		# Exclude non-word or an important words such as a, an, the, on, in ...
+    	if (scalar(@chars) < 4) 
+		{ 
+			# Dont' consider skip if it is the first token as numbers
+      		if (!($i == 0 && $token =~ /\d/))
+			{
+				$is_skip = 1; 
+      		}
+      		
+			next;
+    	} 
+	
+		# Capitalized
+    	if ($token =~ /^[A-Z][A-Za-z]*$/)
+		{ 
+      		$count++;
+      		$line .= "$token ";
+    	}
+    
+    	$n++;
+  	}
+
+	# Consider only if at lest 1 capitalized word
+  	if ($count >0)
+	{
+    	if ($count == $n)
+		{
+      		$cap = ($is_skip) ? "Most" : "All";
+      
+	  		if($line =~ /[a-z]/)
+			{
+				$cap .= "InitCaps";
+      		} 
+			else 
+			{
+				$cap .= "CharCaps";
+      		}
+
+			# First token contains number
+	  		if ($tokens->[0] =~ /\d/) 
+			{
+				$cap = "number$cap";
+      		} 
+			# Two few capitalized letter to conclude any pattern
+			elsif ($count == 1)
+			{
+				$cap = "OthersCaps";
+      		}
+    	}
+  	}
+  
+  	return $cap;
+}
+
+sub FeatureLink 
+{
+	my ($templates, $label, $msg, $start, $end, $rel_pos) = @_;
+
+  	# To constraint on last token features at $rel_pos relative position
+  	my $i = 0;  	
+	push(@{$templates}, $msg);
+  
+  	for (my $j = $start; $j < $end; $j++)
+	{
+    	push(@{$templates}, "$label".$i++.":%x[$rel_pos,$j]\n");
+  	}
+  
+  	push(@{$templates}, "\n");
+}
+
+sub GenerateKeywordFeature 
+{
+
+	my ($tokens, $feats, $keywords, $msg, $label, $templates, $feature_counts) = @_;
+
+  	# Crfpp template
+  	push(@{$templates}, $msg);
+  	my $prev_size = scalar(@{$feats});
+  	$feature_counts->{$label}->{"start"} = $prev_size;
+
+  	foreach (keys %{$all_tags})
+	{
+    	if($all_tags->{$_} == 0) { next; };
+
+    	my $i=0;
+    	for(; $i<scalar(@{$tokens}); $i++)
+		{
+      		if ($keywords->{$_}->{$tokens->[$i]})
+			{
+				push(@{$feats}, "$_-".$tokens->[$i]);
+				last;
+      		}
+    	}
+
+    	if ($i==scalar(@{$tokens}))
+		{
+      		push(@{$feats}, "none");
+    	}
+  	}
+
+  	# For crfpp template
+  	my $cur_size = scalar(@{$feats});
+  	$feature_counts->{$label}->{"end"} = $cur_size;
+  
+  	my $i = 0;
+  	for (my $j = $prev_size; $j < $cur_size; $j++)
+	{
+    	push(@{$templates}, "$label".$i++.":%x[0,$j]\n");
+  	}
+  	
+	push(@{$templates}, "\n");
+}
+
+# Get ngrams
+sub GetNgrams 
+{
+	my ($line, $num_ngram, $ngrams) = @_;
+
+	# $line = lc($line);
+
+  	my @tmp_tokens = split(/\s+/, $line);
+
+  	# Filter out empty token
+  	my @tokens = ();
+	foreach my $token (@tmp_tokens)
+	{ 
+    	if ($token ne "")
+		{
+      		$token =~ s/^\s+//g; 	# Strip off leading spaces
+      		$token =~ s/\s+$//g; 	# Strip off trailing spaces
+      		$token =~ s/^\p{P}+//g; # Strip out leading punctuations
+      		$token =~ s/\p{P}+$//g; # Strip out trailing punctuations
+      		$token =~ s/\d/0/g; 	# Canocalize number into "0"
+			
+			# Email pattern, try to normalize
+      		if ($token =~ /(\w.*)@(.*\..*)/)
+			{
+				# $token =~ /(http:\/\/|www\.)/){
+				$token = $1;
+				
+				my $remain = $2;
+				$token =~ s/\w+/x/g;
+				$token =~ s/\d+/0/g;
+				$token .= "@".$remain;
+      		} 
+
+      		if ($token ne "")
+			{
+				push(@tokens, $token);
+      		}
+    	}
+  	}
+
+  	my $count = 0;  
+	for (my $i = 0; $i <= $#tokens; $i++)
+	{ 
+		# not enough ngrams
+    	if (($#tokens-$i + 1) < $num_ngram) { last; };
+    
+		my $ngram = "";
+    	for (my $j=$i; $j <= ($i+$num_ngram-1); $j++)
+		{
+      		my $token = $tokens[$j];
+
+      		if ($j < ($i+$num_ngram-1))
+			{
+				$ngram .= "$token-";
+      		} 
+			else 
+			{
+				$ngram .= "$token";
+      		}
+    	}
+
+    	if ($ngram =~ /^\s*$/)	{ next; } # Skip those with white spaces
+    	if ($ngram =~ /^\d*$/)	{ next; } # Skip those with only digits
+    	if ($func_word{$ngram})	{ next; } # Skip function words, matter for ngram = 1
+
+		push(@{$ngrams}, $ngram);
+
+    	$count++;
+    	if ($count == 4) { last; }
+  	}
+}
+
+sub GenerateNumberFeature 
+{
+	my ($tokens, $feats, $msg, $label, $templates, $feature_counts) = @_;
+
+  	# crfpp template
+  	push(@{$templates}, $msg);
+  	my $prev_size = scalar(@{$feats});
+  	$feature_counts->{$label}->{"start"} = $prev_size;
+
+  	my $line 	= join("", @{$tokens});
+  	$line 		=~ s/\s+//g;
+  	my @chars	= split(//, $line);
+
+  	my $count 	= 0;
+  	my $n 		= scalar(@chars);
+
+	foreach(@chars)
+	{
+    	if (/\d/) { $count++; }
+  	}
+
+  	my $num = "otherNum";
+  
+  	if ($n > 1)
+	{
+    	my $ratio = $count/$n;
+    	if ($ratio >= 0.7)
+		{
+      		$num = "HighNum";
+    	} 
+		elsif ($ratio >= 0.4)
+		{
+     		$num = "MeidumNum";
+    	}
+  	}
+
+  	push(@{$feats}, $num);
+  
+  	# For crfpp template
+  	my $cur_size = scalar(@{$feats});
+  	$feature_counts->{$label}->{"end"} = $cur_size;
+  
+  	my $i = 0;
+  	for (my $j = $prev_size; $j < $cur_size; $j++)
+	{
+    	push(@{$templates}, "$label".$i++.":%x[0,$j]\n");
+  	}
+  
+  	push(@{$templates}, "\n");
+}
+
+sub GenerateFuncFeature 
+{
+	my ($tokens, $feats, $msg, $label, $templates, $feature_counts) = @_;
+
+  	# Crfpp template
+  	push(@{$templates}, $msg);
+  	my $prev_size = scalar(@{$feats});
+  	$feature_counts->{$label}->{"start"} = $prev_size;
+
+  	my $n 		= scalar(@{$tokens});
+  	my $count 	= 0;
+  
+  	foreach my $token (@{$tokens})
+	{
+    	$token =~ s/^\p{P}+//; # Strip leading punct
+    	$token =~ s/\p{P}+$//; # Strip traiing punct
+    	$token = lc($token);
+
+		if ($func_word{$token})
+		{
+      		$count++;
+    	}
+  	}
+
+  	if ($count == 0)
+	{
+    	push(@{$feats}, "NoFunc");
+  	} 
+	elsif ($count <= 5)
+	{
+    	push(@{$feats}, "FewFunc");
+  	} 
+	else 
+	{
+    	push(@{$feats}, "AlotFunc");
+  	}
+
+  	# For crfpp template
+  	my $cur_size = scalar(@{$feats});
+  	$feature_counts->{$label}->{"end"} = $cur_size;
+  
+  	my $i = 0;
+  	for (my $j = $prev_size; $j < $cur_size; $j++)
+	{
+    	push(@{$templates}, "$label".$i++.":%x[0,$j]\n");
+  	}
+  
+  	push(@{$templates}, "\n");
+}
+
+sub BuildTmpFile 
+{
+    my ($filename) = @_;
+	
+	my $tmpfile = $filename;
+    $tmpfile 	=~ s/[\.\/]//g;
+    $tmpfile 	.= $$ . time;
+    
+	# Untaint tmpfile variable
+    if ($tmpfile =~ /^([-\@\w.]+)$/) 
+	{
+		$tmpfile = $1;
+    }
+    
+    return "/tmp/$tmpfile"; # Altered by Min (Thu Feb 28 13:08:59 SGT 2008)
+}
+
+
+sub Fatal 
+{
+    my $msg = shift;
+    print STDERR "Fatal Exception: $msg\n";
+}
+
+
+sub Decode 
+{
+	my ($infile, $model_file, $outfile) = @_;
+  
+  	my $labeled_file = BuildTmpFile($infile);
+  	Execute("$crf_test -v1 -m $model_file $infile > $labeled_file"); #  -v1: output confidence information
+
+  	open (PIPE, "<:utf8", $labeled_file) || die "# crash\t\tCan't open \"$labeled_file\"";
+  	open (OUT, ">:utf8", $outfile) || die "# crash\t\tCan't open \"$outfile\"";
+
+  	while(<PIPE>)
+	{
+    	chomp;
+    	print OUT "$_\n";
+  	}
+  	
+	close PIPE;
+  	close OUT;
+ 
+ 	unlink($labeled_file);
+  	return 1;
+}
+
+sub ReadKeywordDict 
+{
+  	my ($infile, $keywords) = @_;
+
+  	open (IF, "<:utf8", $infile) || die "fatal\t\tCannot open \"$infile\"!";
+  	# Process input file
+  	while(<IF>)
+  	{
+  		chomp;
+    
+    	if (/^(.+?): (.+)$/)
+		{
+      		my $tag 	= $1;
+      		my @tokens 	= split(/\s+/, $2);
+      
+	  		$keywords->{$tag} = ();
+      		foreach(@tokens)
+			{
+				$keywords->{$tag}->{$_} = 1;
+      		}
+    	}
+  	}
+
+  	close (IF);
+}
+
+sub LoadConfigFile 
+{
+	my ($infile, $configs) = @_;
+
+  	open (IF, "<:utf8", $infile) || die "fatal\t\tCannot open \"$infile\"!";
+  
+  	while(<IF>)
+	{
+    	chomp;
+    
+    	if (/^(.+)=(.+)$/)
+		{
+      		my $name = $1;
+      		my $value = $2;
+      		$configs->{$name} = $value;
+    	}
+  	}
+
+  	close (IF);
+}
+
+sub ReadDict 
+{
+  	my ($dictFileLoc) = @_;
+
+  	my $mode = 0;
+  	open (DATA, "<:utf8", $dictFileLoc) || die "Could not open dict file $dictFileLoc: $!";
+  	while (<DATA>) 
+	{
+    	if (/^\#\# Male/) 			{ $mode = 1; }		# male names
+    	elsif (/^\#\# Female/) 		{ $mode = 2; }		# female names
+    	elsif (/^\#\# Last/) 		{ $mode = 4; }		# last names
+    	elsif (/^\#\# Chinese/) 	{ $mode = 4; }		# last names
+    	elsif (/^\#\# Months/) 		{ $mode = 8; }		# month names
+    	elsif (/^\#\# Place/) 		{ $mode = 16; }		# place names
+    	elsif (/^\#\# Publisher/)	{ $mode = 32; }		# publisher names
+    	elsif (/^\#/) { next; }
+    	else 
+		{
+      		chop;
+      		my $key = $_;
+      		my $val = 0;
+			# Has probability
+      		if (/\t/) { ($key,$val) = split (/\t/,$_); }
+
+      		# Already tagged (some entries may appear in same part of lexicon more than once
+      		if (!$dict{$key})
+			{
+				$dict{$key} = $mode;
+      		} 
+			else 
+			{
+				if ($dict{$key} >= $mode) 
+				{ 
+					next; 
+				}
+				# Not yet tagged
+				else 
+				{ 
+					$dict{$key} += $mode; 
+				}
+      		}
+    	}
+  	}
+  
+	close (DATA);
+}
+
+sub LoadListHash 
+{
+	my ($infile, $hash) = @_;
+
+  	open(IF, "<:utf8", $infile) || die "#Can't open file \"$infile\"";
+
+  	while(<IF>)
+	{
+    	chomp;
+    	$hash->{$_} = 1;
+  	}
+
+  	close IF;
+}
+
+sub Untaint 
+{
+	my ($s) = @_;
+  	if ($s =~ /^([\w \-\@\(\),\.\/<>]+)$/) 
+	{
+    	$s = $1;               # $data now untainted
+  	} 
+	else 
+	{
+    	die "Bad data in $s";  # Log this somewhere
+  	}
+  
+  	return $s;
+}
+
+sub Execute 
+{
+	my ($cmd) = @_;
+  	$cmd = Untaint($cmd);
+  	system($cmd);
+}
+
+1;