biblicit 1.0

data/perl/HeaderParseService/lib/HeaderParse/API/ParserMethods.pm

@@ -0,0 +1,1880 @@
+#
+# Copyright 2007 Penn State University
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+package HeaderParse::API::ParserMethods;
+
+#06/27/2003, start to make this program to handle real data. So there is no evaluation, and off line classifiers should be trained be trained beforehand.
+#02/10/2004 Apply to citeseer data (with the same format of EbizSearch data)
+
+use utf8;
+use Data::Dumper;
+use FindBin;
+use HeaderParse::API::NamePatternMatch;
+use HeaderParse::API::MultiClassChunking; #default to use all export by this module
+use HeaderParse::API::LoadInformation;
+use HeaderParse::Config::API_Config;
+use HeaderParse::API::AssembleXMLMetadata;
+use vars qw($debug %dictH %nameH %firstnameH %lastnameH %BasicFeatureDictH %InverseTagMap);
+use vars qw($Classifier $offlineD $Tmp_Dir $nMinHeaderLength $nMaxHeaderLength);
+use HeaderParse::API::Function qw(&AddrMatch &printDict &GenTrainVecMatrix &LineFeatureRepre &FillSpace &SeparatePunc);
+
+my $FeatureDictH = \%BasicFeatureDictH;
+my $ContextFeatureDictH;
+my $SpaceAuthorFeatureDictH; #do not know if it is OK to define a hash
+my $PuncAuthorFeatureDictH;
+my $NameSpaceTrainVecH;
+my $NameSpaceTrainF = "$offlineD"."NameSpaceTrainF";
+my $SVMNameSpaceModel = "$offlineD"."NameSpaceModel";
+my $TestH;
+my $TrainH;
+my $TotalHea = 0;
+
+my $timestamp;
+
+#my $offlineD = "../../offline/";
+#my $TestOutF = "$TestF"."\.parsed";
+#my $tmpCacheVecB = "$Tmp_Dir/tmpVec";
+#my $SVMTmpResultB = "$Tmp_Dir/tmpresult";
+
+my $FeatureDict = "$offlineD"."WrapperBaseFeaDict";
+my $ContextFeatureDict = "$offlineD"."WrapperContextFeaDict";
+my $SpaceAuthorFeatureDictF = "$offlineD"."WrapperSpaceAuthorFeaDict";
+my $PuncAuthorFeatureDictF = "$offlineD"."WrapperPuncAuthorFeaDict";
+
+my $linear = 1; # just want to be fast
+
+my %evalH; # global hash to record classification result for baseline, each context round and IE
+my $norm = 1;
+my $testp = 1; # this is only to make the program run, no meaning.
+
+
+my %TestDataIndex; #It indexes the header no in the testing dataset
+
+#Read dictionary files
+undef $/;
+open(dumpFH, "$FeatureDict") || die "SVMHeaderParse: could not open $FeatureDict to read: $!";
+my $string = <dumpFH>;
+close(dumpFH);
+eval $string;
+$FeatureDictH = $VAR1;
+$string ="";
+
+open(dumpFH, "$ContextFeatureDict") || die "SVMHeaderParse: could not open $ContextFeatureDict to read: $!";
+$string = <dumpFH>;
+close(dumpFH);
+eval $string;
+$ContextFeatureDictH = $VAR1;
+$string ="";
+
+open(dumpFH, "$SpaceAuthorFeatureDictF") || die "SVMHeaderParse: could not open $SpaceAuthorFeatureDictF to read: $!";
+$string = <dumpFH>;
+close(dumpFH);
+eval $string;
+$SpaceAuthorFeatureDictH = $VAR1;
+$string ="";
+$/ = "\n";
+#End read dictionary files
+
+
+sub Parse{
+    my $header=shift;
+    $timestamp = shift;
+    my $success = 0;
+   # $tmpCacheVec = $tmpCacheVec . "\_$timestamp\_";
+   # $SVMTmpResult = $SVMTmpResult . "\_$timestamp\_";
+    my $tmpCacheVec = "$Tmp_Dir/tmpVec"."\_$timestamp\_";
+
+    my $SVMTmpResult = "$Tmp_Dir/tmpresult"."\_$timestamp\_";
+    $TestH = &HashEbizHeader(\$header);
+    $TestH = &VectorizeUnknownHeaderLine($TestH);
+
+    my $baseline = 1;
+    $TestH = &LineClassify($testp, "", $baseline, $FeatureDictH,
+			   $TestH, $tmpCacheVec, $SVMTmpResult);
+    $TestH = &UpdatePretag($TestH);
+
+    my $maxLoop = 2;
+    for my $loop(1 .. $maxLoop) {
+        $baseline = 0;
+        my $NowContext = "context"."$loop";
+
+        $TestH = &LineClassify($testp, $NowContext, $baseline,
+			       $ContextFeatureDictH, $TestH,
+			       $tmpCacheVec, $SVMTmpResult);
+        $TestH = &UpdatePretag($TestH);
+    }
+
+    #Phase 2: Extraction Information from Multi-Class Lines and Author Lines Chunks
+    my $LastContext = "context"."$maxLoop";
+
+    # BUG: InfoExtract hangs on some documents.
+    # this is reproducible with data extracted using TET from doc 654835
+    # from the legacy citeseer system.
+    eval {
+	local $SIG{'ALRM'} = sub { die "alarm\n"; };
+	alarm 15;
+	$TestH = &InfoExtract($testp, $TestH,$SpaceAuthorFeatureDictH, $PuncAuthorFeatureDictH, $SVMNameSpaceModel, $tmpCacheVec, $SVMTmpResult);
+	alarm 0;
+    };
+    if ($@) {
+	if ($@ eq "alarm\n") {
+	    return 0;
+	}
+    }
+    $rXML = &ExportRDF($TestH);
+
+    for my $i(1..15){
+	unlink "$Tmp_Dir/tmpVec\_$timestamp\_test$i";
+	unlink "$Tmp_Dir/tmpresult\_$timestamp\_$i";
+    }
+    return $rXML;
+}
+
+
+# This is the header extraction module from CiteSeer.
+# Only the parts related to header extraction is used.
+sub ExtractHeaderInformation {
+    my $papertext = shift;
+    my $header='';
+
+    if (!(length($$papertext))){
+	return ('Paper text is empty');
+    }
+
+#  $$papertext =~ s/<[SEFC][\d\.e\+\-]*>//sgi; # remove S|E|F|C tags
+
+    if ($$papertext =~ /^(.*?\b(?:Introduction|INTRODUCTION|Contents|CONTENTS)(?:.*?\n){6})/s) {
+	$header = $1;
+    } else {
+	my $nLines = 150;
+	my @lines = split '\n', $$papertext;
+	my $contentLines = 0;
+	for (my $i=0; $i<=$#lines; $i++) {
+	    if ($lines[$i] !~ m/^\s*$/) {
+		$contentLines++;
+	    }
+	    $header .= $lines[$i]."\n";
+	    if ($contentLines >= $nLines) {
+		last;
+	    }
+	}
+    }
+
+#  if ($$papertext =~ /^(.*?)\b(?:Abstract|ABSTRACT|Introduction|INTRODUCTION|Contents|CONTENTS|[Tt]his\s+(paper|memo|technical|article|document|report|dissertation))\b/s) { $header = $1; }
+#  elsif ($$papertext =~ /^(.*?)\n[\d\.\s]*(Reference|Bibliography)/si) { $header = $1; }
+#  else{
+#      return ('Header could not be extracted');
+#  }
+
+    if ((defined $header) && (length ($header) > $nMaxHeaderLength)) {
+	$header = substr ($header, 0, $nMaxHeaderLength) . '...';
+    }
+    if (length($header) < $nMinHeaderLength) {
+	return ('Header could not be extracted');
+    }
+    return ('',$header);
+}
+
+
+sub UpdatePretag() {
+    my $testH = shift;
+ #   foreach my $testHea(sort {$a <=> $b} keys %{$testH}) {
+	foreach my $LN(sort {$a <=> $b} keys %{$testH}) {
+	    delete($$testH{$LN}{Pretag});
+	    if ($$testH{$LN}{PClass} eq "s") {
+		$$testH{$LN}{Pretag}{$$testH{$LN}{PSClsName}} = 1;
+	    }elsif ($$testH{$LN}{PClass} eq "m") {
+		foreach my $mytag(keys %{$$testH{$LN}{PClsName}}) {
+		    $$testH{$LN}{Pretag}{$mytag} = 1;
+		}
+	    }
+	}
+  #  }
+    return($testH);
+}
+
+
+#input: the file with all Training and testing samples
+#output: $HeaderH{$HeaNO}{$LineNO} = "";
+sub HashAllHeader() {
+    my $simulateHeaNum = shift;
+    my $tagF= shift;
+    my %HeaH = ();
+    my $HeaNO = 1; #start from 1
+    my $LineNO = 1;
+
+    open(tagFH, "$tagF") || die "SVMHeaderParse: could not open tag file\: $tagF to read: $!";
+    while (my $line = <tagFH>) {
+	$line =~ s/\+L\+//g;
+	$line =~ s/^\s+//g;
+	$line =~ s/\s+$//g;
+
+	if ($line =~ /^\s*\<NEW\_HEADER\>/) {
+	    $HeaNO++;
+	    $LineNO = 1;
+	#remove the line with only tag like </author>
+	}elsif (($line =~ /^\s*$/) || ($line =~ /^\<(\/)*(\w+)\>$/)) {
+	    next;
+	}else {
+	    $HeaH{$HeaNO}{$LineNO}{RawContent} = $line;
+	    $LineNO++;
+	}
+
+	if ($simulateHeaNum > 0 && $HeaNO >= $simulateHeaNum) {
+	    last;
+	}
+    }
+    close(tagFH);
+    return($HeaNO, \%HeaH);
+}
+
+
+#HEADER_DID[1]
+#TRECS: Developing a Web-based e-Commerce Business Simulation
+#TRECS: Developing a Web-based
+sub HashEbizHeader() {
+    my $headerRef= shift;
+    my %HeaH = ();
+#    my $HeaNO = 1; #start from 1
+    my $LineNO = 1;
+
+    my @lines = split(/\n/, $$headerRef);
+    my $line;
+
+    #open(FH, "$F") || die "SVMHeaderParse: could not open file\: $F to read: $!";
+    #while (my $line = <FH>) {
+    foreach $line (@lines){
+	$line =~ s/^\s+//g;
+	$line =~ s/\s+$//g;
+
+#	if ($line =~ /^\s*HEADER\_DID\[(\d+)\]/) {
+#	    $HeaNO = $1;
+#	    $LineNO = 1;
+#	}elsif ($line !~ /^\s*$/) {
+	    #$HeaH{$HeaNO}{$LineNO}{RawContent} = $line;
+            $HeaH{$LineNO}{RawContent} = $line;
+	    $LineNO++;
+#	}
+    }
+    #close(FH);
+    return(\%HeaH);
+}
+
+
+sub BaseLineTrainSys() {
+  my $HeaderH = shift;
+  my $FeatureDictH = shift;
+
+  my %InitialHash = ();
+  $InitialHash{FeatureCounter} = 0;
+
+  my $PuncAuthorDictH = \%InitialHash;
+  my $SpaceAuthorDictH;
+  #this is the place to generate feature dictionrauy and name pattern dictionary
+  ($HeaderH, $FeatureDictH, $SpaceAuthorDictH) = &FormFeaDict($HeaderH, $FeatureDictH);
+  #Prune features in Dictionary with DF < 3
+  $FeatureDictH = &PruneDict($FeatureDictH);
+
+  #prune features not in the pruned dict from the feature vector
+  foreach my $HeaNO (sort {$a <=> $b} keys %{$HeaderH}) {
+    foreach my $line(sort {$a <=> $b} keys %{$$HeaderH{$HeaNO}}) {
+      foreach my $fea(keys %{$$HeaderH{$HeaNO}{$line}{FeaVec}}) {
+	if (! $$FeatureDictH{$fea}{ID})  {
+	  delete ($$HeaderH{$HeaNO}{$line}{FeaVec}{$fea});
+	}
+      }
+
+      if ($$HeaderH{$HeaNO}{$line}{FeaVec} ne "") {
+	my $tmpFeaVec = "";
+	foreach my $fea(sort{$$FeatureDictH{$a}{ID} <=> $$FeatureDictH{$b}{ID}} keys %{$$HeaderH{$HeaNO}{$line}{FeaVec}}) {
+
+	  if ($norm) {
+	      #normalization
+	      $$HeaderH{$HeaNO}{$line}{FeaVec}{$fea} = sprintf("%.8f", $$HeaderH{$HeaNO}{$line}{FeaVec}{$fea}/$$FeatureDictH{$fea}{max});
+	  }
+
+	  $tmpFeaVec .= "$$FeatureDictH{$fea}{ID}\:$$HeaderH{$HeaNO}{$line}{FeaVec}{$fea} ";
+        }
+	$$HeaderH{$HeaNO}{$line}{SVMFeaVec} = "$tmpFeaVec";
+      }
+
+    }
+  }
+
+  my %NameSpaceTrainVecH = (); #a separate hash for later printing
+  my $Lcount = 0;
+  #Prune acordingly features
+  foreach my $HeaNO (sort {$a <=> $b} keys %{$HeaderH}) {
+      foreach my $line(sort {$a <=> $b} keys %{$$HeaderH{$HeaNO}}) {
+	  if (exists $$HeaderH{$HeaNO}{$line}{NamePattern}) {
+	      foreach my $CandidateNamePattern(keys %{$$HeaderH{$HeaNO}{$line}{NamePattern}}) {
+		  foreach my $fea(keys %{$$HeaderH{$HeaNO}{$line}{NamePattern}{$CandidateNamePattern}{SpaceNameVec}}) {
+		      if (! $$SpaceAuthorDictH{$fea}{ID})  {
+			  delete($$HeaderH{$HeaNO}{$line}{NamePattern}{$CandidateNamePattern}{SpaceNameVec}{$fea});
+		      }
+		  }
+
+		  #normalization
+		  if ($$HeaderH{$HeaNO}{$line}{NamePattern}{$CandidateNamePattern}{SpaceNameVec} ne "") {
+		      $Lcount++;
+		      my $tmpFeaVec = "$$HeaderH{$HeaNO}{$line}{NamePattern}{$CandidateNamePattern}{tag} ";
+		      my $tmpTextVec = "$$HeaderH{$HeaNO}{$line}{NamePattern}{$CandidateNamePattern}{tag} ";
+
+		      foreach my $fea(sort{$$SpaceAuthorDictH{$a}{ID} <=> $$SpaceAuthorDictH{$b}{ID}} keys %{$$HeaderH{$HeaNO}{$line}{NamePattern}{$CandidateNamePattern}{SpaceNameVec}}) {
+			  $$HeaderH{$HeaNO}{$line}{NamePattern}{$CandidateNamePattern}{SpaceNameVec}{$fea} = sprintf("%.8f", $$HeaderH{$HeaNO}{$line}{NamePattern}{$CandidateNamePattern}{SpaceNameVec}{$fea}/$$SpaceAuthorDictH{$fea}{max});
+			  $tmpFeaVec .= "$$SpaceAuthorDictH{$fea}{ID}\:$$HeaderH{$HeaNO}{$line}{NamePattern}{$CandidateNamePattern}{SpaceNameVec}{$fea} ";
+			  $tmpTextVec .= "$fea\:$$HeaderH{$HeaNO}{$line}{NamePattern}{$CandidateNamePattern}{SpaceNameVec}{$fea} ";
+		      }
+		      $NameSpaceTrainVecH{$Lcount}{SpaceNameVec}=$tmpFeaVec;
+		      $NameSpaceTrainVecH{$Lcount}{SpaceTextNameVec}=$tmpTextVec; #for debugging
+		  }
+	      }
+	  }
+      }
+  }
+
+  return($HeaderH, $FeatureDictH, $PuncAuthorDictH, $SpaceAuthorDictH, \%NameSpaceTrainVecH);
+}
+
+sub ContextTrainSys() {
+  my $FeatureDictH = shift;
+  my $HeaderH = shift;
+
+  foreach my $HeaNO (sort {$a <=> $b} keys %{$HeaderH}) {
+    #assign neighour line's tag
+    ($FeatureDictH, $$HeaderH{$HeaNO}) = &TrainAssignLineTag($FeatureDictH, $$HeaderH{$HeaNO});
+  }
+  return($FeatureDictH, $HeaderH);
+}
+
+#this is to write all the testing lines into one file to speed up
+sub LineClassify() {
+    my ($testp, $nowLoop, $baseline, $FeatureDictH,
+	$HeaderH, $tmpCacheVec, $SVMTmpResult) = @_;
+    my %memoryH = ();
+    my $GlobalLineNO = 0;
+
+    #step1: collect all test data and write into one file
+    #       keep a hash to record the global lineNO and the header no its local line no
+    # here is the file for all the testing data
+
+    #  foreach my $testHea(sort {$a <=> $b} keys %{$HeaderH}) {
+    if ($baseline) {
+	#Filter feature vector by Feature Dictionary
+	### $$HeaderH{$testHea} = &FormTestFeaVec($FeatureDictH, $$HeaderH{$testHea});
+	$HeaderH = &FormTestFeaVec($FeatureDictH, $HeaderH);
+    }else {
+	$HeaderH = &TestAssignLineTag($FeatureDictH, $HeaderH);
+    }
+
+    foreach my $LN(sort {$a <=> $b} keys %{$HeaderH}) {
+	if (! $baseline) {
+	    #To make the iteration correct, we should initialize $$HeaderH{$testHea} by removing all the single and multiple classes in the hash
+	    delete($$HeaderH{$LN}{PClass});
+	    delete($$HeaderH{$LN}{PSClsName});
+	    delete($$HeaderH{$LN}{PClsName});
+	}elsif ($baseline && ($$HeaderH{$LN}{FeaVec} ne "")) {
+	    #modify the feature vector(normalization)
+	    my $tmpFeaVec = "";
+	    foreach my $fea(sort{$$FeatureDictH{$a}{ID} <=> $$FeatureDictH{$b}{ID}} keys %{$$HeaderH{$LN}{FeaVec}}) {
+		if (exists ($$FeatureDictH{$fea}{ID})) {
+
+		    if ($norm) {
+			if ($debug) {
+			    if ($$FeatureDictH{$fea}{max} == 0) {
+				print STDERR "fea $fea has max value 0! \n";
+			    }
+			}
+			$$HeaderH{$LN}{FeaVec}{$fea} = sprintf("%.8f", $$HeaderH{$LN}{FeaVec}{$fea}/$$FeatureDictH{$fea}{max});
+		    }
+
+		    $tmpFeaVec .= "$$FeatureDictH{$fea}{ID}\:$$HeaderH{$LN}{FeaVec}{$fea} ";
+		}
+	    }
+
+	    $$HeaderH{$LN}{SVMFeaVec} = "$tmpFeaVec";
+
+	    #be carefull here!!
+	    if ($$HeaderH{$LN}{SVMFeaVec} eq "") {
+		if ($debug) {
+		    print STDERR "header($testHea) -- Line($LN) has a null feature vector ($$HeaderH{$testHea}{$LN}{RawContent}) \n";
+		}
+		next;
+	    }
+	}
+
+	$GlobalLineNO++;
+	$memoryH{$GlobalLineNO}{HeaNO} = $testHea;
+	$memoryH{$GlobalLineNO}{LocalLineNO} = $LN;
+    }
+ #  }
+
+    #step2:we print 15 files with labelled feature vectors
+    for my $clsNO(1 .. 15) {
+	my $testF = "$tmpCacheVec"."test"."$clsNO";
+	open(testFH, ">$testF") || die "SVMHeaderParse: could not open $testF to write: $!";
+	#      foreach my $HeaNO (sort {$a <=> $b} keys %{$HeaderH}) {
+	foreach my $LN(sort {$a <=> $b} keys %{$HeaderH}) {
+	    my $tag = 1; # just to conform to the format
+	    if ($baseline) {
+		print testFH "$tag $$HeaderH{$LN}{SVMFeaVec}\n";
+	    }else {
+		print testFH "$tag $$HeaderH{$LN}{ContextSVMFeaVec}\n";
+		#print "context feature vec is $$HeaderH{$HeaNO}{$LN}{ContextSVMFeaVec}\n";
+	    }
+	}
+	#     } # end of collecting all the testing data into a file
+	close(testFH);
+    }
+
+    #step3: SVM classify
+    for my $clsNO(1 .. 15) {
+	my $testF = "$tmpCacheVec"."test"."$clsNO";
+	my $mySVMResult = "$SVMTmpResult"."$clsNO";
+	my $SVMModelF;
+	my $printstr = "";
+	if ($baseline) {
+	    $printstr = "baseline";
+	    $SVMModelF = "$offlineD"."$clsNO"."Model"."fold"."$testp";
+	}else {
+	    $printstr = "context"."$nowLoop";
+	    $SVMModelF = "$offlineD"."$clsNO"."ContextModel"."fold"."$testp";
+	}
+#     print "$Classifier -v 0 $testF $SVMModelF $mySVMResult\n";
+#     print "classification result from fold($testp)-class($clsNO)-$printstr\:\n";
+	system("$Classifier -v 0 $testF $SVMModelF $mySVMResult");
+   }
+
+    #step4:Read all the result into a hash
+    my %SVMResultHash = ();
+    my %OrphanTagAssignHash = (); #This records the accuracy of assigned tags
+    my %NegMeanH = (); #record the mean of the negative value each classifier made
+    my %PosMinH = ();
+
+    for my $clsNO(1 .. 15) {
+	my $mySVMResult = "$SVMTmpResult"."$clsNO";
+	my $myLineNO = 0;
+
+	#initialize %PosMinH 's value
+	$PosMinH{$clsNO} = 100;
+
+	open(mySVMResultFH, "$mySVMResult") || die "SVMHeaderParse: could not open $mySVMResult to read: $!";
+	while (my $myline = <mySVMResultFH>) {
+	    $myline =~ s/^\s+//g;
+	    $myline =~ s/\s+$//g;
+	    if ($myline !~ /^\s*$/) {
+		$myLineNO++;
+		if ($debug) {
+		    print STDERR " current lineNo is $myLineNO and score for class $clsNO is $myline \n";
+		}
+		$SVMResultHash{$myLineNO}{$clsNO} = $myline;
+		if ($myline < 0) {
+		    $NegMeanH{$clsNO} += $myline;
+		}else {
+		    if ($PosMinH{$clsNO} > $myline) {
+			$PosMinH{$clsNO} = $myline;
+		    }
+		}
+	    }
+	}
+
+	if ($myLineNO < 1) {
+	    if ($debug) {
+		print STDERR "yahoo: $mySVMResult has myLineNO 0 \n";
+	    }
+	}else {
+	    $NegMeanH{$clsNO} = sprintf("%.8f", $NegMeanH{$clsNO}/$myLineNO);
+	}
+
+	close(mySVMResultFH);
+    }
+
+    my $PredTagbyMinNeg = 0;
+    my $PredValbyMinNeg = 100;
+    my $PredTagbyMinPos = 0;
+    my $PredValbyMinPos = 100;
+
+    #analyze the results from the hash and fill the Test Hash(HeaderH)
+    for my $myline(1 .. $GlobalLineNO) {
+	my @PredictTags = ();
+	my $minVal = 100;
+	my $CandidateTag = -1;
+	my $myHeaNO = $memoryH{$myline}{HeaNO};
+	my $myLineNO = $memoryH{$myline}{LocalLineNO};
+
+	for my $clsNO(1 .. 15) {
+	    my $myresult = $SVMResultHash{$myline}{$clsNO};
+	    #keep the classification results for multi-class line
+	    $$HeaderH{$myLineNO}{ClassifyResult}{$clsNO} = $myresult;
+	    if ($debug) {
+		print STDERR "\t\t result by class $clsNO -- $result \n";
+	    }
+	    my $myRelDiv = 10;
+
+	    if ($myresult > 0) {
+		push @PredictTags, $clsNO;
+	    }else {
+		$myRelDiv = sprintf("%.8f", $myresult/$NegMeanH{$clsNO});
+		if ($myRelDiv < $minVal) {
+		    $minVal = $myRelDiv;
+		    $CandidateTag = $clsNO;
+		}
+		if ( (0 - $myresult) < $PredValbyMinNeg) {
+		    $PredValbyMinNeg = -$myresult;
+		    $PredTagbyMinNeg = $clsNO;
+		}
+		if (($PosMinH{$clsNO}- $myresult) < $PredValbyMinPos) {
+		    $PredValbyMinPos = $PosMinH{$clsNO}- $myresult;
+		    $PredTagbyMinPos = $clsNO;
+		}
+	    }
+	}
+	#Assign ONLY class nearest to the hyperplane to the orphan point
+	if ($#PredictTags < 0) {
+	    push @PredictTags, $CandidateTag;
+	    $OrphanTagAssignHash{TotalLineNum}++;
+	}
+
+	#Fill the hash with the classification result
+	if ($#PredictTags eq 0) {
+	    $$HeaderH{$myLineNO}{PClass} = "s";
+	    $$HeaderH{$myLineNO}{PSClsName} = $PredictTags[0];
+	}elsif ($#PredictTags > 0)  {
+	    $$HeaderH{$myLineNO}{PClass} = "m";
+	    # the multi tags predicted in one line has no sense of the order
+	    for my $i(0 .. $#PredictTags) {
+		$$HeaderH{$myLineNO}{PClsName}{$PredictTags[$i]} = 1;
+		if ($debug) {
+		    print STDERR "hea($myHeaNO)-- line($myLineNO) is classified as multi-class $PredictTags[$i] \n";
+		}
+	    }
+	}else { #impossible
+	    if ($debug) {
+		print STDERR "hea($myHeaNO)-- line($myLineNO) is orphan\n";
+	    }
+	}
+    }
+    return($HeaderH);
+}
+
+
+#this is to
+#(1) populate the predicted items(done in the LineClassify)
+#(2) Extract related information from multi-author line and multi-classline
+#all information to be extracted comes from {Pchunk}
+#all word distribution information comes from {Pline} word dist.;
+
+sub InfoExtract() {
+    my $testp = shift;
+    my $TestH = shift;
+    my $PuncAuthorDictH = shift;
+    my $SpaceAuthorDictH = shift;
+    my $SVMNameSpaceModel = shift;
+    my $tmpCacheVec = shift;
+    my $SVMTmpResult = shift;
+
+#  foreach my $testHea(sort {$a <=> $b} keys %{$TestH}) {
+    foreach my $LN(sort {$a <=> $b} keys %{$TestH}) {
+	if ($$TestH{$LN}{'PClass'} eq "s") { # single class
+	    if  ($$TestH{$LN}{PSClsName} ne '2') { #non-author single class
+		$$TestH{$LN}{Pchunk}{ChunkCounter}++;
+		my $ChunkPos = $$TestH{$LN}{Pchunk}{ChunkCounter};
+		$$TestH{$LN}{Pchunk}{$ChunkPos}{cls} = $$TestH{$LN}{PSClsName};
+		$$TestH{$LN}{Pchunk}{$ChunkPos}{content} = $$TestH{$LN}{PureText};
+	    }else {
+		if ($$TestH{$LN}{SClsWordCount} < 4) { #obvious single name
+		    $$TestH{$LN}{Pchunk}{ChunkCounter}++;
+		    my $ChunkPos = $$TestH{$LN}{Pchunk}{ChunkCounter};
+		    $$TestH{$LN}{Pchunk}{$ChunkPos}{cls} = 2;
+		    $$TestH{$LN}{Pchunk}{$ChunkPos}{content} = $$TestH{$LN}{PureText};
+		}else { #multi-authors
+		    my $Tline = $$TestH{$LN}{RawContent};
+		    $Tline =~ s/<(\/)*author>//g;
+		    if ($debug) {
+			print STDERR "predicted Multi-Author line -- $Tline \n";
+		    }
+		    my $NamePunc = 0;
+		    #judge this is punctuated line or pure text-space
+		    if (($$TestH{$LN}{PureText} =~ /([^\p{IsLower}\p{IsUpper}\s+\-\.\d+])/) || ($$TestH{$LN}{PureText} =~ /\band\b/i))  {
+			#multi-class needs while ... $punc++;
+			$NamePunc = 1;
+		    }else {
+			$NamePunc = 0;
+		    }
+
+		    if ($NamePunc) {
+			#Heuristics bases separation based on features learned.
+			if (($$TestH{$LN}{PureText} =~ /Jr|Dr/) && ($$TestH{$LN}{SClsWordCount} <5)) {
+			    #this is only one name
+			    $$TestH{$LN}{Pchunk}{ChunkCounter}++;
+			    my $ChunkPos = $$TestH{$LN}{Pchunk}{ChunkCounter};
+			    $$TestH{$LN}{Pchunk}{$ChunkPos}{cls} = 2;
+			    $$TestH{$LN}{Pchunk}{$ChunkPos}{content} = $$TestH{$LN}{PureText};
+			}else {
+			    my $nameStr = $$TestH{$LN}{PureText};
+			    $nameStr =~ s/^\s+//g;
+			    $nameStr =~ s/\s+$//g;
+			    my @GuessedNames = split(/\,|\&|and/, $nameStr);
+			    for my $i(0 .. $#GuessedNames) {
+				#chunk starts from 1
+				$GuessedNames[$i] =~ s/^\s+//g;
+				$GuessedNames[$i] =~ s/\s+$//g;
+				if ($GuessedNames[$i] !~ /^\s*$/) {
+				    my @Nameparts = split(/\s+/, $GuessedNames[$i]);
+				    if ($#Nameparts < 3) {
+					$$TestH{$LN}{Pchunk}{ChunkCounter}++;
+					my $ChunkPos = $$TestH{$LN}{Pchunk}{ChunkCounter};
+					$$TestH{$LN}{Pchunk}{$ChunkPos}{cls} = 2;
+					$$TestH{$LN}{Pchunk}{$ChunkPos}{content} = $GuessedNames[$i];
+				    }else {
+					#space separated names [name1 name2 name3 and name4]
+					my $PredictedNames = &HeaderParse::API::NamePatternMatch::NamePatternMatch($GuessedNames[$i]);
+					if ($#$PredictedNames < 1){
+					    #only 1/0 reasonable name pattern, take it
+					    $$TestH{$LN}{Pchunk}{ChunkCounter}++;
+					    my $ChunkPos = $$TestH{$LN}{Pchunk}{ChunkCounter};
+					    $$TestH{$LN}{Pchunk}{$ChunkPos}{cls} = 2;
+					    $$TestH{$LN}{Pchunk}{$ChunkPos}{content} = $GuessedNames[$i];
+					}else { #classify to predict
+					    my $BestNamePattern = &PredictBestNamePattern($PredictedNames, $SVMNameSpaceModel, $SpaceAuthorDictH, $tmpCacheVec, $SVMTmpResult);
+					    my @names = split(/<>/, $BestNamePattern);
+					    for my $i(0 .. $#names) {
+						$$TestH{$LN}{Pchunk}{ChunkCounter}++;
+						my $ChunkPos = $$TestH{$LN}{Pchunk}{ChunkCounter};
+						$$TestH{$LN}{Pchunk}{$ChunkPos}{cls} = 2;
+						$$TestH{$LN}{Pchunk}{$ChunkPos}{content} = $names[$i];
+					    }
+					}
+				    }
+				}
+			    }
+			}
+		    }else {
+			#name Space
+			my $nameStr = $$TestH{$LN}{PureText};
+			my $PredictedNames = &HeaderParse::API::NamePatternMatch::NamePatternMatch($nameStr);
+			if ($#$PredictedNames < 1){
+			    #only 1/0 reasonable name pattern, take the parser-decided chunks
+			    my $tmp_name_container = $$PredictedNames[0];
+			    if ($#$tmp_name_container > 0) {
+				for my $kk(0 .. $#$tmp_name_container) {
+				    $$TestH{$LN}{Pchunk}{ChunkCounter}++;
+				    my $ChunkPos = $$TestH{$LN}{Pchunk}{ChunkCounter};
+				    $$TestH{$LN}{Pchunk}{$ChunkPos}{cls} = 2;
+				    $$TestH{$LN}{Pchunk}{$ChunkPos}{content} = $$tmp_name_container[$kk];
+				}
+			    }else {
+				#this branch is original
+				$$TestH{$LN}{Pchunk}{ChunkCounter}++;
+				my $ChunkPos = $$TestH{$LN}{Pchunk}{ChunkCounter};
+				$$TestH{$LN}{Pchunk}{$ChunkPos}{cls} = 2;
+				$$TestH{$LN}{Pchunk}{$ChunkPos}{content} = $nameStr;
+			    }
+			}else {
+			    #classify to predict
+			    my $BestNamePattern = &PredictBestNamePattern($PredictedNames, $SVMNameSpaceModel, $SpaceAuthorDictH, $tmpCacheVec, $SVMTmpResult);
+			    my @names = split(/<>/, $BestNamePattern);
+			    for my $i(0 .. $#names) {
+				$$TestH{$LN}{Pchunk}{ChunkCounter}++;
+				my $ChunkPos = $$TestH{$LN}{Pchunk}{ChunkCounter};
+				$$TestH{$LN}{Pchunk}{$ChunkPos}{cls} = 2;
+				$$TestH{$LN}{Pchunk}{$ChunkPos}{content} = $names[$i];
+			    }
+			}
+		    }
+		}
+	    }
+	    #multiple class
+	}elsif ($$TestH{$LN}{PClass} eq "m"){
+	    my (%TagH, $emailChunkH, $URLChunkH, @ArrayofHash);
+	    #get a hash of all tags
+	    foreach my $tag(keys %{$$TestH{$LN}{PClsName}}) {
+		$TagH{counter}++;
+		$TagH{$tag}++;
+	    }
+	    my ($PuncNum, $SepH, $component) = &GetSeparatorIndex($$TestH{$LN}{PureText});
+	    #Preprocess -- extract email and URL out
+	    if ($$TestH{$LN}{PClsName}{6}) {
+		#component has holes of "-1", after extracting emailchunk out
+		($emailChunkH, $component) = &LocateEmailFromComponent($component);
+		delete($TagH{6});
+		$TagH{counter}--;
+		push @ArrayofHash, $emailChunkH;
+	    }
+	    if ($$TestH{$LN}{PClsName}{12}) {
+		($URLChunkH, $component) = &LocateURLFromComponent($component);
+		delete($TagH{12});
+		$TagH{counter}--;
+		push @ArrayofHash, $URLChunkH;
+	    }
+
+	    if($TagH{counter} <1){ #no additional class
+		#exception: what if still text left ???????
+		$$TestH{$LN} = &FillChunkH($$TestH{$LN},$component, \@ArrayofHash);
+		#tag each word
+	    }elsif ($TagH{counter} == 1){
+		#only one class left ..
+		my $lastTag = "";
+		foreach my $tag(keys %TagH) {
+		    if ($tag ne "counter") {
+			$lastTag = $tag;
+		    }
+		}
+		#Get the rest possible chunks separated by the email and URL
+		my $UnIdentifiedChunk = &LocateUnIdentifiedChunk($component);
+		#Tag all the test chunk as the only left class
+		foreach my $chunkNO(sort{$a<=>$b} keys %{$UnIdentifiedChunk}) {
+		    $$UnIdentifiedChunk{$chunkNO}{cls} = $lastTag;
+		}
+		push @ArrayofHash, $UnIdentifiedChunk; #or\%myHash--must be pointer
+		#fill in the TestH chunk in a ordered way and tag each word
+		$$TestH{$LN} = &FillChunkH($$TestH{$LN}, $component, \@ArrayofHash);
+		# two class module
+	    }elsif ($TagH{counter} == 2) {
+		#needs maping!
+		my @TagsArray = ();
+		foreach my $mytag(sort keys %TagH) {
+		    if ($mytag ne "counter") {
+			push @TagsArray, $mytag;
+		    }
+		}
+
+		my $UnIdentifiedChunk = &LocateUnIdentifiedChunk($component);
+		my $chunk1start = $$UnIdentifiedChunk{1}{startPos};
+		my $chunk1end = $$UnIdentifiedChunk{1}{endPos};
+		my $IdentifiedChunk;
+		#continuous
+		if ($$UnIdentifiedChunk{counter} == 1) {
+		    my $offset;
+		    my $newComponent = $component;
+		    my $newSepH = $SepH;
+		    if (($chunk1start == 0) &&  ($chunk1end == $#$component)) {
+			$offset = 0;
+		    }else {
+			$offset = $chunk1start;
+			#adjust $component and $SepH
+			$newComponent = ();
+			for my $tmpi($chunk1start .. $chunk1end) {
+			    $$newComponent[$tmpi-$offset] = $$component[$tmpi];
+			}
+
+			foreach my $tmpSep(sort keys %{$newSepH}) {
+			    if (($tmpSep >= $chunk1start) && ($tmpSep <= $chunk1end)) {
+				my $newSep =  $tmpSep -  $offset;
+				$$newSepH{$newSep} = $$newSepH{$tmpSep};
+			    }
+			    delete($$newSepH{$tmpSep});
+			}
+		    }
+
+		    if ($PuncNum > 1) {
+			$IdentifiedChunk = &Cont2ClassChunking($testp, \@TagsArray, "punc", $newSepH, $newComponent, $FeatureDictH, $tmpCacheVec, $SVMTmpResult);
+		    }else {
+			$IdentifiedChunk = &Cont2ClassChunking($testp, \@TagsArray, "space", $newSepH, $newComponent, $FeatureDictH, $tmpCacheVec, $SVMTmpResult);
+		    }
+		    #adjust back $chunk
+		    if ($offset > 0) {
+			foreach my $tmpi(sort keys %{$IdentifiedChunk}) {
+			    $$IdentifiedChunk{$tmpi}{startPos} += $offset;
+			    $$IdentifiedChunk{$tmpi}{endPos} += $offset;
+			}
+		    }
+		    push @ArrayofHash, $IdentifiedChunk;
+		    $$TestH{$LN} = &FillChunkH($$TestH{$LN}, $component, \@ArrayofHash);
+		}elsif ($$UnIdentifiedChunk{counter} == 2) { #discrete
+		    $IdentifiedChunk = &Disc2ClassChunking_2chunk($testp, \@TagsArray, $UnIdentifiedChunk, $FeatureDictH, $tmpCacheVec, $SVMTmpResult);
+		    push @ArrayofHash, $IdentifiedChunk;
+		    $$TestH{$LN} = &FillChunkH($$TestH{$LN}, $component, \@ArrayofHash);
+		}elsif ($$UnIdentifiedChunk{counter} > 2) { #disc
+		    if ($debug) {
+			print STDERR "2 classes with 3+ chunks\n";
+		    }
+		}
+		#see 3 and 4 as one class
+	    }elsif (($TagH{counter} == 3) && $TagH{3} && $TagH{4}) {
+		#tag array includes only 4 and the other tag
+		my @TagsArray = ();
+		foreach my $mytag(sort keys %TagH) {
+		    if (($mytag ne "3") && ($mytag ne "4") && ($mytag ne "counter")) {
+			push @TagsArray, $mytag;
+		    }
+		}
+		push @TagsArray, 4;
+
+		my $UnIdentifiedChunk = &LocateUnIdentifiedChunk($component);
+		my $chunk1start = $$UnIdentifiedChunk{1}{startPos};
+		my $chunk1end = $$UnIdentifiedChunk{1}{endPos};
+
+		my $IdentifiedChunk;
+		my $startPos34 = 0;
+		my $endPos34 = 0;
+		#continuous
+		if ($$UnIdentifiedChunk{counter} == 1) {
+		    my $offset;
+		    my $newComponent = $component;
+		    my $newSepH = $SepH;
+
+		    if (($chunk1start == 0) &&  ($chunk1end == $#$component)) {
+			$offset = 0;
+		    }else {
+			$offset = $chunk1start;
+			#adjust $component and $SepH
+			$newComponent = ();
+			for my $tmpi($chunk1start .. $chunk1end) {
+			    $$newComponent[$tmpi-$offset] = $$component[$tmpi];
+			}
+
+			foreach my $tmpSep(sort keys %{$newSepH}) {
+			    if (($tmpSep >= $chunk1start) && ($tmpSep <= $chunk1end)) {
+				my $newSep =  $tmpSep -  $offset;
+				$$newSepH{$newSep} = $$newSepH{$tmpSep};
+			    }
+			    delete($$newSepH{$tmpSep});
+			}
+		    }
+
+		    #find the boundary between 34 and the other tag
+		    if ($PuncNum > 1) {
+			$IdentifiedChunk = &Cont2ClassChunking($testp, \@TagsArray, "punc", $newSepH, $newComponent, $FeatureDictH, $tmpCacheVec, $SVMTmpResult);
+		    }else {
+			$IdentifiedChunk = &Cont2ClassChunking($testp, \@TagsArray, "space", $newSepH, $newComponent, $FeatureDictH, $tmpCacheVec, $SVMTmpResult);
+		    }
+
+		    #adjust back $chunk
+		    #get the position of the 3 4
+		    foreach my $tmpi(sort keys %{$IdentifiedChunk}) {
+			if ($offset > 0) {
+			    $$IdentifiedChunk{$tmpi}{startPos} += $offset;
+			    $$IdentifiedChunk{$tmpi}{endPos} += $offset;
+			}
+			if ($$IdentifiedChunk{$tmpi}{cls} eq 4) {
+			    $startPos34 = $$IdentifiedChunk{$tmpi}{startPos}; #absolute pos
+			    $endPos34 = $$IdentifiedChunk{$tmpi}{endPos};
+			    delete($$IdentifiedChunk{$tmpi});
+			}
+		    }
+		    push @ArrayofHash, $IdentifiedChunk;
+
+		}else { #if 2 discrete chunks
+		    $IdentifiedChunk = &Disc2ClassChunking_2chunk($testp, \@TagsArray, $UnIdentifiedChunk, $FeatureDictH, $tmpCacheVec, $SVMTmpResult);
+		    foreach my $tmpi(sort keys %{$IdentifiedChunk}) {
+			if ($$IdentifiedChunk{$tmpi}{cls} eq 4) {
+			    $startPos34 = $$IdentifiedChunk{$tmpi}{startPos};
+			    $endPos34 = $$IdentifiedChunk{$tmpi}{endPos};
+			    delete($$IdentifiedChunk{$tmpi});
+			}
+		    }
+		    push @ArrayofHash, $IdentifiedChunk;
+		}
+
+		#find the boundary between 3 and 4
+		my $newComponent = (); #modified by Hui 03/19
+		my $newSepH = $SepH;
+		my $newPuncNum = 0;
+		my $offset = $startPos34;
+		for (my $tmpi=$startPos34; $tmpi<=$endPos34; $tmpi++) {
+		    #modified by Hui 03/19/03 -$offset
+		    $$newComponent[$tmpi-$offset] = $$component[$tmpi];
+		    if ($$newComponent[$tmpi-$offset] =~ /^\W+$/) {
+			$newPuncNum++;
+		    }
+		}
+
+		if ($newPuncNum > 1) {
+		    foreach my $tmpSep(sort keys %{$$newSepH{punc}}) {
+			if (($tmpSep >= $startPos34) && ($tmpSep <= $endPos34)) {
+			    my $newSep =  $tmpSep -  $offset;
+			    $$newSepH{punc}{$newSep} = $$newSepH{punc}{$tmpSep};
+			}
+			delete($$newSepH{punc}{$tmpSep});
+		    }
+		}else {
+		    foreach my $tmpSep(sort keys %{$$newSepH{space}}) {
+			if (($tmpSep >= $startPos34) && ($tmpSep <= $endPos34)) {
+			    my $newSep =  $tmpSep -  $offset;
+			    $$newSepH{space}{$newSep} = $$newSepH{space}{$tmpSep};
+			}
+			delete($$newSepH{space}{$tmpSep});
+		    }
+		}
+
+		my @NewTagsArray = ();
+		push @NewTagsArray, 3;
+		push @NewTagsArray, 4;
+		if ($newPuncNum > 1) {
+		    $IdentifiedChunk = &Cont2ClassChunking($testp, \@NewTagsArray, "punc", $newSepH, $newComponent, $FeatureDictH, $tmpCacheVec, $SVMTmpResult);
+		}else {
+		    $IdentifiedChunk = &Cont2ClassChunking($testp, \@NewTagsArray, "space", $newSepH, $newComponent, $FeatureDictH, $tmpCacheVec, $SVMTmpResult);
+		}
+
+		#adjust back $chunk
+		if ($offset > 0) {
+		    foreach my $tmpi(sort keys %{$IdentifiedChunk}) {
+			$$IdentifiedChunk{$tmpi}{startPos} += $offset;
+			$$IdentifiedChunk{$tmpi}{endPos} += $offset;
+		    }
+		}
+		push @ArrayofHash, $IdentifiedChunk;
+		$$TestH{$LN} = &FillChunkH($$TestH{$LN}, $component, \@ArrayofHash);
+	    }elsif ($TagH{counter} > 2) { #3+ cases.
+		#consider about 3 discrete chunks for 3 tags????
+		if ($debug) {
+		    print STDERR "do not care yet -- here is the case for 3+ classes after preprocessing \n";
+		    #find the most likely position and expand to arround some(like 3) words
+		}
+	    }
+	}
+    }
+#  }
+
+    return($TestH);
+}
+
+
+sub ExportInfo(){
+    my $TestH = shift;
+    my $outF = "output.txt";
+    open(WRITER, ">$outF") || die "SVMHeaderParse: could not open $outF to write: $!";
+    #  foreach my $testHea(sort {$a <=> $b} keys %{$TestH}) {
+    print WRITER "headerno($testHea) -- ";
+    foreach my $LN(sort {$a <=> $b} keys %{$TestH}) {
+	print WRITER "lineno($LN)\: \n ";
+	foreach my $chunk(sort {$a <=> $b} keys %{$$TestH{$LN}{Pchunk}}) {
+	    if ($chunk ne "ChunkCounter") {
+		print WRITER "\t chunk($chunk) -- class($$TestH{$LN}{Pchunk}{$chunk}{cls} <> content($$TestH{$LN}{Pchunk}{$chunk}{content} \n";
+	    }
+	}
+    }
+    # }
+    close(WRITER);
+}
+
+
+sub ExportRDF(){
+    my $TestH = shift;
+    my $str='';
+    my $tempStr='';
+    foreach my $LN(sort {$a <=> $b} keys %{$TestH}) {
+	foreach my $chunk(sort {$a <=> $b} keys %{$$TestH{$LN}{Pchunk}}) {
+	    my $tag = $InverseTagMap{$$TestH{$LN}{Pchunk}{$chunk}{cls}};
+	    my $content = $$TestH{$LN}{Pchunk}{$chunk}{content};
+	    if ($content =~ /\w+/) {
+		$str .="<$tag>$content</$tag>\n";
+#                    if($tag =~/(url|note|date|abstract|intro|keyword|web|degree|pubnum|page)/){
+#                      $tempStr .= "\n<cs_header:$tag>$content<\/cs_header:$tag>";
+#                        }
+	    }
+	}
+    }
+
+#    print "RDF:\n\n $str\n";
+#    print "$str\n";
+    $rXML = &HeaderParse::API::AssembleXMLMetadata::assemble(\$str);
+    return $rXML;
+}
+
+
+#Basic function: popuate information from line -- feature vector and class assignment and name patterns.
+#no dictionary would be formed here
+sub PopulateLineInfo4Header_unit() {
+    my $HeaderH = shift;
+    my %curState = ();
+
+    foreach my $line(sort {$a <=> $b} keys %{$HeaderH}) {
+	my $PureTextLine = $$HeaderH{$line}{RawContent};
+	$PureTextLine =~ s/(\<)*\<(\/)*(\w+)\>(\>)*/ /g; # remove the tags
+	$PureTextLine =~ s/\+L\+//g;
+	$PureTextLine =~ s/^\s+//g;
+	$PureTextLine =~ s/\s+$//g;
+	#should make punctuation separate!
+	$$HeaderH{$line}{PureText} = &SeparatePunc($PureTextLine);
+	$$HeaderH{$line}{FeaVec} = &LineFeatureRepre($$HeaderH{$line}{PureText});
+	#add the position of the line here!!!!
+	$$HeaderH{$line}{FeaVec}{Clinepos} = $line;
+	my $textFeaVec = "";
+	foreach my $fea(keys %{$$HeaderH{$line}{FeaVec}}) {
+	    if($$HeaderH{$line}{FeaVec}{$fea} == 0) {
+		delete ($$HeaderH{$line}{FeaVec}{$fea});
+	    }else {
+		$textFeaVec .= "$fea($$HeaderH{$line}{FeaVec}{$fea})  ";
+	    }
+	}
+	$$HeaderH{$line}{TextFeaVec} = $textFeaVec; # for read and debug
+
+	#assign class tag to each line -- not separator <<sep>><</sep>> here
+	if ($$HeaderH{$line}{RawContent} =~ /([^\<]+|(^\s*))\<(\/)*(\w+)\>($|[^\<]+)/) {
+	    %curState = ();
+	    my $tmpIndex = 0; # the order of this tag showed up last time
+	    my $preTag = -1;
+	    my $mul = 0;
+	    while ($$HeaderH{$line}{RawContent} =~ /([^\<]+|(^\s*))\<(\/)*(\w+)\>($|[^\<]+)/g) {
+		$tmpIndex++;
+		my $tmptag = $4;
+		if (($preTag > 0) && ($preTag ne $tagMap{$tmptag})) {
+		    $mul = 1;
+		}
+		$curState{$tagMap{$tmptag}} = $tmpIndex;
+		$preTag = $tagMap{$tmptag};
+	    }
+
+	    if ($mul) {
+		$$HeaderH{$line}{TClass} = "m";
+		my $order = 1;
+		foreach my $tag(sort {$curState{$a} <=> $curState{$b}} keys %curState)         {
+		    $$HeaderH{$line}{MClsName}{$tag} = $order;
+		    $order++;
+		}
+
+		#represent the class distribution only for this multi-class case.
+		my $Tline = $$HeaderH{$line}{RawContent};
+		#main purpose is to combine </phone><email> as one <s>
+		$Tline =~ s/\<(\/)*(\w+)\>/<s>/g; #replace the tags with <s>
+		$Tline =~ s/^\s*<s>\s*//g;
+		$Tline =~ s/\s*<s>\s*$//g;
+		$Tline =~ s/<s>\s*<s>/<s>/g;
+		$Tline =~ s/\s+/ /g;
+
+		$Tline = &SeparatePunc($Tline);
+
+		while ($Tline =~ /(\s+(\W+)\s+<s>)/g) {
+		    my $whole = $1;
+		    my $punc = $2;
+		    $punc =~ s/^\s+//g;
+		    $punc =~ s/\s+$//g;
+
+		    if ($punc eq "\|") {
+			$Tline =~ s/\|/\!\!\!/g;
+			$whole =~ s/\|/\!\!\!/g;
+		    }
+		    $Tline =~ s/$whole/<<sep>>$punc<<\/sep>>/g; #only once no "g"
+		    if ($punc eq "\|") {
+			$Tline =~ s/\!\!\!//g;
+			$whole =~ s/\!\!\!//g;
+		    }
+		}
+		while ($Tline =~ /(<s>\s+(\W+)\s+)/g) {
+		    my $whole = $1;
+		    my $punc = $2;
+		    $punc =~ s/^\s+//g;
+		    $punc =~ s/\s+$//g;
+		    if ($punc eq "\|") {
+			$Tline =~ s/\|/\!\!\!/g;
+			$whole =~ s/\|/\!\!\!/g;
+		    }
+		    $Tline =~ s/$whole/<<sep>>$punc<<\/sep>>/g; #only once no "g"
+		    if ($punc eq "\|") {
+			$Tline =~ s/\!\!\!/\|/g;
+			$whole =~ s/\!\!\!/\|/g;
+		    }
+		}
+		$Tline =~ s/<s>/<<sep>><<\/sep>>/g;
+		my ($PuncNum, $SepH, $component) = &GetSeparatorIndex($Tline);
+		#Populate Truth Hash by the chunk and word-class distribution
+		$$HeaderH{$line} = &AssignWordTagFromChunk($$HeaderH{$line}, $SepH, $component);
+	    }else {
+		$$HeaderH{$line}{TClass} = "s";
+		my @Tarr = split(/\s+/, $PureTextLine);
+		$$HeaderH{$line}{SClsWordCount} = $#Tarr +1;
+		foreach my $tag(sort {$curState{$a} <=> $curState{$b}} keys %curState) {
+		    $$HeaderH{$line}{SClsName} = $tag;
+		}
+
+		#Fill in the word-class distribution for single class line
+		my $lineContent = &SeparatePunc($$HeaderH{$line}{PureText});
+		my @wordArray = split(/\s+/, $lineContent);
+		undef $lineContent;
+
+		$$HeaderH{$line} = &AssignWordTag4SingleClassLine("truth", $$HeaderH{$line}{SClsName}, $$HeaderH{$line}, \@wordArray);
+
+		#but only multi-author has multiple chunks
+		#all reasonable name patterns for space separated names
+		#feature vec for each space namepatterns and puncutation separators
+		#Test/prediction will base on the predicted line tag in another module
+
+		#single author
+		if ($$HeaderH{$line}{SClsName} eq "2") {
+		    #From Truth
+		    if ($$HeaderH{$line}{RawContent} !~ /<<sep>>/) {
+			#could we save space by indicating the pure text directly
+			$$HeaderH{$line}{Tchunk}{$i}{cls} = 2;
+			$$HeaderH{$line}{Tchunk}{$i}{content} = $$HeaderH{$line}{PureText};
+		    #multiple authors
+		    }else {
+			my $Tline = $$HeaderH{$line}{RawContent};
+			$Tline =~ s/<(\/)*author>//g;
+
+			my ($PuncNum, $SepH, $component) = &GetSeparatorIndex($Tline);
+			my $nameStr = join(" ", @$component);
+
+			#judge this is punctuated line or pure text-space
+			if ($$HeaderH{$line}{PureText} =~ /([^\p{IsLower}\p{IsUpper}\s+\-\.\d+])|(\W+and\W+)/ig)  {
+			    #multi-class needs while ... $punc++;
+			    $$HeaderH{$line}{NamePunc} = 1;
+			}else {
+			    $$HeaderH{$line}{NameSpace} = 1;
+			}
+
+			#{NamePuncFeaVec} and {NameSpaceFeaVec} based on number of puncs (>2)
+			#{MulClsPuncFeaVec}
+
+			######common to both name space and name punc ######
+			my $TrueNames = &HeaderParse::API::NamePatternMatch::GetTrueName($nameStr);
+			for my $i(0 .. $#$TrueNames) {
+			    my $j = $i+1; #chunk should start from 1
+			    $$HeaderH{$line}{Tchunk}{$j}{cls} = 2;
+			    $$HeaderH{$line}{Tchunk}{$j}{content} = "$$TrueNames[$i]";
+			}
+			################################################
+
+			if ($$HeaderH{$line}{NamePunc}) {
+			}else {
+			    my $PredictedNames = &HeaderParse::API::NamePatternMatch::NamePatternMatch($nameStr);
+			    if ($#$PredictedNames < 1) {
+				#only one pattern -- do not fill name pattern
+			    }else {
+				my $TrueIndex = &HeaderParse::API::NamePatternMatch::Duplicate($TrueNames, $PredictedNames);
+				#must solve the problem
+				if ($TrueIndex eq "-1") {
+				    if ($debug) {
+					print STDERR "here the true name($TrueNames) is null from the line $content \n";
+				    }
+				}else {
+				    #populate all reasonable name patterns
+				    for my $i(0 .. $#$PredictedNames) {
+					my $candidateName = "";
+					for my $j(0 .. $#{$$PredictedNames[$i]}) {
+					    if ($$PredictedNames[$i][$j]) {
+						$candidateName .= "$$PredictedNames[$i][$j]<>";
+					    }
+					}
+					#    print "candidate name\: $candidateName ";
+					$$HeaderH{$line}{NamePattern}{$candidateName}{content} = $candidateName;
+					($$HeaderH{$line}{NamePattern}{$candidateName}{SpaceNameVec}) = &SpaceNameLnFeaRepre_unit($candidateName);
+					if ($i eq $TrueIndex) {
+					    $$HeaderH{$line}{NamePattern}{$candidateName}{tag} = 1;
+					}else {
+					    $$HeaderH{$line}{NamePattern}{$candidateName}{tag} = -1;
+					}
+				    }
+				}
+			    }
+			}
+		    }
+		}
+	    }
+	}else { #if there is no explicit tag for this line, this line only belongs to the last class of the previous line
+	    my $tmpI = 0;
+	    foreach my $state (sort {$curState{$b} <=> $curState{$a}} keys %curState) {
+		if ($tmpI > 0) {
+		    delete ($curState{$state});
+		} #only keep the last tag
+		$tmpI++;
+	    }
+	    $$HeaderH{$line}{TClass} = "s";
+	    foreach my $tag(sort {$curState{$a} <=> $curState{$b}} keys %curState) {
+		$$HeaderH{$line}{SClsName} = $tag;
+	    }
+	}
+    }
+
+    return($HeaderH);
+}
+
+
+sub VectorizeUnknownHeaderLine () {
+  my $HeaderH = shift;
+
+  my %curState = ();
+  foreach my $line(sort {$a <=> $b} keys %{$HeaderH}) {
+      my $PureTextLine = $$HeaderH{$line}{RawContent};
+#      print "LINE $line: $PureTextLine\n";
+      $PureTextLine =~ s/^\s+//g;
+      $PureTextLine =~ s/\s+$//g;
+      #should make punctuation separate!
+      $$HeaderH{$line}{PureText} = &SeparatePunc($PureTextLine);
+
+      my @Tarr = split(/\s+/, $PureTextLine);
+      $$HeaderH{$line}{SClsWordCount} = $#Tarr +1;
+      $$HeaderH{$line}{FeaVec} = &LineFeatureRepre($$HeaderH{$line}{PureText});
+#      foreach my $key (keys %{$$HeaderH{$line}{FeaVec}}) {
+#	  print "$key :: ".${$$HeaderH{$line}{FeaVec}}{$key}."\n";
+#      }
+#      print "\n";
+      #add the position of the line here!!!!
+      $$HeaderH{$line}{FeaVec}{Clinepos} = $line;
+
+      my $textFeaVec = "";
+      foreach my $fea(keys %{$$HeaderH{$line}{FeaVec}}) {
+	  if($$HeaderH{$line}{FeaVec}{$fea} == 0) {
+	      delete ($$HeaderH{$line}{FeaVec}{$fea});
+	  }else {
+	      $textFeaVec .= "$fea($$HeaderH{$line}{FeaVec}{$fea})  ";
+	  }
+      }
+      $$HeaderH{$line}{TextFeaVec} = $textFeaVec; # for read and debug
+  }
+
+  return($HeaderH);
+}
+
+
+#training data are assigned the true neighbour lines' tag
+sub TrainAssignLineTag() {
+    my $FeatureDictH = shift;
+    my $HeaderH = shift;
+    my %curState = ();
+
+    foreach my $line(sort {$a <=> $b} keys %{$HeaderH}) {
+	my $PC = 1; # 0 means the tag for current line (which might be useful)
+	my $Pline = $line - $PC;
+	while (($PC < 5) && ($Pline > 0)) { #previous line
+	    if (exists $$HeaderH{$Pline}{TClass}) {
+		if ($$HeaderH{$Pline}{TClass} eq "s") {
+		    my $ContextFea = "P"."$PC"."$$HeaderH{$Pline}{SClsName}";
+		    if (! $$FeatureDictH{$ContextFea}{ID}) {
+			$$FeatureDictH{FeatureCounter}++;
+			$$FeatureDictH{$ContextFea}{ID} = $$FeatureDictH{FeatureCounter};
+			$$FeatureDictH{$ContextFea}{max} = 0.5;
+		    }
+
+		    if ($$FeatureDictH{$ContextFea}{ID}) {
+			$$HeaderH{$line}{ContextFeaVec}{$ContextFea} = 0.5;
+			$$FeatureDictH{$ContextFea}{DF}++;
+		    }
+		}else { # consider the order of the tag
+		    foreach my $tag(sort {$$HeaderH{$Pline}{MClsName}{$a} <=> $$HeaderH{$Pline}{MClsName}{$b}} keys %{$$HeaderH{$Pline}{MClsName}}){
+			my $ContextFea = "P"."$PC"."$tag";
+			if (! $$FeatureDictH{$ContextFea}{ID}) {
+			    $$FeatureDictH{FeatureCounter}++;
+			    $$FeatureDictH{$ContextFea}{ID} = $$FeatureDictH{FeatureCounter};
+			    $$FeatureDictH{$ContextFea}{max} = 0.5;
+			}
+			if ($$FeatureDictH{$ContextFea}{ID}) {
+			    $$HeaderH{$line}{ContextFeaVec}{$ContextFea} = 0.5;
+			    $$FeatureDictH{$ContextFea}{DF}++;
+			}
+		    }
+		}
+		$PC++;
+		$Pline = $line - $PC;
+	    }else {
+		last;
+	    }
+	}
+
+	my $NC = 1;
+	my $Nline = $line + $NC;
+	while (($NC < 5) && (exists $$HeaderH{$Nline})) { #next line
+	    if ($$HeaderH{$Nline}{TClass} eq "s") {
+		my $ContextFea = "N"."$NC"."$$HeaderH{$Nline}{SClsName}";
+		if (! $$FeatureDictH{$ContextFea}{ID}) {
+		    $$FeatureDictH{FeatureCounter}++;
+		    $$FeatureDictH{$ContextFea}{ID} = $$FeatureDictH{FeatureCounter};
+		    $$FeatureDictH{$ContextFea}{max} = 0.5;
+		}
+		if ($$FeatureDictH{$ContextFea}{ID}) {
+		    $$HeaderH{$line}{ContextFeaVec}{$ContextFea} = 0.5;
+		    $$FeatureDictH{$ContextFea}{DF}++;
+		}
+	    }else { # consider the order of the tag
+		foreach my $tag(sort {$$HeaderH{$Nline}{MClsName}{$a} <=> $$HeaderH{$Nline}{MClsName}{$b}} keys %{$$HeaderH{$Nline}{MClsName}}){
+		    my $ContextFea = "N"."$NC"."$tag";
+		    if (! $$FeatureDictH{$ContextFea}{ID}) {
+			$$FeatureDictH{FeatureCounter}++;
+			$$FeatureDictH{$ContextFea}{ID} = $$FeatureDictH{FeatureCounter};
+			$$FeatureDictH{$ContextFea}{max} = 0.5;
+		    }
+		    if ($$FeatureDictH{$ContextFea}{ID}) {
+			$$HeaderH{$line}{ContextFeaVec}{$ContextFea} = 0.5;
+			$$FeatureDictH{$ContextFea}{DF}++;
+		    }
+		}
+	    }
+	    $NC++;
+	    $Nline = $line + $NC;
+	}
+
+	#assemble features and their weight into string without normalization
+	my $tmpFeaVec = $$HeaderH{$line}{SVMFeaVec};
+	foreach my $fea(sort{$$FeatureDictH{$a}{ID} <=> $$FeatureDictH{$b}{ID}} keys %{$$HeaderH{$line}{ContextFeaVec}}) {
+	    if (exists $$FeatureDictH{$fea}{ID}) {
+		$tmpFeaVec .= "$$FeatureDictH{$fea}{ID}\:$$HeaderH{$line}{ContextFeaVec}{$fea} ";
+	    }
+	}
+	$$HeaderH{$line}{ContextSVMFeaVec} = "$tmpFeaVec";
+    }
+    return($FeatureDictH, $HeaderH);
+}
+
+sub TestAssignLineTag() {
+    my $FeatureDictH = shift;
+    my $HeaderH = shift;
+    my %curState = ();
+
+    foreach $line(sort {$a <=> $b} keys %{$HeaderH}) {
+	#Initialize-remove the $$HeaderH{$line}{ContextFeaVec}
+	if(exists ($$HeaderH{$line}{ContextFeaVec})) {
+	    delete($$HeaderH{$line}{ContextFeaVec});
+	}
+
+	my $PC = 1; # 0 means the tag for current line (which might be useful)
+	my $Pline = $line - $PC;
+	while (($PC < 5) && ($Pline > 0)) { #previous line
+	    if (exists $$HeaderH{$Pline}{Pretag}) {
+		foreach my $tag(sort keys %{$$HeaderH{$Pline}{Pretag}}){
+		    my $ContextFea = "P"."$PC"."$tag";
+		    if ($$FeatureDictH{$ContextFea}{ID}) {
+			$$HeaderH{$line}{ContextFeaVec}{$ContextFea} = 0.5;
+		    }
+		}
+	    }
+	    $PC++;
+	    $Pline = $line - $PC;
+	}
+
+	my $NC = 1;
+	my $Nline = $line + $NC;
+	while (($NC < 5) && (exists $$HeaderH{$Nline})) { #next line
+	    foreach my $tag(sort keys %{$$HeaderH{$Nline}{Pretag}}){
+		my $ContextFea = "N"."$NC"."$tag";
+		if ($$FeatureDictH{$ContextFea}{ID}) {
+		    $$HeaderH{$line}{ContextFeaVec}{$ContextFea} = 0.5;
+		}
+	    }
+	    $NC++;
+	    $Nline = $line + $NC;
+	}
+
+	#assemble features and their weight into string without normalization
+	my $tmpFeaVec = $$HeaderH{$line}{SVMFeaVec};
+
+	foreach my $fea(sort{$$FeatureDictH{$a}{ID} <=> $$FeatureDictH{$b}{ID}} keys %{$$HeaderH{$line}{ContextFeaVec}}) {
+	    if (exists $$FeatureDictH{$fea}{ID}) {
+		$tmpFeaVec .= "$$FeatureDictH{$fea}{ID}\:$$HeaderH{$line}{ContextFeaVec}{$fea} ";
+	    }
+	}
+	$tmpFeaVec =~ s/\s+$//g;
+	$$HeaderH{$line}{ContextSVMFeaVec} = "$tmpFeaVec";
+    }
+    return($FeatureDictH, $HeaderH);
+}
+
+#given a line, check the number and the position of punctuation/space it contains
+sub GetSeparatorIndex() {
+    my $line = shift;
+    my %SeparatorH = ();
+
+    my $PuncNum = 0;
+    $line =~ s/^\s+//g;
+    $line =~ s/\s+$//g;
+
+    #punc means this line contains punc or only space
+    #each space occupies a position and punctuations are separate
+    my ($punc, $spaceLine) = &FillSpace($line);
+
+    #punctuation is specific; space separator contains punctuation separators.
+    my @component = split(/\s+/, $spaceLine);
+    foreach my $i(0 .. $#component) {
+	if ($component[$i] =~ /<<sep>>(\W+|\s*)<<\/sep>>/) {
+	    $component[$i] = $1;
+	    if ($component[$i] eq "") {
+		$component[$i] = "<<sep>><<\/sep>>";
+		$SeparatorH{space}{$i} = 2;
+	    }else {
+		$SeparatorH{punc}{$i} = 2;
+		$PuncNum++;
+		$SeparatorH{space}{$i} = 2;
+	    }
+	}elsif ($component[$i] =~ /<space>/) {
+	    $SeparatorH{space}{$i} = 1;
+	}elsif ($component[$i] =~ /^[^\p{IsLower}\p{IsUpper}\s+\-\d+]+$/) {
+	    $SeparatorH{punc}{$i} = 1; #position(not what punc)
+	    $PuncNum++;
+	    $SeparatorH{space}{$i} = 1;
+	}
+    }
+    return($PuncNum, \%SeparatorH, \@component);
+}
+
+
+#multi-Authors line still has only one class, although 1+ authors
+sub AssignWordTagFromChunk() {
+    my ($LineH, $SepH, $component) = @_;
+    my @tags = ();
+    foreach my $tag(sort {$$LineH{MClsName}{$a} <=> $$LineH{MClsName}{$b}} keys %{$$LineH{MClsName}}) {
+	push @tags, $tag;
+    }
+
+    my $ChunkNO = 1;
+    my $curTag = $tags[$tagP];
+    my $WordPos = 1;
+    my $chunk = "";
+    for my $i(0 .. $#$component) {
+	#we do not assign class to separators
+	if ($$SepH{space}{$i} >1) {
+	    if ($chunk ne "") {
+		$$LineH{Tchunk}{$ChunkNO}{cls} = $curTag;
+		$$LineH{Tchunk}{$ChunkNO}{content} = $chunk;
+		$chunk = "";
+		$curTag = $tags[$ChunkNO];
+		$ChunkNO++;
+	    }
+	}elsif ($$component[$i] !~ /<space>|^\W+$|\<\<.*\>\>/) { # such as <<sep>> <</sep>>
+	    $chunk .= "$$component[$i] ";
+	    $$LineH{Tline}{$WordPos}{cls} = $curTag;
+	    $$LineH{Tline}{$WordPos}{OriginalWord} = $$component[$i];
+	    $WordPos++;
+	}
+    };
+
+    #Fill in the last chunk
+    $$LineH{Tchunk}{$ChunkNO}{cls} = $curTag;
+    $$LineH{Tchunk}{$ChunkNO}{content} = $chunk;
+
+    return ($LineH);
+}
+
+
+sub AssignWordTag4SingleClassLine() {
+    my ($type, $curTag, $LineH, $component) = @_;
+
+    my $WordPos = 1;
+    for my $i(0 .. $#$component) {
+	if ($$component[$i] !~ /<space>|^\W+$|\<\<.*\>\>/) { # such as <<sep>> <</sep>>
+	    if ($type eq "truth") {
+		$$LineH{Tline}{$WordPos}{cls} = $curTag;
+		#added 01/08 the original word in a position
+		$$LineH{Tline}{$WordPos}{OriginalWord} = $$component[$i];
+	    }elsif ($type eq "predict") {
+		$$LineH{Pline}{$WordPos}{cls} = $curTag;
+		$$LineH{Pline}{$WordPos}{OriginalWord} = $$component[$i];
+	    }
+	    $WordPos++;
+	}
+    }
+
+    return ($LineH);
+}
+
+
+sub Analyze() {
+    my $resultF = shift;
+    open(resultFH, "$resultF") || die "SVMHeaderParse: could not open $resultF to read: $!";
+    my $result = <resultFH>;
+    close(resultFH);
+    $result =~ s/\s+$//g;
+    return($result);
+}
+
+
+sub ReadFeatureDict() {
+    my $Fname = shift;
+    my %FeatureDictH;
+
+    open (FH, "$Fname") || die "SVMHeaderParse: could not open $Fname to read: $!";
+    while (my $line = <FH>) {
+	my ($ID, $fea, $max, $DF) = split(/<>/, $line);
+	$ID =~ s/^\s+//g;
+	$ID =~ s/\s+$//g;
+
+	if ($fea =~ /FeatureCounter/) {
+	    $FeatureDictH{$fea}{num} = $ID;
+	    next;
+	}
+
+	$fea =~ s/^\s+//g;
+	$fea =~ s/\s+$//g;
+	$max =~ s/^\s+//g;
+	$max =~ s/\s+$//g;
+	$DF =~ s/^\s+//g;
+	$Df =~ s/\s+$//g;
+	$FeatureDictH{$fea}{ID} = $ID;
+	$FeatureDictH{$fea}{max} = $max;
+	$FeatureDictH{$fea}{DF} = $DF;
+    }
+    close(FH);
+    return(\%FeatureDictH);
+}
+
+
+sub printTrainData() {
+    my $affix = shift;
+    my $HeaderH = shift;
+
+    #Sometimes $$HeaderH{$HeaNO}{$LN}{ContextSVMFeaVec} is not null, but
+    #$$HeaderH{$HeaNO}{$LN}{SVMFeaVec} is null so they have different file length!
+    for my $clsNO(1 .. 15) {
+	my $F = "$offlineD"."$clsNO"."\."."$affix";
+	open(FH, ">$F") || die "SVMHeaderParse: could not open $F to write: $!";
+	foreach my $HeaNO (sort {$a <=> $b} keys %{$HeaderH}) {
+	    foreach my $LN(sort {$a <=> $b} keys %{$$HeaderH{$HeaNO}}) {
+		if ($$HeaderH{$HeaNO}{$LN}{SVMFeaVec} ne "") {
+		    if ($affix eq "train") {
+			if ($$HeaderH{$HeaNO}{$LN}{SVMFeaVec} ne "") {
+			    if (($$HeaderH{$HeaNO}{$LN}{SClsName} eq "$clsNO") || exists($$HeaderH{$HeaNO}{$LN}{MClsName}{$clsNO})) {
+				print FH "1 $$HeaderH{$HeaNO}{$LN}{SVMFeaVec}\n";
+			    }else {
+				print FH "-1 $$HeaderH{$HeaNO}{$LN}{SVMFeaVec}\n";
+			    }
+			}
+		    }elsif ($affix eq "context") {
+			#if ($$HeaderH{$HeaNO}{$LN}{ContextSVMFeaVec} ne "") {
+			if (($$HeaderH{$HeaNO}{$LN}{SClsName} eq "$clsNO") || exists($$HeaderH{$HeaNO}{$LN}{MClsName}{$clsNO})) {
+			    print FH "1 $$HeaderH{$HeaNO}{$LN}{ContextSVMFeaVec}\n";
+			}else {
+			    print FH "-1 $$HeaderH{$HeaNO}{$LN}{ContextSVMFeaVec}\n";
+			}
+		    }else {
+			print "weired -- $affix is not context nor train \n";
+		    }
+		}
+	    }
+	}
+	close(FH);
+    }
+}
+
+
+sub printNameSpaceTrainData(){
+    my $printF = shift;
+    my $NameSpaceTrainVecH = shift;
+
+    open(FH, ">$printF") || die "SVMHeaderParse: could not open $printF to write: $!";
+    foreach my $Lcount(sort{$a<=>$b} keys %{$NameSpaceTrainVecH}) {
+	print FH "$$NameSpaceTrainVecH{$Lcount}{SpaceNameVec}\n";
+    }
+    close(FH);
+}
+
+
+sub SpaceNameLnFeaRepre() {
+    my $type = shift;
+    my $NamePatternStr = shift;
+    my $NameDictH = shift;
+
+    #feature generation and representation
+    #It is good to make each of the apple's feature(color, shape..) separate.
+    my %FeatureH = ();
+    $NamePatternStr =~ s/\<\>$//g; #remove the last <>
+    my @Names = split(/<>/, $NamePatternStr);
+
+    #try making features binary
+    for my $i(0 .. $#Names) {
+	my @NameComponent = split(/\s+/, $Names[$i]);
+	for my $j(0 .. $#NameComponent){
+
+	    #feature generation($i = 0 is the first one)
+	    $FeatureH{"Name"."$i"."part"."$j"."form"} = &HeaderParse::API::NamePatternMatch::RichNameType($NameComponent[$j]);
+	    if ($j eq $#NameComponent) {
+		$FeatureH{"Name"."$i"."part"."$j"."pos"} = "Last";
+	    }elsif ($j eq $#NameComponent -1) {
+		$FeatureH{"Name"."$i"."part"."$j"."pos"} = "SecLast";
+	    }else {
+		$FeatureH{"Name"."$i"."part"."$j"."pos"} = $j;
+	    }
+
+	    #firstname, lastname information
+#	    print "hello: ".lc($NameComponent[$j])."\n";
+	    if (($firstnameH{lc($NameComponent[$j])}) && (!$lastnameH{lc($NameComponent[$j])})) {
+		$FeatureH{"Name"."$i"."part"."$j"."FN"} = 1;
+	    }elsif (($lastnameH{lc($NameComponent[$j])}) && (!$firstnameH{lc($NameComponent[$j])})) {
+		$FeatureH{"Name"."$i"."part"."$j"."LN"} = 1;
+	    }elsif (! $dictH{lc($NameComponent[$j])}) {
+		$FeatureH{"Name"."$i"."part"."$j"."NonDict"} = 1;
+	    }
+
+	    #space for more features
+	}
+    }
+
+    #Build up FeatureVec
+    #code for the attribute ID separately so that the ID for features would be continuous
+    if ($type eq "train") {
+	foreach my $fea(sort {$a <=> $b} keys %FeatureH) {
+	    if (! $$NameDictH{$fea}{ID}) {
+		$$NameDictH{FeatureCounter}++;
+		$$NameDictH{$fea}{ID} = $$NameDictH{FeatureCounter};
+	    }
+
+	    if (! IsNumber($FeatureH{$fea})) {
+		if (! exists  $$NameDictH{$FeatureH{$fea}}{ID}) {
+		    $$NameDictH{FeatureCounter}++;
+		    $$NameDictH{$FeatureH{$fea}}{ID} = $$NameDictH{FeatureCounter};
+		}
+		$FeatureH{$fea} = $$NameDictH{$FeatureH{$fea}}{ID};
+	    }
+
+	    if ($FeatureH{$fea} == 0) {
+		delete($FeatureH{$fea});
+	    }else {
+		if ((! exists $$NameDictH{$fea}{max}) || ($$NameDictH{$fea}{max} < $FeatureH{$fea})) {
+		    $$NameDictH{$fea}{max} =  $FeatureH{$fea};
+		}
+	    }
+	}
+	return(\%FeatureH, $NameDictH);
+    #test
+    }else {
+	my $SpaceNameFeaVec = "";
+	my $SpaceNameTextFeaVec = "";
+	foreach my $fea(sort {$$NameDictH{$a}{ID} <=> $$NameDictH{$b}{ID}} keys %FeatureH) {
+	    if (! &IsNumber($FeatureH{$fea})) {
+		if (exists  $$NameDictH{$FeatureH{$fea}}{ID}) {
+		    $FeatureH{$fea} = $$NameDictH{$FeatureH{$fea}}{ID};
+		}else {
+		    delete($FeatureH{$fea});
+		}
+	    }
+
+	    if (! ($FeatureH{$fea} && $$NameDictH{$fea}{ID})) {
+		delete($FeatureH{$fea});
+	    }else {
+		$FeatureH{$fea} = sprintf("%.8f", $FeatureH{$fea}/$$NameDictH{$fea}{max});
+		$SpaceNameFeaVec .= "$$NameDictH{$fea}{ID}\:$FeatureH{$fea} ";
+		$SpaceNameTextFeaVec .= "$fea\:$FeatureH{$fea} ";
+	    }
+	}
+	return($SpaceNameFeaVec, $SpaceNameTextFeaVec);
+    }
+}
+
+
+sub SpaceNameLnFeaRepre_unit() {
+    my $NamePatternStr = shift;
+
+    #feature generation and representation
+    #It is good to make each of the apple's feature(color, shape..) separate.
+    my %FeatureH = ();
+    $NamePatternStr =~ s/\<\>$//g; #remove the last <>
+    my @Names = split(/<>/, $NamePatternStr);
+
+    #try making features binary
+    for my $i(0 .. $#Names) {
+	my @NameComponent = split(/\s+/, $Names[$i]);
+	for my $j(0 .. $#NameComponent){
+	    #feature generation($i = 0 is the first one)
+	    $FeatureH{"Name"."$i"."part"."$j"."form"} = &HeaderParse::API::NamePatternMatch::RichNameType($NameComponent[$j]);
+	    if ($j eq $#NameComponent) {
+		$FeatureH{"Name"."$i"."part"."$j"."pos"} = "Last";
+	    }elsif ($j eq $#NameComponent -1) {
+		$FeatureH{"Name"."$i"."part"."$j"."pos"} = "SecLast";
+	    }else {
+		$FeatureH{"Name"."$i"."part"."$j"."pos"} = $j;
+	    }
+	    #firstname, lastname information
+#	    print "hello2: ".lc($NameComponent[$j])."\n";
+	    if (($firstnameH{lc($NameComponent[$j])}) && (!$lastnameH{lc($NameComponent[$j])})) {
+#		print "NAME MATCH: ".lc($NameComponent[$j])."\n";
+		$FeatureH{"Name"."$i"."part"."$j"."FN"} = 1;
+	    }elsif (($lastnameH{lc($NameComponent[$j])}) && (!$firstnameH{lc($NameComponent[$j])})) {
+#		print "NAME MATCH: ".lc($NameComponent[$j])."\n";
+
+		$FeatureH{"Name"."$i"."part"."$j"."LN"} = 1;
+	    }elsif (! $dictH{lc($NameComponent[$j])}) {
+#		print "NAME MATCH: ".lc($NameComponent[$j])."\n";
+
+		$FeatureH{"Name"."$i"."part"."$j"."NonDict"} = 1;
+	    }
+
+	    #space for more features
+	}
+    }
+    return(\%FeatureH);
+}
+
+
+sub IsNumber ()
+{
+    my $in = shift;
+    if ($in =~ m/^(\d+)(\.\d+)*$/) {
+	return 1;
+    }else {
+	return 0;
+    }
+}
+
+
+sub FormFeaDict() {
+    my $DataH = shift;
+    my $FeatureDictH = shift;
+    my %NameSpaceFeaDictH = ();
+
+    foreach my $HeaNO (sort {$a <=> $b} keys %{$DataH}) {
+	foreach my $line (sort {$a <=> $b} keys %{$$DataH{$HeaNO}}) {
+	    foreach my $fea(keys %{$$DataH{$HeaNO}{$line}{FeaVec}}) {
+		if ($$DataH{$HeaNO}{$line}{FeaVec}{$fea} == 0) {
+		    delete ($$DataH{$HeaNO}{$line}{FeaVec}{$fea});
+		    next;
+		}else {
+		    if (! $$FeatureDictH{$fea}{ID}) {
+			$$FeatureDictH{FeatureCounter}++;
+			$$FeatureDictH{$fea}{ID} = $$FeatureDictH{FeatureCounter};
+		    }
+		    if ($$DataH{$HeaNO}{$line}{FeaVec}{$fea} > $$FeatureDictH{$fea}{max}) {
+			$$FeatureDictH{$fea}{max} = $$DataH{$HeaNO}{$line}{FeaVec}{$fea};
+		    }
+		    $$FeatureDictH{$fea}{DF}++;
+		}
+		#test needs this line!
+		if ((! $$FeatureDictH{$fea}{ID}) || ($$DataH{$HeaNO}{$line}{FeaVec}{$fea} == 0)) { #some basic feature defined in initialization such as pubnumber could be 0
+		    delete ($$DataH{$HeaNO}{$line}{FeaVec}{$fea});
+		}
+	    }
+
+	    #form the Name Space Feature Dict
+	    if (exists $$DataH{$HeaNO}{$line}{NamePattern}) {
+		foreach my $CandidateNamePattern(keys %{$$DataH{$HeaNO}{$line}{NamePattern}}) {
+		    foreach my $fea(keys %{$$DataH{$HeaNO}{$line}{NamePattern}{$CandidateNamePattern}{SpaceNameVec}}) {
+			my $wt = $$DataH{$HeaNO}{$line}{NamePattern}{$CandidateNamePattern}{SpaceNameVec}{$fea};
+			if (! $NameSpaceFeaDictH{$fea}{ID}) {
+			    $NameSpaceFeaDictH{FeatureCounter}++;
+			    $NameSpaceFeaDictH{$fea}{ID} = $NameSpaceFeaDictH{FeatureCounter};
+			}
+			if (! &IsNumber($wt)) {
+			    if (! exists  $NameSpaceFeaDictH{$wt}{ID}) {
+				$NameSpaceFeaDictH{FeatureCounter}++;
+				$NameSpaceFeaDictH{$wt}{ID} = $NameSpaceFeaDictH{FeatureCounter};
+			    }
+			    $$DataH{$HeaNO}{$line}{NamePattern}{$CandidateNamePattern}{SpaceNameVec}{$fea} = $NameSpaceFeaDictH{$wt}{ID};
+			}
+
+			if ($wt  == 0) {
+			    delete($$DataH{$HeaNO}{$line}{NamePattern}{$CandidateNamePattern}{SpaceNameVec}{$fea});
+			}else {
+			    if ((! exists $NameSpaceFeaDictH{$fea}{max}) || ($NameSpaceFeaDictH{$fea}{max} < $wt)) {
+				$NameSpaceFeaDictH{$fea}{max} =  $wt;
+			    }
+			}
+		    }
+		}
+	    }
+	    #end of form the dictionary for the name
+	}
+    }
+    return($DataH, $FeatureDictH, \%NameSpaceFeaDictH);
+}
+
+
+sub FormTestFeaVec(){
+    my $FeatureDictH = shift;
+    my $TestHeaderH = shift;
+
+    foreach my $line(sort{$a<=>$b} keys %{$TestHeaderH}) {
+	foreach my $fea(keys %{$$TestHeaderH{$line}{FeaVec}}) {
+	    if ((! $$FeatureDictH{$fea}{ID}) || ($$TestHeaderH{$line}{FeaVec}{$fea} == 0)) {
+		delete($$TestHeaderH{$line}{FeaVec}{$fea});
+	    }
+	}
+    }
+    return($TestHeaderH);
+}
+
+
+sub PruneDict() {
+    my $FeatureDictH = shift;
+    my $Recount = 1;
+
+    foreach my $DictFea(sort{$$FeatureDictH{$a}{ID} <=> $$FeatureDictH{$b}{ID}} keys %{$FeatureDictH}) {
+	if ((($DictFea ne "FeatureCounter") && ($$FeatureDictH{$DictFea}{max} == 0)) || ($$FeatureDictH{$DictFea}{DF} < 2))  {
+	    delete($$FeatureDictH{$DictFea});
+	}else {
+	    $$FeatureDictH{$DictFea}{ID} = $Recount;
+	    $Recount++;
+	}
+    }
+
+    $$FeatureDictH{FeatureCounter} = $Recount-1;
+
+    return($FeatureDictH);
+}
+
+#input is an array of name patterns
+#return a string of the best name pattern
+sub PredictBestNamePattern() {
+    my $PredictedNames = shift;
+    my $SVMNameSpaceModel = shift;
+    my $SpaceNameDictH = shift;
+    my $tmpCacheVec = shift;
+    my $SVMTmpResult = shift;
+
+    my $MaxVal = -10;
+    my $BestNamePattern = "";
+
+    for my $i(0 .. $#$PredictedNames) {
+	my $candidateName = "";
+	for my $j(0 .. $#{$$PredictedNames[$i]}) {
+	    if ($$PredictedNames[$i][$j]) {
+		$candidateName .= "$$PredictedNames[$i][$j]<>";
+	    }
+	}
+
+	my ($RawNameFeaVec) = &SpaceNameLnFeaRepre_unit($candidateName);
+	#filter out the non-dictinary features
+	my $SpaceNameVec = "";
+	my $SpaceNameTextFeaVec = "";
+	foreach my $fea(sort {$$SpaceNameDictH{$a}{ID} <=> $$SpaceNameDictH{$b}{ID}} keys %{$RawNameFeaVec}) {
+	    my $wt = $$RawNameFeaVec{$fea};
+	    if (! &IsNumber($wt)) {
+		if (exists  $$SpaceNameDictH{$wt}{ID}) {
+		    $$RawNameFeaVec{$fea} = $$SpaceNameDictH{$wt}{ID};
+		}else {
+		    delete($$RawNameFeaVec{$fea});
+		}
+	    }
+
+	    if (! (($$RawNameFeaVec{$fea}>0) && $$SpaceNameDictH{$fea}{ID})) {
+		delete($$RawNameFeaVec{$fea});
+	    }else {
+		$$RawNameFeaVec{$fea} = sprintf("%.8f", $$RawNameFeaVec{$fea}/$$SpaceNameDictH{$fea}{max}
+						);
+		$SpaceNameVec .= "$$SpaceNameDictH{$fea}{ID}\:$$RawNameFeaVec{$fea} ";
+		$SpaceNameTextFeaVec .= "$fea\:$$RawNameFeaVec{$fea} ";
+	    }
+	}
+	open(testVec, ">$tmpCacheVec") || die "SVMHeaderParse: could not open $tmpCacheVec to write: $!";
+	# print "NamePattern FeatureVec is\: $SpaceNameTextVec\n";
+	print testVec "$SpaceNameVec";
+	close(testVec);
+	`$Classifier -v 0 $tmpCacheVec $SVMNameSpaceModel $SVMTmpResult`;
+	my $result = &Analyze($SVMTmpResult);
+	if ($result > $MaxVal) {
+	    $MaxVal = $result;
+	    $BestNamePattern = $candidateName;
+	}
+    }
+
+    unlink $tmpCacheVec;
+    unlink $SVMTmpResult;
+
+    #split the multiple names in order
+    $BestNamePattern =~ s/\<\>$//g; #remove the last <>
+
+    return($BestNamePattern);
+}
+
+
+sub WordCount() { #didn't try, but should be OK, since it is borrowed from AddrMatch in function.pm
+    my $inStr = shift;
+    $inStr =~ s/^\s+//g;
+    $inStr =~ s/\s+$//g;
+
+    my $senLen = 0;
+    my @words = split(/\s+/, $inStr);
+    for my $i(0 .. $#words) {
+	if ($words[0] !~ /^\W+\s*$/) { #punctuation
+	    $senLen ++;
+	}
+    }
+    return($senLen);
+}
+
+1;