biblicit 1.0 → 2.0.3

data/parscit/bin/parsHed/keywordGen.pl

@@ -0,0 +1,308 @@
+#!/usr/bin/perl -wT
+# Author: Luong Minh Thang <luongmin@comp.nus.edu.sg>, generated at Tue, 02 Jun 2009 01:30:42
+
+# Modified from template by Min-Yen Kan <kanmy@comp.nus.edu.sg>
+
+require 5.0;
+use strict;
+
+# I do not know a better solution to find a lib path in -T mode.
+# So if you know a better solution, I'd be glad to hear.
+# See this http://www.perlmonks.org/?node_id=585299 for why I used the below code
+use FindBin;
+my $path;
+BEGIN {
+  if ($FindBin::Bin =~ /(.*)/) {
+        $path = $1;
+      }
+}
+use lib "$path/../../lib";
+
+use Getopt::Long;
+use ParsHed::Config;
+
+### USER customizable section
+$0 =~ /([^\/]+)$/; my $progname = $1;
+my $outputVersion = "1.0";
+### END user customizable section
+
+sub License {
+  print STDERR "# Copyright 2009 \251 by Luong Minh Thang\n";
+}
+
+### HELP Sub-procedure
+sub Help {
+  print STDERR "Create keyword info from a tagged header file\n";
+  print STDERR "usage: $progname -h\t[invokes help]\n";
+  print STDERR "       $progname -in taggedHeaderFile -out outFile [-n topN -nGram numNGram -lowercase]\n";
+  print STDERR "Options:\n";
+  print STDERR "\t-q\tQuiet Mode (don't echo license)\n";
+  print STDERR "\t-n: Default topN = 100.\n";
+  print STDERR "\t-nGram: Default numNGram = 1.\n";
+  print STDERR "\t-lowercase: enable lowercasing (default no lowercasign).\n";
+}
+my $QUIET = 0;
+my $HELP = 0;
+my $outFile = undef;
+my $inFile = undef;
+my $topN = 100;
+my $numNGram = 1;
+my $isLowercase = 0;
+$HELP = 1 unless GetOptions('in=s' => \$inFile,
+			    'out=s' => \$outFile,
+			    'n=i' => \$topN,
+			    'nGram=i' => \$numNGram,
+			    'lowercase' => \$isLowercase,
+			    'h' => \$HELP,
+			    'q' => \$QUIET);
+
+if ($HELP || !defined $inFile || !defined $outFile) {
+  Help();
+  exit(0);
+}
+
+if (!$QUIET) {
+  License();
+}
+
+### Untaint ###
+$inFile = untaintPath($inFile);
+$outFile = untaintPath($outFile);
+$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
+### End untaint ###
+
+# keyword statistics
+my %keywords = (); #hash of hash $keywords{"affiliation"}->{"Institute"} = freq of "Institute" for affiliation tag
+
+processFile($inFile, $outFile, $numNGram);
+
+## 
+# main routine to count frequent keywords/bigrams in $inFile and output to $outFile
+##
+sub processFile {
+  my ($inFile, $outFile, $numNGram) = @_;
+  
+  if (!(-e $inFile)) { die "# $progname crash\t\tFile \"$inFile\" doesn't exist"; }
+  open (IF, "<:utf8", $inFile) || die "# $progname crash\t\tCan't open \"$inFile\"";
+  
+  while (<IF>) { #each line contains a header
+    if (/^\#/) { next; }			# skip comments
+    elsif (/^\s+$/) { next; }		# skip blank lines
+    else {
+      my @tokens = split(/ +/);
+      
+      my $tag = "";
+      my $line = "";
+      foreach(@tokens){
+	if (/^\s*$/) { # spaces
+	  next; 
+	} elsif (/^\<\/([a-z]+)/) { #end  tag
+	  my @sub_lines = split(/\s*\+L\+\s*/, $line);
+
+	  foreach my $sub_line (@sub_lines){ #go through each subline of a header field
+	    countKeywords($sub_line, $tag, $numNGram);
+	  }
+	  $line = ""; # reset
+	  next;
+	} elsif (/^\<([a-z]+)/) { #beginning tag
+	  $tag = $1;
+	  
+	  if(!$keywords{$tag}){
+	    $keywords{$tag} = ();
+	    $keywords{$tag}->{""} = 0;
+	  }
+	  
+	  next;
+	} else { #contents inside tag
+	  $line .= "$_ ";
+	}
+      }
+      
+      next;
+    }
+  } # end while IF
+  close IF;
+
+  ## obtain top keyWords
+  my %topKeywords = ();
+  foreach my $tag (keys %keywords){
+    $topKeywords{$tag} = ();
+
+    my %freqs = %{$keywords{$tag}};
+    my @sorted_keys = sort { $freqs{$b} <=> $freqs{$a} } keys %freqs;
+    
+    my $count = 0;    
+    foreach my $keyWord (@sorted_keys){
+      $topKeywords{$tag}->{$keyWord} = 1;
+
+      $count++;
+      if($count == $topN){
+	last;
+      }      
+    }
+  }
+
+  ## filter duplicate keywords
+  my %filteredKeywords = ();
+  filterDuplicate(\%topKeywords, \%filteredKeywords);
+
+  ## output results
+  outputKeywords(\%filteredKeywords, $outFile);
+}
+
+##
+# Output keyword hash to file
+##
+sub outputKeywords {
+  my ($hash, $outFile) = @_;
+
+  open(OF, ">:utf8", "$outFile") || die"#Can't open file \"$outFile\"\n";
+  
+  # list of tags trained in parsHed
+  # those with value 0 do not have frequent keyword features
+  my $tags = $ParsHed::Config::tags; # hash
+
+  foreach my $tag (keys %{$tags}){
+    if($tags->{$tag} == 0){
+      next;
+    }
+
+    print OF "$tag:";
+
+    my @keywords = sort{$a cmp $b} keys %{$hash->{$tag}};    
+    foreach my $keyword (@keywords){
+      print OF " $keyword";
+    }
+
+    print OF "\n";
+  }
+  close OF;
+}
+
+##
+# Remove keywords that appear in more than one field
+##
+sub filterDuplicate {
+  my ($hash, $filteredHash) = @_;
+
+  my @tags = keys %{$hash};
+  foreach my $tag (@tags){
+    $filteredHash->{$tag} = ();
+    my @keywords = keys %{$hash->{$tag}};
+    
+    foreach my $keyword (@keywords){
+      my $isDuplicated = 0;
+
+      # check for duplication
+      foreach(@tags){
+	if($_ ne $tag){ # a different tag
+	  if($hash->{$_}->{$keyword}){ # duplicated
+	    $isDuplicated = 1;
+	    last;
+	  }
+	}
+      } # end for @tags
+
+      if(!$isDuplicated){
+	$filteredHash->{$tag}->{$keyword} = 1;
+      }
+    }
+  }
+}
+
+##
+# Count keyword or nGrams
+##
+sub countKeywords {
+  my ($line, $tag, $numNGram) = @_;
+
+  if($isLowercase){
+    $line = lc($line);
+  }
+
+  my @tmpTokens = split(/\s+/, $line);
+
+  #filter out empty token
+  my @tokens = ();
+  foreach my $token (@tmpTokens){ 
+    if($token ne ""){
+      $token =~ s/^\s+//g; # strip off leading spaces
+      $token =~ s/\s+$//g; # strip off trailing spaces
+      push(@tokens, $token);
+    }
+  }
+
+  my $funcWordCount = 0;
+  my $count = 0;  
+  for(my $i=0; $i<=$#tokens; $i++){
+    if(($#tokens-$i + 1) < $numNGram) { last; }; # not enough ngrams
+    my $nGram = "";
+    for(my $j=$i; $j <= ($i+$numNGram-1); $j++){
+      my $token = $tokens[$j];
+      $token =~ s/^\p{P}+//g; #strip out leading punctuations
+      $token =~ s/\p{P}+$//g; #strip out trailing punctuations
+      $token =~ s/^\s+//g; #strip out leading spaces
+      $token =~ s/\s+$//g; #strip out trailing spaces
+      $token =~ s/\d/0/g; #canocalize number into "0"
+
+      if($numNGram > 1){ # lowercase to reduce data sparseness
+	$token = lc($token);
+      }
+
+      if($j < ($i+$numNGram-1)){
+	$nGram .= "$token-";
+      } else {
+	$nGram .= "$token";
+      }
+    }
+
+    if($nGram =~ /^\s*$/){ next; } #skip those with white spaces
+    if($nGram =~ /^\d*$/){ next; } #skip those with only digits
+    
+    #print STDERR "$nGram
+    if(!$keywords{$tag}->{$nGram}){
+      $keywords{$tag}->{$nGram} = 0;
+    }
+    $keywords{$tag}->{$nGram}++;
+
+    $count++;
+    if($count == 2){
+      last;
+    }
+  } # end while true
+}
+
+sub untaintPath {
+  my ($path) = @_;
+
+  if ( $path =~ /^([-_\/\w\.]+)$/ ) {
+    $path = $1;
+  } else {
+    die "Bad path $path\n";
+  }
+
+  return $path;
+}
+
+sub untaint {
+  my ($s) = @_;
+  if ($s =~ /^([\w \-\@\(\),\.\/]+)$/) {
+    $s = $1;               # $data now untainted
+  } else {
+    die "Bad data in $s";  # log this somewhere
+  }
+  return $s;
+}
+
+sub execute {
+  my ($cmd) = @_;
+  print STDERR "Executing: $cmd\n";
+  $cmd = untaint($cmd);
+  system($cmd);
+}
+
+sub newTmpFile {
+  my $tmpFile = `date '+%Y%m%d-%H%M%S-$$'`;
+  chomp($tmpFile);
+  return $tmpFile;
+}

data/parscit/bin/parsHed/parseXmlHeader.pl

@@ -0,0 +1,141 @@
+#!/usr/bin/perl -wT
+# Author: Luong Minh Thang <luongmin@comp.nus.edu.sg>, generated at Tue, 02 Jun 2009 01:30:42
+
+# Modified from template by Min-Yen Kan <kanmy@comp.nus.edu.sg>
+
+require 5.0;
+use strict;
+use Getopt::Long;
+
+### USER customizable section
+$0 =~ /([^\/]+)$/; my $progname = $1;
+my $outputVersion = "1.0";
+### END user customizable section
+
+sub License {
+  print STDERR "# Copyright 2009 \251 by Luong Minh Thang\n";
+}
+
+### HELP Sub-procedure
+sub Help {
+  print STDERR "Parse tagged header file into line-level data\n";
+  print STDERR "usage: $progname -h\t[invokes help]\n";
+  print STDERR "       $progname -in taggedHeaderFile -out outFile\n";
+  print STDERR "Options:\n";
+  print STDERR "\t-q\tQuiet Mode (don't echo license)\n";
+}
+my $QUIET = 0;
+my $HELP = 0;
+my $outFile = undef;
+my $inFile = undef;
+
+$HELP = 1 unless GetOptions('in=s' => \$inFile,
+			    'out=s' => \$outFile,
+			    'h' => \$HELP,
+			    'q' => \$QUIET);
+
+if ($HELP || !defined $inFile || !defined $outFile) {
+  Help();
+  exit(0);
+}
+
+if (!$QUIET) {
+  License();
+}
+
+### Untaint ###
+$inFile = untaintPath($inFile);
+$outFile = untaintPath($outFile);
+$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
+### End untaint ###
+processFile($inFile, $outFile);
+
+sub processFile {
+  my ($inFile, $outFile) = @_;
+  
+  if (!(-e $inFile)) { die "# $progname crash\t\tFile \"$inFile\" doesn't exist"; }
+  open (IF, $inFile) || die "# $progname crash\t\tCan't open \"$inFile\"";
+  open(OF, ">:utf8", "$outFile") || die"#Can't open file \"$outFile\"\n";
+  
+  while (<IF>) { #each line contains a header
+    if (/^\#/) { next; }			# skip comments
+    elsif (/^\s+$/) { next; }		# skip blank lines
+    else {
+      my @tokens = split(/ +/);
+      
+      my $tag = "";
+      my $line = "";
+      foreach(@tokens){
+	if (/^\s*$/) { # spaces
+	  next; 
+	} elsif (/^\<\/([a-z]+)/) { #end  tag
+	  if($line =~ /^\s*$/) { print STDERR "Skip \"$line\"\n"; next; }
+
+	  my @sub_lines = split(/\s*\+L\+\s*/, $line);
+	  for(my $i=0; $i<$#sub_lines; $i++){#go through each subline of a header field
+
+	    if($sub_lines[$i] !~ /^\s*$/){
+	      print OF "$sub_lines[$i] +L+\n";
+	    }
+	  }
+	  
+	  #check if $line end with +L+
+	  if($line =~ /\+L\+\s*$/){
+	    print OF "$sub_lines[$#sub_lines] +L+\n";
+	  } else {
+	    print OF "$sub_lines[$#sub_lines]\n";
+	  }
+
+	  $line = ""; # reset
+	  next;
+	} elsif (/^\<([a-z]+)/) { #beginning tag
+	  $tag = $1;
+	  
+	  print OF "//$tag\n";
+	  next;
+	} else { #contents inside tag
+	  $line .= "$_ ";
+	}
+      }
+    }
+    print OF "\n";
+  }
+  
+  close OF;
+  close IF;
+}
+
+sub untaintPath {
+  my ($path) = @_;
+
+  if ( $path =~ /^([-_\/\w\.]+)$/ ) {
+    $path = $1;
+  } else {
+    die "Bad path $path\n";
+  }
+
+  return $path;
+}
+
+sub untaint {
+  my ($s) = @_;
+  if ($s =~ /^([\w \-\@\(\),\.\/]+)$/) {
+    $s = $1;               # $data now untainted
+  } else {
+    die "Bad data in $s";  # log this somewhere
+  }
+  return $s;
+}
+
+sub execute {
+  my ($cmd) = @_;
+  print STDERR "Executing: $cmd\n";
+  $cmd = untaint($cmd);
+  system($cmd);
+}
+
+sub newTmpFile {
+  my $tmpFile = `date '+%Y%m%d-%H%M%S-$$'`;
+  chomp($tmpFile);
+  return $tmpFile;
+}

data/parscit/bin/parsHed/redo.parsHed.pl

@@ -0,0 +1,198 @@
+#!/usr/bin/env perl
+# -*- cperl -*- 
+
+require 5.0;
+use strict;
+use Getopt::Long;
+use FindBin;
+
+### USER customizable section
+my $tmpfile .= $0; $tmpfile =~ s/[\.\/]//g;
+$tmpfile .= $$ . time;
+if ($tmpfile =~ /^([-\@\w.]+)$/) { $tmpfile = $1; }		      # untaint tmpfile variable
+$tmpfile = "/tmp/" . $tmpfile;
+$0 =~ /([^\/]+)$/; my $progname = $1;
+my $outputVersion = "1.0";
+
+my $parscitHome = "$FindBin::Bin/../..";
+my $tr2crfppLoc = "$parscitHome/bin/parsHed/tr2crfpp_parsHed.pl";
+my $convertLoc = "$parscitHome/bin/parsHed/convert2TokenLevel.pl"; #new model
+my $keywordLoc = "$parscitHome/bin/parsHed/keywordGen.pl"; #new model
+my $crf_learnLoc = "$ENV{'CRFPP_HOME'}/bin/crf_learn";
+my $crf_testLoc = "$ENV{'CRFPP_HOME'}/bin/crf_test";
+my $conllevalLoc = "$parscitHome/bin/conlleval.pl";
+### END user customizable section
+
+## Thang add ##
+sub Help {
+  print STDERR "usage: $progname -h\t[invokes help]\n";
+  print STDERR "       $progname -in trainFile -t templateFile -n folds [-p numCpus -oldModel]\n";
+  print STDERR "Options:\n";
+  print STDERR "\t\t-p: Default is 6 cpus\n";
+  print STDERR "\t\t-p: Default: use the new line-level model. Specify -oldModel to run with the old one.\n";
+  print STDERR "\t\t-tr2crfpp: Default $ENV{ParsCit}/bin/parsHed/tr2crfpp.pl\n";
+}
+
+my $HELP = 0;
+my $trainingFile = undef;
+my $crfTemplateLoc = undef;
+my $folds = undef;
+my $numCpus = 6;
+my $isOldModel = 0;
+$HELP = 1 unless GetOptions('in=s' => \$trainingFile,
+			    't=s' => \$crfTemplateLoc,
+			    'n=i' => \$folds,
+			    'p=i' => \$numCpus,
+			    'oldModel' => \$isOldModel,
+			    'h' => \$HELP);
+
+if ($HELP || !defined $trainingFile || !defined $folds || !defined $crfTemplateLoc) {
+  Help();
+  exit(0);
+}
+## End Thang add ##
+
+if($isOldModel){
+  $tr2crfppLoc = "$parscitHome/bin/tr2crfpp.pl";
+}
+
+print STDERR "### Note the number of CPU for parallel crfpp is $numCpus\n";
+
+# construct test data
+print STDERR "### Constructing $folds-fold test files $tmpfile.*.test.src...\n"; # Thang add
+open (IF, $trainingFile) || die "# $progname fatal\tTraining file cannot be opened \"$trainingFile\"!";
+my $i = 0;
+while (<IF>) {
+  open (OF, ">>$tmpfile.$i.test.src") || die "$progname fatal\tCan't append to file \"$tmpfile.$i.test.src\"!";
+  print OF $_;
+  $i++;
+  $i = $i % $folds;
+}
+close (IF);
+
+# construct crf features
+if(!$isOldModel){
+  for (my $i = 0; $i < $folds; $i++) {
+    # construct src training data first
+    for (my $j = 0; $j < $folds; $j++) {
+      if ($j == $i) {next; }
+      else {
+	execute("cat $tmpfile.$j.test.src >> $tmpfile.$i.train.src");
+      }
+    }
+
+    #construct keywordFile, using topN = 100
+    my $topN = 100;
+    execute("$keywordLoc -in $tmpfile.$i.train.src -out $tmpfile.$i.keywords -n $topN"); # keyword file
+    execute("$keywordLoc -in $tmpfile.$i.train.src -out $tmpfile.$i.bigram -n $topN -nGram 2"); # bigram file
+
+    # create crf features
+    execute("$tr2crfppLoc -in $tmpfile.$i.test.src -out $tmpfile.$i.test -k $tmpfile.$i.keywords -b $tmpfile.$i.bigram 1>/dev/null");
+  }
+} else {
+  for (my $i = 0; $i < $folds; $i++) {
+    execute("$tr2crfppLoc $tmpfile.$i.test.src > $tmpfile.$i.test");
+  }
+}
+
+# construct training data
+for (my $i = 0; $i < $folds; $i++) {
+  for (my $j = 0; $j < $folds; $j++) {
+    if ($j == $i) {next; }
+    else {
+      execute("cat $tmpfile.$j.test >> $tmpfile.$i.train");
+    }
+  }
+
+}
+
+# train
+print STDERR "### Training ...\n"; # Thang add
+for (my $i = 0; $i < $folds; $i++) {
+  execute("$crf_learnLoc -m 100 -p $numCpus -f 3 -c 3 $crfTemplateLoc $tmpfile.$i.train $tmpfile.$i.model");
+}
+
+# test
+print STDERR "### Testing ...\n"; # Thang add
+for (my $i = 0; $i < $folds; $i++) {
+  execute("$crf_testLoc -m $tmpfile.$i.model $tmpfile.$i.test > $tmpfile.$i.out");
+  
+  # convert from line-level format to token-level format
+  if(!$isOldModel){
+    print STDERR "### Convert from line-level format to token-level format...\n";
+    execute("$convertLoc -in1 $tmpfile.$i.test.src -in2 $tmpfile.$i.out -out $tmpfile.$i.out.convert 2>$tmpfile.$i.out.log");
+    execute("cat $tmpfile.$i.out.convert >> $tmpfile.all.out");
+  } else {
+    execute("cat $tmpfile.$i.out >> $tmpfile.all.out");
+  }
+}
+
+  
+# eval
+#for (my $i = 0; $i < $folds; $i++) {
+#  my $cmd = "$conllevalLoc -r -d \"	\" < $tmpfile.$i.out";
+#  print "$cmd\n";
+#  system ($cmd);
+#}
+print STDERR "### Evaluating ...\n"; # Thang add
+execute("$conllevalLoc -r -d \"	\" < $tmpfile.all.out");
+
+# clean up
+#`rm -f $tmpfile*`;
+
+## Thang add ##
+sub execute {
+  my ($cmd) = @_;
+  print STDERR "Executing: $cmd\n";
+  system($cmd);
+}
+## End Thang add ##
+
+######################################################################
+# 2-fold -m 100 -f 3 -c 3 on svm_headerparse.txt 
+### Token-level model
+# accuracy:  96.76%; precision:  96.76%; recall:  96.76%; FB1:  96.76
+# abstract: precision:  99.04%; recall:  99.58%; FB1:  99.31  121194
+# address: precision:  93.75%; recall:  89.43%; FB1:  91.54  5342
+# affiliation: precision:  92.65%; recall:  92.06%; FB1:  92.35  9823
+# author: precision:  94.11%; recall:  92.57%; FB1:  93.33  7825
+# date: precision:  89.94%; recall:  92.59%; FB1:  91.25  1084
+# degree: precision:  80.67%; recall:  84.30%; FB1:  82.45  2163
+# email: precision:  91.54%; recall:  90.30%; FB1:  90.91  1667
+# intro: precision:  97.56%; recall:  96.49%; FB1:  97.03  1354
+# keyword: precision:  92.73%; recall:  81.24%; FB1:  86.60  1966
+# note: precision:  86.70%; recall:  87.27%; FB1:  86.99  11220
+# page: precision: 100.00%; recall: 100.00%; FB1: 100.00  288
+# phone: precision:  92.17%; recall:  79.37%; FB1:  85.29  434
+# pubnum: precision:  91.55%; recall:  85.83%; FB1:  88.60  556
+# title: precision:  94.22%; recall:  95.10%; FB1:  94.66  9043
+# web: precision:  77.32%; recall:  45.45%; FB1:  57.25  97
+#
+### Line-level model
+#accuracy:  97.15%; precision:  97.15%; recall:  97.15%; FB1:  97.15
+# abstract: precision:  98.70%; recall:  99.77%; FB1:  99.23  121842
+# address: precision:  95.81%; recall:  97.01%; FB1:  96.40  5653
+# affiliation: precision:  95.33%; recall:  95.94%; FB1:  95.63  9955
+# author: precision:  93.31%; recall:  95.05%; FB1:  94.17  8084
+# date: precision:  87.00%; recall:  94.07%; FB1:  90.40  1131
+# degree: precision:  98.38%; recall:  58.55%; FB1:  73.41  1232
+# email: precision:  97.37%; recall:  96.96%; FB1:  97.16  1671
+# intro: precision:  90.77%; recall:  99.85%; FB1:  95.10  1506
+# keyword: precision:  95.29%; recall:  80.17%; FB1:  87.08  1888
+# note: precision:  89.86%; recall:  82.55%; FB1:  86.05  10236
+# page: precision: 100.00%; recall: 100.00%; FB1: 100.00  288
+# phone: precision:  98.02%; recall:  88.49%; FB1:  93.01  455
+# pubnum: precision:  92.18%; recall:  85.93%; FB1:  88.95  550
+# title: precision:  93.66%; recall:  98.05%; FB1:  95.80  9379
+# web: precision:  98.40%; recall:  75.93%; FB1:  85.71  125
+#
+# Model                                                       Type of features   FB1   # features
+# Token-level model                                                             96.76% 1091340
+# Line-level  model
+#  fullToken                                                  Parscit features  95.72%   60285
+#  fullToken.firstToken                                       Parscit features  95.94%  128745
+#  fullToken.firstSecondLastToken                             Parscit features  96.59%  259050
+#  fullToken.firstSecondSecondLastToken                       Parscit features  96.68%  324255
+#  fullToken.firstSecondSecondLastToken.back1                 linking           96.86%  392625
+#  fullToken.firstSecondSecondLastToken.back1.forw1           linking           96.98%  458175 further linking reduces performance
+#  fullToken.firstSecondSecondLastToken.back1.forw1.keyword   keyword           97.15%  461085