opener-tokenizer-base 1.0.0

data/core/split-sentences.pl

@@ -0,0 +1,114 @@
+#!/usr/bin/perl -w
+
+# Based on Preprocessor written by Philipp Koehn
+# Changed by aazpeitia (aazpeitia@vicomtech.org)
+
+use strict;
+
+my %NONBREAKING_PREFIX = ();
+my $LANGUAGE;
+
+sub init_sentence_splitter {
+	$LANGUAGE = shift(@_);
+	%NONBREAKING_PREFIX = %{ shift(@_) };
+}
+
+sub split_sentences {
+
+	my $input_text = shift(@_);
+	chomp($input_text);
+	my $text = "";
+	if (/^<.+>$/ || /^\s*$/) {
+		#time to process this block, we've hit a blank or <p>
+		#&do_it_for($text,$input_text);
+		#print "<P>\n" if (/^\s*$/ && $text); ##if we have text followed by <P>
+		#$text = "";
+		$text .= &do_it_for($text,$input_text);
+		$text .= "<P>\n" if (/^\s*$/ && $text); ##if we have text followed by <P>
+	}
+	else {
+		#append the text, with a space
+		$text .= $input_text. " ";
+	}
+	$text = &do_it_for($text,$input_text);
+	return split("\n", $text);
+}
+
+sub do_it_for {
+	my($text,$markup) = @_;
+	$text = &preprocess($text) if $text;
+	#print "$markup\n" if ($markup =~ /^<.+>$/);
+	return $text;
+}
+
+sub preprocess {
+	#this is one paragraph
+	my($text) = @_;
+
+	# clean up spaces at head and tail of each line as well as any double-spacing
+	$text =~ s/ +/ /g;
+	$text =~ s/\n /\n/g;
+	$text =~ s/ \n/\n/g;
+	$text =~ s/^ //g;
+	$text =~ s/ $//g;
+		
+	#####add sentence breaks as needed#####
+	#non-period end of sentence markers (?!) followed by sentence starters.
+
+#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+	#$text =~ s/([?!]) +([\'\"\(\[\¿\¡\p{IsPi}]*[\p{IsUpper}])/$1\n$2/g;
+	$text =~ s/([?!]) +([\'\"\(\[\¿\¡\p{IsPi}]*[\w])/$1\n$2/g;
+#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+		
+	#multi-dots followed by sentence starters
+	$text =~ s/(\.[\.]+) +([\'\"\(\[\¿\¡\p{IsPi}]*[\p{IsUpper}])/$1\n$2/g;
+	
+	# add breaks for sentences that end with some sort of punctuation inside a quote or parenthetical and are followed by a possible sentence starter punctuation and upper case
+	$text =~ s/([?!\.][\ ]*[\'\"\)\]\p{IsPf}]+) +([\'\"\(\[\¿\¡\p{IsPi}]*[\ ]*[\p{IsUpper}])/$1\n$2/g;
+		
+	# add breaks for sentences that end with some sort of punctuation are followed by a sentence starter punctuation and upper case
+	$text =~ s/([?!\.]) +([\'\"\(\[\¿\¡\p{IsPi}]+[\ ]*[\p{IsUpper}])/$1\n$2/g;
+	
+	# special punctuation cases are covered. Check all remaining periods.
+	my $word;
+	my $i;
+	my @words = split(/ /,$text);
+	$text = "";
+	for ($i=0;$i<(scalar(@words)-1);$i++) {
+		if ($words[$i] =~ /([\p{IsAlnum}\.\-]*)([\'\"\)\]\%\p{IsPf}]*)(\.+)$/) {
+			#check if $1 is a known honorific and $2 is empty, never break
+			my $prefix = $1;
+			my $starting_punct = $2;
+			if($prefix && $NONBREAKING_PREFIX{$prefix} && $NONBREAKING_PREFIX{$prefix} == 1 && !$starting_punct) {
+
+				#not breaking;
+			} elsif ($words[$i] =~ /(\.)[\p{IsUpper}\-]+(\.+)$/) {
+				#not breaking - upper case acronym	
+			} elsif($words[$i+1] =~ /^([ ]*[\'\"\(\[\¿\¡\p{IsPi}]*[ ]*[\p{IsUpper}0-9])/) {
+				#the next word has a bunch of initial quotes, maybe a space, then either upper case or a number
+				$words[$i] = $words[$i]."\n" unless ($prefix && $NONBREAKING_PREFIX{$prefix} && $NONBREAKING_PREFIX{$prefix} == 2 && !$starting_punct && ($words[$i+1] =~ /^[0-9]+/));
+				#we always add a return for these unless we have a numeric non-breaker and a number start
+			}
+			
+		}
+		$text = $text.$words[$i]." ";
+	}
+	
+	#we stopped one token from the end to allow for easy look-ahead. Append it now.
+	$text = $text.$words[$i];
+	
+	# clean up spaces at head and tail of each line as well as any double-spacing
+	$text =~ s/ +/ /g;
+	$text =~ s/\n /\n/g;
+	$text =~ s/ \n/\n/g;
+	$text =~ s/^ //g;
+	$text =~ s/ $//g;
+	
+	#add trailing break
+	$text .= "\n" unless $text =~ /\n$/;
+	
+	return $text;
+	
+}
+
+1;

data/core/text-fixer.pl

@@ -0,0 +1,169 @@
+#!/usr/bin/perl -w
+
+# reads input text and fixes some mistakes
+# developed by Andoni Azpeitia
+
+use utf8;
+
+my %NONBREAKING_PREFIX = ();
+my $LANGUAGE;
+
+my $START_QUOTES_REGEX = "“|‘|«|‹";
+my $END_QUOTES_REGEX = "”|’|»|›";
+
+sub init_text_fixer {
+	$LANGUAGE = shift(@_);
+	%NONBREAKING_PREFIX = %{ shift(@_) };
+}
+
+sub fix_text {
+
+	my($text) = shift(@_);
+	
+	chomp($text);
+	
+	#fix encoding
+	$text = &fix_encoding($text);
+	
+
+	#word token method
+	my @words = split(/\s/,$text);
+	$text = "";
+	for (my $i=0;$i<(scalar(@words));$i++) {
+		my $word = $words[$i];
+		#Kumi Naidoo said: “bla bla bla.”Bla bla => Kumi Naidoo said: “bla bla bla”. Bla bla
+		
+		if ( $word =~ /^(\S+)\.($END_QUOTES_REGEX)($START_QUOTES_REGEX*\p{IsUpper}\S*)$/ ) {
+			my $pre = $1;
+			my $quote = $2;
+			my $post = $3;
+			
+			$word = $pre.$quote.". ".$post;
+		}
+		#to a "breach of trust." A German => to a "breach of trust". A German
+		elsif ( $word =~ /^(\S+)\.($END_QUOTES_REGEX)$/ ) {
+			my $pre = $1;
+			my $quote = $2;
+			if ( ($i<scalar(@words)-1 && $words[$i+1] =~ /^$START_QUOTES_REGEX*\p{IsUpper}\S*$/ )) {
+				$word = $pre.$quote.".";
+			}
+			elsif ($i==scalar(@words)-1) {
+				$word = $pre.$quote.".";
+			}
+		}
+		#OpeNER is amazing.OpeNER is cool. => OpeNER is amazing. OpeNER is cool.
+		elsif ( $word =~ /^(\S+)\.(\S+)$/) {
+			my $pre = $1;
+			my $post = $2;
+			if (($pre =~ /\./ && $pre =~ /\p{IsAlpha}/) || ($NONBREAKING_PREFIX{$pre} && $NONBREAKING_PREFIX{$pre}==1) || ($post =~ /^[\p{IsLower}]/) ) {
+				#no change
+			} elsif (($NONBREAKING_PREFIX{$pre} && $NONBREAKING_PREFIX{$pre}==2) &&  ($post =~ /^[0-9]+/) ) {
+				#no change
+			} else {
+				$word = $pre.". ".$post;
+			}
+		}
+		#OpeNER is amazing .OpeNER is cool. => OpeNER is amazing. OpeNER is cool.
+		elsif ( $word =~ /^\.(\p{IsUpper}\S+)$/ ) {
+			my $post = $1;
+			if ( $i>0 &&  $words[$i-1] =~ /^(\S+)$/) {
+				$word = ". ".$post;
+			}
+		}
+		$text .= $word." ";
+	}
+	#freedoms." 'Outrageous'Although => freedoms". 'Outrageous' Although
+	#$text =~ s/(\")([^\"]+)(\. ?)(\")/$1$2$4$3/g;
+	#$text =~ s/(\')([^\']+)(\. ?)(\')/$1$2$4$3/g;
+	return $text;
+}
+
+sub fix_encoding {
+
+	my $text = shift(@_);	
+
+	$text =~ s/’/'/g;
+	$text =~ s/À/À/g;
+	$text =~ s/“/“/g;
+	$text =~ s///g;
+	$text =~ s//\"/g;
+############################################
+############################################
+	$text =~ s/…/…/g; # elipsis
+	$text =~ s/…/…/g; # elipsis
+	$text =~ s/–/–/g; # long hyphen
+	$text =~ s/’/’/g; #curly apostrophe
+	$text =~ s/“/“/g; # curly open quote
+	$text =~ s/â€/”/g; # curly close quote
+	$text =~ s/»/»/g;
+	$text =~ s/«/«/g;
+############################################
+	$text =~ s/á/á/g;
+	$text =~ s/é/é/g;
+	$text =~ s/Ã\*/í/g;
+	$text =~ s/ó/ó/g;
+	$text =~ s/ú/ú/g;
+
+	$text =~ s/Á/Á/g;
+	$text =~ s/É/É/g;
+	$text =~ s/Í/Í/g;
+	$text =~ s/Ó/Ó/g;
+	$text =~ s/Ú/Ú/g;
+############################################
+	$text =~ s/ñ/ñ/g;
+	$text =~ s/ç/ç/g;
+	$text =~ s/Å“/œ/g;
+
+	$text =~ s/Ñ/Ñ/g;
+	$text =~ s/Ç/Ç/g;
+	$text =~ s/Å’/Œ/g;
+############################################
+	$text =~ s/©/©/g;
+	$text =~ s/®/®/g;
+	$text =~ s/â„¢/™/g;
+	$text =~ s/Ø/Ø/g;
+	$text =~ s/ª/ª/g;
+############################################
+	$text =~ s/ä/ä/g;
+	$text =~ s/ë/ë/g;
+	$text =~ s/ï/ï/g;
+	$text =~ s/ö/ö/g;
+	$text =~ s/ü/ü/g;
+
+	$text =~ s/Ä/Ä/g;
+	$text =~ s/Ë/Ë/g;
+	$text =~ s/Ï /Ï/g;
+	$text =~ s/Ö /Ö/g;
+	$text =~ s/Ãœ/Ü/g;
+############################################
+	$text =~ s/à/à/g;
+	$text =~ s/è/è/g;
+	$text =~ s/ì/ì/g;
+	$text =~ s/ò/ò/g;
+	$text =~ s/ù/ù/g;
+
+	$text =~ s/À/À/g;
+	$text =~ s/È/È/g;
+	$text =~ s/ÃŒ/Ì/g;
+	$text =~ s/Ã’/Ò/g;
+	$text =~ s/Ù/Ù/g;
+############################################
+	$text =~ s/â/â/g;
+	$text =~ s/ê/ê/g;
+	$text =~ s/î/î/g;
+	$text =~ s/ô/ô/g;
+	$text =~ s/û/û/g;
+
+	$text =~ s/Â/Â/g;
+	$text =~ s/Ê/Ê/g;
+	$text =~ s/ÃŽ/Î/g;
+	$text =~ s/Ô/Ô/g;
+	$text =~ s/Û/Û/g;
+############################################
+	$text =~ s/É/E/g;
+
+
+	return $text;
+}
+
+1;

data/core/tokenizer-cli.pl

@@ -0,0 +1,363 @@
+#!/usr/bin/perl -w
+
+# Sample Tokenizer
+# written by Josh Schroeder, based on code by Philipp Koehn
+# changed by Haritz Arzelus (#2012/11/19) Aitor García and Andoni Azpeitia
+
+use FindBin;
+
+use lib "$FindBin::Bin/lib";
+
+use Encode::Guess;
+use Time::Stamp;
+
+require "$FindBin::Bin"."/text-fixer.pl";
+require "$FindBin::Bin"."/split-sentences.pl";
+require "$FindBin::Bin"."/tokenizer.pl";
+require "$FindBin::Bin"."/load-prefixes.pl";
+
+no warnings;
+use encoding 'utf8';
+
+binmode(STDIN, ":utf8");
+binmode(STDOUT, ":utf8");
+binmode(STDERR, ":utf8");
+
+#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+#use FindBin qw($Bin);
+#use strict;
+#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+#use Time::HiRes;
+
+#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+#my $mydir = "$Bin/nonbreaking_prefixes";
+#changed by me (aitor) to point to the directory of the script, instead of current working directory
+#my $mydir = "$FindBin::Bin"."/nonbreaking_prefixes";
+#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+
+#my $start = [ Time::HiRes::gettimeofday( ) ];
+
+#while (@ARGV) {
+#	$_ = shift;
+#	/^-l$/ && ($language = shift, next);
+#	/^-q$/ && ($QUIET = 1, next);
+#	/^-h$/ && ($HELP = 1, next);
+#}
+
+#if (!$QUIET) {
+#	print STDERR "Tokenizer v3\n";
+#	print STDERR "Language: $language\n";
+#}
+
+#argument variables
+my %NONBREAKING_PREFIX = ();
+my $SENT_VERSION = "0.0.1";
+my $TOK_VERSION = "1.0.1";
+my $FILE = "";
+my $LANGUAGE;
+my $NOTIMESTAMP = 0;
+my $HELP = 0;
+
+my $SUBSTITUTE = "####";
+
+if (checkArguments(\@ARGV) == 1) {
+  if ($HELP == 1) {
+    displayHelp();
+    exit 0;
+  }
+}
+else {
+  displayHelp();
+  exit -1;
+}
+
+# load nonbreaking prefixes and init both tokenizer and sentence splitter
+%NONBREAKING_PREFIX = %{ &load_prefixes($LANGUAGE) };
+
+&init_text_fixer($LANGUAGE, \%NONBREAKING_PREFIX);
+&init_sentence_splitter($LANGUAGE, \%NONBREAKING_PREFIX);
+&init_tokenizer($LANGUAGE, \%NONBREAKING_PREFIX);
+
+
+# get timestamp
+my $timestamp = "0000-00-00T00:00:00Z";
+if ($NOTIMESTAMP == 0) {
+  $timestamp = timestamp();
+}
+
+# print kaf header
+if ($FILE ne "") {
+
+  my $i = rindex($FILE, ".");
+  my $filename = substr($FILE, 0, $i);
+  my $filetype = uc(substr($FILE, $i+1, length($FILE)-length($filename)-1));
+  print_kafheader($filename, $filetype, $timestamp, $LANGUAGE);
+}
+else {
+  print_kafheader_nofile($timestamp);
+}
+print "  <text>\n";
+
+# process text
+my $sent = 1;
+my $para = 1;
+my $counter = 0;
+my $charcount = 0;
+while(<STDIN>) {
+
+  if (/^<.+>$/ || /^\s*$/) {
+    #don't try to tokenize XML/HTML tag lines
+    chomp($_);
+    #print $_;
+  }
+  else {
+
+    #fix input text
+    my $text = &fix_text($_);
+
+    #split sentences
+    my @sentences = &split_sentences($text);
+    my $index = 0;
+    my $last_index = 0;
+    my $last_offset = -1;
+    my $j = 0;
+    foreach my $sentence (@sentences) {
+#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+      #print &tokenize($_);
+      $tok = tokenize($sentence);
+      #tokenize some especial characters
+      $tok =~ s/([“|”|…|—|–|«|»])([^ ])/$1 $2/g;
+      $tok =~ s/([^ ])([“|”|…|—|–|«|»])/$1 $2/g;
+      #detokenize tokens with @
+      $tok =~ s/ @ /@/g;
+      #detokenize some tokens with '
+      $tok =~ s/([DLNO]) '/$1'/g;
+      #$tok =~ s/([DLNO])' /$1'/g;      cambiado por Andoni Azpeitia Vicomtech    L' armée => L'armée
+      $tok =~ s/o( )?'( )?clock/o'clock/g;
+      $tok =~ s/ ' ([0-9][0-9]s)/ '$1/g;
+      #detokenize some time formats
+      $tok =~ s/([0-9][0-9]*) ' ([0-9][0-9]*) "/$1'$2"/g;
+      $tok =~ s/([0-9][0-9]*) : ([0-9][0-9])/$1:$2/g;
+      #detokenize some height formats
+      $tok =~ s/([0-9][0-9]*) ' ([0-9][0-9])/$1'$2/g;
+      #tokenize two dashes
+      $tok =~ s/\-\-/ \-\-/g;
+      #correct ºC tokenization
+      $tok =~ s/([0-9])( )?º( )?C/$1 ºC/g;
+      $tok =~ s/ +/ /g;
+#<<<<<<<<<<<<<<<
+      #changed by me (aitor) to format the output as a kind of dummy KAF format
+      chomp($tok);
+
+      @tokens = split(/ /, $tok);
+
+      my $i = 0;
+      foreach my $token (@tokens) {
+
+        $index = index($_, $token, $last_index);
+
+	#if token was substituted at tokenization, be careful
+        if ( $token eq "\"".$SUBSTITUTE ) {
+          $index = index($_, "'", $last_index);
+	  $token = "\"";
+        }
+        elsif ( $token eq "\'".$SUBSTITUTE ) {
+          $index = index($_, "`", $last_index);
+	  $token = "\'";
+        }
+
+        my $offset = $charcount + $index;
+	#if input text has been preprocesed and tokens has been moved ( 'hello.' => 'hello'.),
+        # offset of the "." char is at the left of "'" char not at the right
+        if ( $index==-1 ) {
+	  $index = index($_, $token, $last_index-2);
+	  $offset = $charcount + $index;
+        }
+	#make sure that found offset is not an offset of the same char at other position so,
+	# find offset of the next token and compare
+	# next token is at the same sentence
+	elsif ( $i<scalar(@tokens)-1 ) {
+	  my $next_token_index = $charcount + index($_, $tokens[$i+1], length($token) + $last_index-1);
+          if ( $index==-1 || ($next_token_index > -1 && $offset > $next_token_index+1) ) {
+	    $index = index($_, $token, $last_index-2);
+	    $offset = $charcount + $index;
+	  }
+	}
+	#make sure that found offset is not an offset of the same char at other position so,
+	# find offset of the next token and compare
+	# next token is at next sentence
+	elsif ( $sent < scalar(@sentences)) {
+	  my $next_sentence = $sentences[$j+1];
+	  $next_sentence = &tokenize($next_sentence);
+	  my @next_tokens = split(/ /, $next_sentence);
+          my $next_token = $next_tokens[0];
+	  my $next_token_index = $charcount + index($_, $next_token, length($token) + $last_index-1);
+	  if ( ($next_token_index > -1 && $offset > $next_token_index+1) ) {
+	    $index = index($_, $token, $last_index-2);
+	    $offset = $charcount + $index;
+	  }
+	}
+
+	my $token_length = length($token);
+	&print_line(++$counter, $sent, $para, $offset, $token_length, $token);
+
+        $last_index = $index + $token_length;
+	$last_offset = $offset;
+        $i++;
+      }
+
+#>>>>>>>>>>>>>>>
+      $j++;
+      $sent++;
+#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+    }#foreach sentence
+
+    if (length($_) == 0) {
+      $charcount += 1;
+    }
+    else {
+      $charcount += length($_);
+    }
+    $para++;
+  }
+}#while(<STDIN>)
+print "  </text>\n";
+print "</KAF>\n";
+
+
+#prints word form in kaf format
+sub print_line {
+
+  my $wid=shift(@_);
+  my $sent=shift(@_);
+  my $para=shift(@_);
+  my $offset=shift(@_);
+  my $length=shift(@_);
+  my $token=shift(@_);
+
+  print "    <wf wid=\"w".$wid."\" sent=\"".$sent."\" para=\"".$para."\" offset=\"".$offset."\" length=\"". $length."\"><![CDATA[".$token."]]></wf>\n";
+}
+
+#prints kaf xml fomat header
+sub print_kafheader {
+  my $filename = shift(@_);
+  my $filetype = shift(@_);
+  my $timestamp = shift(@_);
+  my $LANGUAGE = shift(@_);
+  print "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n";
+  print "<KAF xml:lang=\"".$LANGUAGE."\" version=\"v1.opener\">\n";
+  print "  <kafHeader>\n";
+  print "    <fileDesc filename=\"".$filename."\" filetype=\"".$filetype."\" />\n";
+  print "    <linguisticProcessors layer=\"text\">\n";
+  print "      <lp name=\"opener-sentence-splitter-$LANGUAGE\" version=\"".$SENT_VERSION."\" timestamp=\"".$timestamp."\"/>\n";
+  print "      <lp name=\"opener-tokenizer-$LANGUAGE\" version=\"".$TOK_VERSION."\" timestamp=\"".$timestamp."\"/>\n";
+  print "    </linguisticProcessors>\n";
+  print "  </kafHeader>\n";
+}
+#prints kaf xml fomat header whithout filedesc
+sub print_kafheader_nofile {
+  my $timestamp = shift(@_);
+  print "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n";
+  print "<KAF xml:lang=\"".$LANGUAGE."\" version=\"v1.opener\">\n";
+  print "  <kafHeader>\n";
+  print "    <fileDesc />\n";
+  print "    <linguisticProcessors layer=\"text\">\n";
+  print "      <lp name=\"opener-sentence-splitter-$LANGUAGE\" version=\"".$SENT_VERSION."\" timestamp=\"".$timestamp."\"/>\n";
+  print "      <lp name=\"opener-tokenizer-$LANGUAGE\" version=\"".$TOK_VERSION."\" timestamp=\"".$timestamp."\"/>\n";
+  print "    </linguisticProcessors>\n";
+  print "  </kafHeader>\n";
+}
+
+sub checkArguments {
+  my $argref = shift(@_);
+  my @arg = @ { $argref };
+  my $correct = 1;
+  if (scalar(@arg) > 0) {
+    for (my $i = 0; $i < scalar(@arg); $i++) {
+      if (lc($arg[$i]) eq "-l") {
+        if(scalar(@arg) > $i+1) {
+          if (lc($arg[$i+1]) ne "-t" && lc($arg[$i+1]) ne "-f" && lc($arg[$i+1]) ne "-l" && checkLanguage($arg[$i+1]) == 1) {
+            $LANGUAGE = $arg[$i+1];
+          }
+          else {
+            $correct = 0;
+            print STDERR "Error: language \"".$arg[$i+1]."\" not supported\n";
+          }
+        }
+        else {
+          $correct = 0;
+          print STDERR "Error: language don't specified\n";
+        }
+      }
+      elsif (lc($arg[$i]) eq "-f") {
+        if(scalar(@arg) > $i+1 && lc($arg[$i+1]) ne "-t" && lc($arg[$i+1]) ne "-f" && lc($arg[$i+1]) ne "-l") {
+          $FILE = $arg[$i+1];
+        }
+        else {
+          $correct = 0;
+          print STDERR "Error: file's name empty\n";
+        }
+      }
+      elsif (lc($arg[$i]) eq "-t") {
+        $NOTIMESTAMP = 1;
+      }
+      elsif (lc($arg[$i]) eq "--help") {
+        $HELP = 1;
+      }
+    }
+    if ($LANGUAGE ne "") {
+      return $correct;
+    }
+    else {
+      print STDERR "Error: language don't specified\n";
+      return 0;
+    }
+  }
+  else {
+    print STDERR "Error: language don't specified\n";
+    return 0
+  }
+}
+
+sub checkLanguage {
+  my $language = shift(@_);
+  if ($language eq "en") { return 1; }
+  elsif ($language eq "es") { return 1; }
+  elsif ($language eq "fr") { return 1; }
+  elsif ($language eq "it") { return 1; }
+  elsif ($language eq "de") { return 1; }
+  elsif ($language eq "nl") { return 1; }
+  else { return -1 }
+}
+
+sub displayHelp {
+  print STDERR "\nThis aplication reads a text from standard input in order to tokenize.\n";
+  print STDERR "Application arguments:\n";
+  print STDERR "-l, --language  input text's language.\n";
+  print STDERR "-f, --filename  (optional) file's name.\n";
+  print STDERR "-t,             (optional) o use static timestamp at KAF header.\n";
+  print STDERR "--help,         outputs aplication help.\n";
+}
+
+sub timestamp {
+  my $time = Time::Stamp::gmstamp();
+  return $time;
+}
+
+sub detect_encoding {
+  my $file = shift(@_);
+  my $enc;
+  open(FILE,$file);
+  binmode(FILE);
+  if(read(FILE,my $filestart, 500)) {
+    $enc = guess_encoding($filestart);
+  }
+  close(FILE);
+  if (ref($enc)) {
+    return $enc->name;
+  }
+  else {
+    return "utf8";
+  }
+}
+