opener-tokenizer-base 1.0.0

data/core/tokenizer.pl

@@ -0,0 +1,145 @@
+#!/usr/bin/perl -w
+
+# This class tokenizes input sentence
+# Implements Moses tokenizer and it has been modified
+# for OpeNER by Aitor García and Andoni Azpeitia
+
+use FindBin;
+use utf8;
+
+my %NONBREAKING_PREFIX = ();
+my $LANGUAGE;
+my $SUBSTITUTE = "####";
+
+sub init_tokenizer {
+	$LANGUAGE = shift(@_);
+	%NONBREAKING_PREFIX = %{ shift(@_) };
+}
+
+sub tokenize {
+	
+	my($text) = shift(@_);
+	chomp($text);
+#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+	#tokenize the dashes of the beginning of the lines
+	$text =~ s/^\-([^ ])/\- $1/g;
+
+	# turn ’ into '
+	$text =~ s/Ž/\'/g;
+#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+
+	$text = " $text ";
+	# seperate out all "other" special characters
+	$text =~ s/([^\p{IsAlnum}\s\.\'\`\,\-\’])/ $1 /g;
+        #$text =~ s/([^\p{IsAlnum}\s\.\'\`\,\-])/ $1 /g;
+	#multi-dots stay together
+	$text =~ s/\.([\.]+)/ DOTMULTI$1/g;
+	while($text =~ /DOTMULTI\./) {
+		$text =~ s/DOTMULTI\.([^\.])/DOTDOTMULTI $1/g;
+		$text =~ s/DOTMULTI\./DOTDOTMULTI/g;
+	}
+	# seperate out "," except if within numbers (5,300)
+	$text =~ s/([^\p{IsN}])[,]([^\p{IsN}])/$1 , $2/g;
+	# separate , pre and post number
+	$text =~ s/([\p{IsN}])[,]([^\p{IsN}])/$1 , $2/g;
+	$text =~ s/([^\p{IsN}])[,]([\p{IsN}])/$1 , $2/g;
+
+	# turn `into '
+	$text =~ s/\`/\'$SUBSTITUTE/g;
+
+	#turn '' into "
+	$text =~ s/\'\'/ \"$SUBSTITUTE /g;
+#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+	#tokenize the words like '05-'06
+	$text =~ s/(['|’])([0-9][0-9])\-(['|’])([0-9][0-9])/$1$2 - $3$4/g;
+	#replace the ' with ### to don't tokenize words like '90
+	$text =~ s/ ['|’]([0-9][0-9])/ ###$1/g;
+#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+	if ($LANGUAGE eq "en") {
+		#split contractions right
+		$text =~ s/([^\p{IsAlpha}])(['|’])([^\p{IsAlpha}])/$1 $2 $3/g;
+		$text =~ s/([^\p{IsAlpha}\p{IsN}])(['|’])([\p{IsAlpha}])/$1 $2 $3/g;
+		$text =~ s/([\p{IsAlpha}])(['|’])([^\p{IsAlpha}])/$1 $2 $3/g;
+		$text =~ s/([\p{IsAlpha}])(['|’])([\p{IsAlpha}])/$1 $2$3/g;
+		#special case for "1990's"
+		$text =~ s/([\p{IsN}])(['|’])([s])/$1 $2$3/g;
+	} elsif ($LANGUAGE eq "fr") {
+		#split contractions left
+		$text =~ s/([^\p{IsAlpha}])(['|’])([^\p{IsAlpha}])/$1 $2 $3/g;
+		$text =~ s/([^\p{IsAlpha}])(['|’])([\p{IsAlpha}])/$1 $2 $3/g;
+		$text =~ s/([\p{IsAlpha}])(['|’])([^\p{IsAlpha}])/$1 $2 $3/g;
+		$text =~ s/([\p{IsAlpha}])(['|’])([\p{IsAlpha}])/$1$2 $3/g;
+	} elsif ($LANGUAGE eq "it") {
+		#split contractions left
+		$text =~ s/([^\p{IsAlpha}])(['|’])([^\p{IsAlpha}])/$1 $2 $3/g;
+		$text =~ s/([^\p{IsAlpha}])(['|’])([\p{IsAlpha}])/$1 $2 $3/g;
+		$text =~ s/([\p{IsAlpha}])(['|’])([^\p{IsAlpha}])/$1 $2 $3/g;
+		$text =~ s/([\p{IsAlpha}])(['|’])([\p{IsAlpha}])/$1$2 $3/g;
+		$text =~ s/([^\p{IsAlpha}\p{IsN}]po) (['|’])([^\p{IsAlpha}])/$1$2 $3/g; # rule for "po'"
+	} else {
+		$text =~ s/\'/ \' /g;
+	}
+#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+	#replace the ### with ' to tokenize words like '90
+	$text =~ s/ ###([0-9][0-9])/ '$1/g;
+#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+
+	#word token method
+	my @words = split(/\s/,$text);
+	$text = "";
+	for (my $i=0;$i<(scalar(@words));$i++) {
+		my $word = $words[$i];
+		if ( $word =~ /^(\S+)\.$/) {
+			my $pre = $1;
+			if (($pre =~ /\./ && $pre =~ /\p{IsAlpha}/) || ($NONBREAKING_PREFIX{$pre} && $NONBREAKING_PREFIX{$pre}==1) || ($i<scalar(@words)-1 && ($words[$i+1] =~ /^[\p{IsLower}]/))) {
+				#no change
+			} elsif (($NONBREAKING_PREFIX{$pre} && $NONBREAKING_PREFIX{$pre}==2) && ($i<scalar(@words)-1 && ($words[$i+1] =~ /^[0-9]+/))) {
+				#no change
+			} else {
+				$word = $pre." .";
+			}
+		}
+		$text .= $word." ";
+	}
+
+	# clean up extraneous spaces
+	$text =~ s/ +/ /g;
+	$text =~ s/^ //g;
+	$text =~ s/ $//g;
+
+	#restore multi-dots
+	while($text =~ /DOTDOTMULTI/) {
+		$text =~ s/DOTDOTMULTI/DOTMULTI./g;
+	}
+	$text =~ s/DOTMULTI/./g;
+
+	#detokenize URLs
+	$text = &detokenize_urls($text);
+	
+	#ensure final line break
+	$text .= "\n" unless $text =~ /\n$/;
+	return $text;
+}
+
+sub detokenize_urls {
+
+	my($text) = shift(@_);
+
+	$text =~ s/(\w{3,9}) : \/ \/ /$1:\/\//g;
+	my $URL_HEAD_PATTERN = "\\w{3,9}:\\/\\/|www";
+	my $URL_BODY_PATTERN = "\\w\\d\\.\\/\\-\\#;:=\\+\\?&_";
+	my $URL_SPECIAL_PATTERN = "\\/|\\?|=|&|\\+|_|\\#|:|;|\\-";
+	while ( $text =~ /($URL_HEAD_PATTERN)[$URL_BODY_PATTERN]+ ($URL_SPECIAL_PATTERN)/ ) {
+		$text =~ s/($URL_HEAD_PATTERN)([$URL_BODY_PATTERN]+) ($URL_SPECIAL_PATTERN) {0,1}(($URL_SPECIAL_PATTERN? {0,1})+)/$1.$2.$3.&clean($4)/eg;
+	}
+	
+	return $text;
+}
+
+sub clean {
+	my $text = shift(@_);
+	$text = s/ //g;
+	return $text;
+}
+
+1;

data/lib/opener/tokenizers/base.rb

@@ -0,0 +1,84 @@
+require_relative 'base/version'
+
+module Opener
+  module Tokenizers
+    class Base
+      attr_reader :language
+
+      def initialize(opts={})
+        @language ||= opts[:language] || lang
+      end
+
+      def command(opts=[])
+        "perl -I #{lib} #{kernel} #{language} #{opts.join(' ')}"
+      end
+
+      def run(opts=ARGV)
+        `#{command(opts)}`
+      end
+
+      def set_language(language)
+        @language = language
+      end
+
+      protected
+
+      def core_dir
+        File.expand_path("../../../../core", __FILE__)
+      end
+
+      def kernel
+        File.join(core_dir,'tokenizer-cli.pl')
+      end
+
+      def lib
+        File.join(core_dir,'lib/') # Trailing / is required
+      end
+
+      def language
+        return @language.nil? ? nil : "-l #{@language}"
+      end
+
+      def lang
+        'en'
+      end
+
+    end
+
+    class EN < Base
+      def lang
+        'en'
+      end
+    end
+
+    class DE < Base
+      def lang
+        'de'
+      end
+    end
+
+    class NL < Base
+      def lang
+        'nl'
+      end
+    end
+
+    class ES < Base
+      def lang
+        'es'
+      end
+    end
+
+    class IT < Base
+      def lang
+        'it'
+      end
+    end
+
+    class FR < Base
+      def lang
+        'fr'
+      end
+    end
+  end
+end

data/lib/opener/tokenizers/base/version.rb

@@ -0,0 +1,8 @@
+module Opener
+  module Tokenizers
+    class Base
+      VERSION = "1.0.0"
+    end
+  end
+end
+

data/opener-tokenizer-base.gemspec

@@ -0,0 +1,25 @@
+require File.expand_path('../lib/opener/tokenizers/base/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name                  = 'opener-tokenizer-base'
+  gem.version               = Opener::Tokenizers::Base::VERSION
+  gem.authors               = ['development@olery.com']
+  gem.summary               = 'Tokenize English, Dutch, German, Italian and Spanish to KAF'
+  gem.description           = gem.summary
+  gem.homepage              = 'http://opener-project.github.com/'
+  gem.has_rdoc              = "yard"
+  gem.required_ruby_version = ">= 1.9.2"
+
+  gem.files = Dir.glob([
+    'core/**/*',
+    'lib/**/*',
+    '*.gemspec',
+    'README.md'
+  ]).select { |file| File.file?(file) }
+
+  gem.executables = Dir.glob('bin/*').map { |file| File.basename(file) }
+
+  gem.add_development_dependency 'cucumber'
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'rake'
+end

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: opener-tokenizer-base
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- development@olery.com
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Tokenize English, Dutch, German, Italian and Spanish to KAF
+email: 
+executables:
+- tokenizer-en
+- tokenizer-it
+- tokenizer-nl
+- tokenizer-base
+- tokenizer-es
+- tokenizer-fr
+- tokenizer-de
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/tokenizer-base
+- bin/tokenizer-de
+- bin/tokenizer-en
+- bin/tokenizer-es
+- bin/tokenizer-fr
+- bin/tokenizer-it
+- bin/tokenizer-nl
+- core/lib/Data/OptList.pm
+- core/lib/Params/Util.pm
+- core/lib/Sub/Exporter.pm
+- core/lib/Sub/Exporter/Cookbook.pod
+- core/lib/Sub/Exporter/Tutorial.pod
+- core/lib/Sub/Exporter/Util.pm
+- core/lib/Sub/Install.pm
+- core/lib/Time/Stamp.pm
+- core/load-prefixes.pl
+- core/nonbreaking_prefixes/abbreviation_list.kaf
+- core/nonbreaking_prefixes/abbreviation_list.txt
+- core/nonbreaking_prefixes/nonbreaking_prefix.ca
+- core/nonbreaking_prefixes/nonbreaking_prefix.de
+- core/nonbreaking_prefixes/nonbreaking_prefix.el
+- core/nonbreaking_prefixes/nonbreaking_prefix.en
+- core/nonbreaking_prefixes/nonbreaking_prefix.es
+- core/nonbreaking_prefixes/nonbreaking_prefix.fr
+- core/nonbreaking_prefixes/nonbreaking_prefix.is
+- core/nonbreaking_prefixes/nonbreaking_prefix.it
+- core/nonbreaking_prefixes/nonbreaking_prefix.nl
+- core/nonbreaking_prefixes/nonbreaking_prefix.pl
+- core/nonbreaking_prefixes/nonbreaking_prefix.pt
+- core/nonbreaking_prefixes/nonbreaking_prefix.ro
+- core/nonbreaking_prefixes/nonbreaking_prefix.ru
+- core/nonbreaking_prefixes/nonbreaking_prefix.sk
+- core/nonbreaking_prefixes/nonbreaking_prefix.sl
+- core/nonbreaking_prefixes/nonbreaking_prefix.sv
+- core/split-sentences.pl
+- core/text-fixer.pl
+- core/tokenizer-cli.pl
+- core/tokenizer.pl
+- lib/opener/tokenizers/base.rb
+- lib/opener/tokenizers/base/version.rb
+- opener-tokenizer-base.gemspec
+homepage: http://opener-project.github.com/
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Tokenize English, Dutch, German, Italian and Spanish to KAF
+test_files: []
+has_rdoc: yard