tokenizer 0.0.1.prealpha

data/Rakefile

@@ -0,0 +1,37 @@
+begin
+  require 'rake'
+rescue LoadError
+  require 'rubygems'
+  require 'rake'
+end
+
+# we can require 'rake/clean' to add 'clobber' and 'clean' tasks
+require 'rake/clean'
+
+
+
+SRC = FileList['**/*.rb']
+
+CLOBBER.include('doc', '**/*.html', '**/*.gem')
+
+# testing
+require 'rake/testtask'
+Rake::TestTask.new do |t|
+  t.test_files = FileList.new('test/**/*.rb').to_a
+end
+
+# Build the gem package
+load 'tokenizer.gemspec'
+require 'rubygems/package_task'
+Gem::PackageTask.new(GEMSPEC).define
+
+# Generate documentation
+require 'rdoc/task'
+RDoc::Task.new do |rdoc|
+  rdoc.rdoc_files.include('README', 'LICENSE', 'lib/**/*')
+end
+
+desc "Open an irb session preloaded with this library."
+task :console do
+  sh "irb -rubygems -I lib -r tokenizer.rb"
+end

data/bin/tokenize

@@ -0,0 +1,8 @@
+#!/usr/bin/ruby -nw
+
+BEGIN {
+  $/ = " "
+}
+
+puts $_
+

data/lib/tokenizer/version.rb

@@ -0,0 +1,3 @@
+module Tokenizer
+  VERSION = '0.0.1.prealpha'
+end

metadata

@@ -0,0 +1,76 @@
+--- !ruby/object:Gem::Specification 
+name: tokenizer
+version: !ruby/object:Gem::Version 
+  hash: 196621262
+  prerelease: 6
+  segments: 
+  - 0
+  - 0
+  - 1
+  - prealpha
+  version: 0.0.1.prealpha
+platform: ruby
+authors: 
+- Andrei Beliankou
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-05-05 00:00:00 Z
+dependencies: []
+
+description: A simple tokenizer for NLP tasks.
+email: a.belenkow@uni-trier.de
+executables: 
+- tokenize
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- LICENSE
+files: 
+- lib/tokenizer/version.rb
+- bin/tokenize
+- README.rdoc
+- LICENSE
+- Rakefile
+- test/test_tokenizer.rb
+homepage: http://www.uni-trier.de/index.php?id=34451
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- -m
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 31
+      segments: 
+      - 1
+      - 8
+      version: "1.8"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      hash: 25
+      segments: 
+      - 1
+      - 3
+      - 1
+      version: 1.3.1
+requirements: []
+
+rubyforge_project: tokenizer
+rubygems_version: 1.7.2
+signing_key: 
+specification_version: 3
+summary: Tokenizer is a linguistic tool intended to split a text into tokens.
+test_files: 
+- test/test_tokenizer.rb