rstt 0.9.0

data/.gitignore

@@ -0,0 +1,8 @@
+.rvmrc
+spec/rstt_rake_spec.rb
+
+lib/rstt/tt_settings.rb
+
+tmp/temp.html.haml
+todos.task
+lib/rstt/tt_settings.rb

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile

@@ -0,0 +1,15 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in rstt.gemspec
+gemspec
+
+group :development, :test do
+  gem "rspec"
+  gem "yard"
+  gem "syntax"
+end
+
+gem "slop"
+gem "thor"
+gem "activesupport"
+gem "celluloid"

data/Gemfile.lock

@@ -0,0 +1,44 @@
+PATH
+  remote: .
+  specs:
+    rstt (0.0.1)
+      activesupport
+      celluloid
+      slop
+      thor
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activesupport (3.2.1)
+      i18n (~> 0.6)
+      multi_json (~> 1.0)
+    celluloid (0.8.0)
+    diff-lcs (1.1.3)
+    i18n (0.6.0)
+    multi_json (1.0.4)
+    rspec (2.8.0)
+      rspec-core (~> 2.8.0)
+      rspec-expectations (~> 2.8.0)
+      rspec-mocks (~> 2.8.0)
+    rspec-core (2.8.0)
+    rspec-expectations (2.8.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.8.0)
+    slop (3.0.4)
+    syntax (1.0.0)
+    thor (0.14.6)
+    yard (0.7.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport
+  celluloid
+  rspec
+  rstt!
+  slop
+  syntax
+  thor
+  yard

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 LeFnord
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,55 @@
+TreeTagger for Ruby
+===================
+
+DESCRIPTION
+-----------
+
+The Ruby based wrapper for the TreeTagger by Helmut Schmid.
+Check it out if you are interested
+in Natural Language Processing (NLP) and Human Language Technology (HLT).
+
+INSTALLATION + REQUIREMENTS
+---------------------------
+
+Before you install the treetagger-ruby package please ensure you have downloaded and installe the [TreeTagger](http://www.ims.uni-stuttgart.de/projekte/corplex/TreeTagger/) itself.  
+(And pls, respect his terms of license)
+
+		gem install rstt
+		thor config:init /path/to/your/TreeTagger/
+
+or:
+
+		rstt -i /path/to/your/TreeTagger/
+
+USAGE
+-----
+
+1. You have some `class`, where you want to use Rstt ..  
+the input is given by: `Rstt.set_input lang: lang, content: content` with default language '`en`';  
+accessible languages could be found by `Rstt.language_codes`;  
+installed languages are stored in `Rstt::LANGUAGES` (cause it could be different)
+
+		class Foo
+			include Rstt
+			
+			def pos_tagging(lang,content)
+				Rstt.set_input lang: lang, content: content
+				Rstt.preprocessing
+				Rstt.tagging
+				processed_ data = Rstt.tagged
+			end
+		end
+
+	that's all, the processed data are accessible via `Rstt.tagged`,  
+	it is an Array, thereby each element self is an Array with following elements
+
+	1. the input word itself
+	2. the word class
+	3. the lemma of the input; depends on your input language
+
+2. or via CLI, check usage with `rstt -h`
+
+LICENSE
+-------
+
+see License.txt

data/Rakefile

@@ -0,0 +1,14 @@
+require "bundler/gem_tasks"
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+task :default => :spec
+
+require 'yard'
+YARD::Rake::YardocTask.new
+
+Dir["lib/tasks/**/*.rake"].sort.each { |ext| load ext }

data/bin/rstt

@@ -0,0 +1,93 @@
+#!/usr/bin/env ruby
+
+require "rstt"
+require "slop"
+require "awesome_print"
+
+# ToDo: add task for initializing ✓
+#       add task to show installed languages ✓
+#       add preprocessing task ✓
+#       add task for running from terminal ✓
+
+opts = Slop.parse do
+  banner "Usage:
+  rstt [-c /path/to/TreeTagger] | [{-t|-p|-t -p} -l lang {-i 'input text'|-f /input/file}] | [-l]\n
+  tagging:
+  rstt -t -l <language> -i <\"input text\">
+  rstt -t -l <language> -f <path/to/file>
+  preprocessing:
+  rstt -p -l <language> -i <\"input text\">
+  rstt -p -l <language> -f <path/to/file>
+  both:
+  rstt -t -p -l <language> -i <\"input text\">
+  rstt -t -p -l <language> -f <path/to/file>\n"
+  
+  on :h, :help, "print help message" do
+    puts help
+  end
+  
+  on :c, :configure=, "configure Rstt by setting TreeTagger path and installed languages [ARG path]", optional: false do
+    print "initilizing ..\n"
+    print `thor config:init #{self[:configure]}`
+  end
+  
+  
+  on :t, :tag, "run tagger", argument: false
+  on :p, :preprocess, "clean up html sites, strip non word characters", argument: false
+  on :l, :lang=, "language [with ARG input|without ARG show]",argument: :optional, optional: true
+  on :f, "input file", optional: true, argument: true
+  on :i, "input text", optional: true, argument: true
+end
+
+# show installed languages
+if opts.l? && opts[:lang].nil? && !(opts.t? || opts.p?)
+  puts "installed languages\n(use languages codes on the left)\n"
+  Rstt.installed_language_codes.each do |lang|
+    puts lang.first.to_s + ": " + lang.last
+  end
+end
+
+# input data
+# get input from file
+if opts.f? && !opts[:f].nil?
+  @input = ""
+  input_file = opts[:f]
+  # test if file exists
+  if File.exists?(input_file)
+    File.open(input_file).each do |line|
+      @input += line.strip + " "
+    end
+  else
+    puts "not an regular file"
+  end
+
+# get STDIN
+elsif opts.i? && !opts[:i].nil?
+  @input = opts[:i]
+elsif opts.p? || opts.t?
+  puts "you must specify some input"
+end
+
+# tagging stage
+if opts.t?
+  p @input.length
+  
+  if opts.l? && !opts[:lang].nil?
+    lang = opts[:lang]
+    Rstt.set_input lang: lang, content: @input
+  else
+    Rstt.set_input content: @input
+  end
+  
+  if opts.p?
+    Rstt.preprocessing
+  end
+  
+  Rstt.tagging
+  puts "input .."
+  puts Rstt.content
+  puts "\n"
+  Rstt.tagged.each do |tag|
+    p tag
+  end
+end

data/lib/rstt.rb

@@ -0,0 +1,109 @@
+# coding: utf-8
+require "celluloid"
+require "active_support/all"
+
+# own dependencies
+require "rstt/version"
+require "rstt/preprocess"
+require "rstt/tt_settings"
+
+module Rstt
+  # added celluloid for for concurrency
+  include Celluloid
+  mattr_accessor :lang, :content, :origin, :tagged, :sentences, :tags
+  
+  def self.set_input(input = {lang: "", content: ""})
+    if input[:lang]
+      @@lang = input[:lang]
+    else
+      @@lang = "en"
+    end
+    @@content = input[:content]
+  end
+  
+  # tagging stage related methods
+  def self.tagging
+    bar = `echo #{self.content} | #{TT_HOME}/cmd/#{build_tagging_command}`
+    # @@tagged = bar.split("\n").collect{|word| word.split("\t") }
+    @@tagged = bar.split("\n").collect do |word|
+      metrik = word.split("\t")
+      # use singular attribute names
+      {word: metrik[0], tag: metrik[1], stem: metrik[2]}
+    end
+  end
+  
+  def self.get_sentences
+    @@sentences = Preprocess.split_sentences(self.content)
+  end
+  
+  def self.build_tagging_command
+    lang = get_command_language
+    if LANGUAGES[lang][:utf8]
+      cmd = "tree-tagger-#{lang}-utf8"
+    else
+      cmd = "tree-tagger-#{lang}"
+    end
+    
+    cmd
+  end
+  
+  def self.get_command_language
+    lang = language_codes[self.lang.to_sym]
+
+    if lang.nil?
+      raise "language not supported"
+    elsif LANGUAGES[lang].nil?
+      raise "language supported, but not installed"
+    end
+    
+    lang
+  end
+  
+  def self.preprocessing
+    @@origin = @@content
+    # its important, that first html tags would be stripped and then non word characters
+    Preprocess.strip_html_tags(self.content)
+    Preprocess.strip_punctation_and_non_word_caracters(self.content)
+  end
+  
+  def self.language_codes
+    { bg: "bulgarian",
+      nl: "dutch",
+      en: "english",
+      et: "estonian",
+      fr: "french",
+      de: "german",
+      el: "greek",
+      it: "italian",
+      la: "latin",
+      ru: "russian", 
+      es: "spanish",
+      sw: "swahili"
+    }
+  end
+  
+  def self.installed_language_codes
+    installed = {}
+    language_codes.each do |lang|
+      installed[lang.first] = lang.last unless LANGUAGES[lang.last].nil?
+    end
+    
+    installed
+  end
+  
+  # output and processing helpers
+  def self.print
+    p @@lang
+    p @@content
+  end
+  
+  # ToDo 2012-02-25: work with `method_missing?`: DRY
+  # methods are plural @@tagged keys
+  %w(words tags stems).each do |meth|
+    self.define_singleton_method(meth) do
+      foo = []
+      @@tagged.each{ |tag| foo << tag[meth.singularize.to_sym] }
+      foo
+    end
+  end
+end

data/lib/rstt/preprocess.rb

@@ -0,0 +1,25 @@
+# coding: utf-8
+module Rstt
+  module Preprocess
+    def self.strip_html_tags(content)
+      unless content.nil?
+        content.gsub!(/\s*<\/?[^>]*>\s*/," ")
+        content.gsub!(/&\w+;/i," ")
+      end
+    end
+    
+    def self.strip_punctation_and_non_word_caracters(content)
+      unless content.nil?
+        content.gsub!(/\b[\#\%\^\$\@\(\)✗✓=\/"']{1,2}\b/i,'')
+        content.gsub!(/\s*[\#\%\^\$\@\(\)✗✓=\/"']{1,2}\s*/i,' ')
+        content.gsub!(/(–|--)/,'-')
+        content.gsub!(/\s+/,' ')
+        content.strip!
+        # content.gsub!(/(–|--)/,'-')
+        # content.gsub!(/\s+/,' ')
+      end
+    end
+  end # module Preprocess
+end # module Rstt
+
+

data/lib/rstt/version.rb

@@ -0,0 +1,3 @@
+module Rstt
+  VERSION = "0.9.0"
+end

data/lib/tasks/config.thor

@@ -0,0 +1,138 @@
+class Config < Thor
+  include Thor::Actions
+
+  desc "init PATH", "add TreeTagger path to module (--force replace old one)"
+  method_options :force => :boolean
+  def init(path)
+    if options.force?
+      remove
+    end
+    case
+    # 1. check if path set
+    when path
+      # a) check, if it is an accessible dir
+      if ::File.exists?(path)
+        insert(path)
+        set_languages
+      else
+        puts "not an regular directory"
+      end
+    # 2. is env set?
+    when ENV['TREETAGGERHOME']
+      path = ENV['TREETAGGERHOME']
+      insert(path)
+      set_languages
+    else
+      puts "specify path as argument:\n"
+      puts "  thor config:init /path/to/your/TreeTagger/installation/"
+    end
+
+  end
+  
+  desc "set_languages", "set installed languages"
+  def set_languages
+    puts "setting languages .."
+    languages = {}
+    
+    lib_path = File.join(get_path, 'lib')
+    Dir.entries(lib_path).each do |file|
+      foos = file.split("-")
+      languages[foos.first] = {utf8: foos.last.include?("utf")} unless foos.first.include?(".")
+    end
+
+    insert_into_file "lib/rstt/tt_settings.rb", after: /^  TT_HOME = (.+)\n/ do
+      "  LANGUAGES = #{languages}\n"
+    end
+
+  end
+
+  desc "get_languages", "see possible languages"
+  def get_languages
+    languages = []
+    get_files_of_dir do |file|
+      languages << file.split("-")[2] if file.include?("tree-tagger")
+    end
+    
+    languages.uniq
+  end
+  
+  # desc "get_workflows", "get installed workflows"
+  # def get_workflows
+  #   puts "get workflow commands from command files"
+  #   tree_tagger_scripts = []
+  #   get_files_of_dir do |file|
+  #     tree_tagger_scripts << file
+  #   end
+  #   
+  #   tree_tagger_scripts.each do |file|
+  #     p "_____________"
+  #     p file
+  #     File.open(file).readlines.each do |line|
+  #       p line
+  #     end
+  #     p "\n"
+  #   end
+  # end
+  
+  private
+  def insert(path)
+    puts "insert path .."
+    unless File.exists?("lib/rstt/tt_settings.rb")
+      create_file "lib/rstt/tt_settings.rb" do
+"module Rstt
+  TT_HOME = \"#{path}\"
+end"
+      end
+    else
+      insert_into_file "lib/rstt/tt_settings.rb", after: "module Rstt\n" do
+        "  TT_HOME = \"#{path}\"\n"
+      end
+    end
+    
+    insert_require
+  end
+  
+  def remove
+    puts "removing old .."
+    gsub_file "lib/rstt/tt_settings.rb", /^  TT_HOME = (.+)\n/ do |match|
+      match = ""
+    end
+    
+    remove_require
+  end
+
+  def get_path
+    path = ""
+    gsub_file "lib/rstt/tt_settings.rb", /^  TT_HOME = (.+)\n/ do |match|
+      path = match
+    end
+    path.chomp.split(" = ").last.gsub("\"",'')
+  end
+  
+  def get_files_of_dir(dir = "cmd", pattern = "tree-tagger*")
+    cmd_path = File.join(get_path, dir, pattern)
+    Dir.glob(cmd_path).each do |file|
+      yield file
+    end
+    # Dir.glob(tt_files).each do |file|
+    #   p file
+    #   yield file
+    # end
+    
+    # Dir.entries(cmd_path).each do |file|
+    #   yield file
+    # end
+  end
+  
+  def insert_require
+    insert_into_file "lib/rstt.rb", after: "require \"rstt/preprocess\"\n" do
+      "require \"rstt/tt_settings\"\n"
+    end
+  end
+  
+  def remove_require
+    gsub_file "lib/rstt.rb", "require \"rstt/tt_settings\"" do |match|
+      match = ""
+    end
+  end
+end

data/rstt.gemspec

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "rstt/version"
+
+Gem::Specification.new do |s|
+  s.name        = "rstt"
+  s.version     = Rstt::VERSION
+  s.authors     = ["LeFnord"]
+  s.email       = ["pscholz.le@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{another ruby wrapper for Stuttgarter Tree Tagger}
+  s.description = %q{another ruby wrapper for Stuttgarter Tree Tagger}
+
+  s.rubyforge_project = "rstt"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib","bin"]
+
+  # specify any dependencies here; for example:
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "yard"
+  s.add_development_dependency "syntax"
+  s.add_runtime_dependency "thor"
+  s.add_runtime_dependency "activesupport"
+  s.add_runtime_dependency "celluloid"
+  s.add_runtime_dependency "slop"
+end

data/spec/rstt_preprocessing_spec.rb

@@ -0,0 +1,57 @@
+# coding: utf-8
+require "spec_helper"
+
+describe Rstt do
+  describe "Preprocessing" do
+    before(:each) do
+      @control_1 = "Am 13.04.1899 ist es geschehen - was steht woanders. Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Nam cursus. Morbi ut mi. Nullam enim leo, egestas id, condimentum at, laoreet mattis, massa."
+      @string    = "Am 13.04.1899 ist es geschehen – was steht woanders. Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Nam cursus. Morbi ut mi. Nullam enim leo, egestas id, condimentum at, laoreet mattis, massa."
+      @control_2 = "Am 13.04.1899 ist es geschehen - was steht woanders. Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Nam cursus. Morbi ut mi. Null-am e-nim leo, egestas id, condimentum at, laoreet mattis, massa."
+      @extra     = "Am 13.04.1899 ist es geschehen – was steht woanders. Lo#rem ipsum d#olor sit am%et, cons^ect$etuer adipiscing e@lit. Nam cursus. Morbi ut mi. Null-am e--nim leo, egestas id, condimentum at, laoreet mattis, massa."
+      @control_3 = "Am 13.04.1899 ist es geschehen - was steht woanders. Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Nam cursus. Morbi ut mi. Null-am e-nim leo, egestas id, condimentum at, laoreet mattis, massa."
+      @getaggt   = "Am 13.04.1899 ist es geschehen – was steht woanders. Lo#rem ipsum d#olor sit am%et, cons^ect$etuer adipiscing e@lit. Nam cursus. Morbi ut mi. Null-am e--nim leo, egestas id, condimentum at, laoreet mattis, massa."
+      
+      @real      = "Title editierbar machen ✗ mmh, ist wohl besser, wenn es so bleibt, beim ersten anlegen wird ein Titel gebastelt und gut ist, so bleibt auch der slug konsistent deployen auf Arielle, evtl. über einen BitBucket Account (als Repository) ✓"
+      @control_r = "Title editierbar machen mmh, ist wohl besser, wenn es so bleibt, beim ersten anlegen wird ein Titel gebastelt und gut ist, so bleibt auch der slug konsistent deployen auf Arielle, evtl. über einen BitBucket Account als Repository"
+      
+      @real_2    = "<h3>ToDos: Think</h3><div><ul><li>Title editierbar machen&nbsp;✗</li><ul><li>mmh, ist wohl besser, wenn es so bleibt, beim ersten anlegen wird ein Titel gebastelt und gut ist, so bleibt auch der slug konsistent</li></ul><li>deployen auf Arielle, evtl. über einen BitBucket Account (als Repository)&nbsp;✓</li><li>Abbildungen und andere Objekte als Attachement ermöglichen</li><li>Tabellen erstellen (evtl. Berechnungen?)</li><ul><li>dafür wird wohl ein anderer Editor nötig sein, folgende kämen dafür in Frage:</li><ul><li><a href='http://jejacks0n.github.com/mercury/' title='MercuryEdit' target='_blank'>MercuryEdit</a><br></li><li><a href='http://www.aloha-editor.org/' title='AlohaEditor' target='_blank'>AlohaEditor</a><br></li><li><a href='http://ckeditor.com/' title='CKEditor' target='_blank'>CKEditor</a><br></li></ul><li>wahrscheinlich kommt der&nbsp;<a href='http://www.aloha-editor.org/' title='AlohaEditor' target='_blank' style='margin-top: 0px; margin-right: 0px; margin-bottom: 0px; margin-left: 0px; padding-top: 2px; padding-right: 3px; padding-bottom: 2px; padding-left: 3px; border-top-width: 0px; border-right-width: 0px; border-bottom-width: 0px; border-left-width: 0px; border-style: initial; border-color: initial; font-size: 13px; font: inherit; vertical-align: baseline; border-style: initial; border-color: initial; border-style: initial; border-color: initial; border-style: initial; border-color: initial; border-style: initial; border-color: initial; border-style: initial; border-color: initial; color: rgb(153, 153, 153); text-decoration: none; '>AlohaEditor</a>&nbsp;in Frage ( insbesondere:&nbsp;<a href='http://www.aloha-editor.org/demos/aloha-world-example/' title='AlohaEditor|World' target='_blank'>AlohaEditor|World</a>)</li></ul><li>Referenzierung unter den Notes ermöglichen</li></ul></div>"
+    end
+    
+    it "should replace a string by itself" do
+      Rstt.set_input lang: "en", content: @string
+      Rstt.preprocessing
+      Rstt.content.should == @control_1
+    end
+
+    it "should replace non alphanum caharacters within words with nothings" do
+      Rstt.set_input lang: "en", content: @extra
+      Rstt.preprocessing
+      Rstt.content.should == @control_2
+    end
+    
+    it "should delete non letter characters" do
+      Rstt.set_input lang: "de", content: @real
+      Rstt.preprocessing
+      Rstt.content.should == @control_r
+    end
+    
+    it "should clean up input from html and non word characters" do
+      Rstt.set_input lang: "de", content: @real_2
+      Rstt.preprocessing
+    end
+    
+    describe "preprocessing:html" do
+      before(:each) do
+        path = File.join(File.dirname(__FILE__), '..', 'tmp','tmp.html')
+        @html_file = File.open(path)
+      end
+
+      it "should load the given html file" do
+        file = @html_file.read
+        Rstt.set_input lang: "en", content: file
+        Rstt.preprocessing
+        Rstt.content.should_not match(/<\/?[^>]*>/)
+      end
+    end
+  end
+end

data/spec/rstt_spec.rb

@@ -0,0 +1,117 @@
+# coding: utf-8
+require "spec_helper"
+
+describe Rstt do
+  describe "module variables" do
+    describe "responds to .." do
+      it "should respond to :lang" do
+        Rstt.respond_to?(:lang).should be_true
+      end
+    
+      it "should respond to :content" do
+        Rstt.respond_to?(:content).should be_true
+      end
+      
+      it "should respond to :origin" do
+        Rstt.respond_to?(:origin).should be_true
+      end
+      
+      it "should respond to :tagged" do
+        Rstt.respond_to?(:tagged).should be_true
+      end    
+    end # responds to ..
+    
+    describe "getting" do
+      it "should print both" do
+        Rstt.set_input lang: "de", content: "Das ist ein einfacher Dummy Satz."
+        Rstt.print.should be_true
+      end
+    end
+    
+    describe "setting" do
+      before(:each) do
+        @input = {lang: "de", content: "Das ist ein einfacher Dummy Satz."}
+      end
+      
+      it "should set both by :set_input " do
+        Rstt.set_input @input
+        Rstt.lang.should == @input[:lang]
+        Rstt.content.should == @input[:content]
+      end
+    end # setting
+  end # module variables
+  
+  describe "tagging stages" do
+    before(:each) do
+      @input = {lang: "de", content: "Das ist ein einfacher Dummy Satz."}
+    end
+    
+    it "should pos tagging on given input data" do
+      Rstt.set_input @input
+      Rstt.tagging
+    end
+    
+    it "origin should be content after preprocessing" do
+      Rstt.set_input @input
+      Rstt.preprocessing
+      Rstt.origin.should == @input[:content]
+    end
+    
+    describe "language finding and command building" do
+      it "should find the right language dependent on input" do
+        Rstt.set_input lang: "en", content: ""
+        Rstt.get_command_language.should == ("english")
+      end
+    
+      it "should raise an exception if language not supported" do
+        Rstt.set_input lang: "xy", content: ""
+        expect {Rstt.get_command_language}.to raise_error
+      end
+    
+      it "should raise an exception if language not installed" do
+        Rstt.set_input lang: "ru", content: ""
+        expect {Rstt.get_command_language}.to raise_error
+      
+      end
+    
+      it "should build the correct tagging command, dependend on input language" do
+        Rstt.set_input lang: "en", content: ""
+        Rstt.build_tagging_command.should == ("tree-tagger-english")
+      end
+    
+      it "should prefer utf8 over other" do
+        Rstt.set_input lang: "de", content: ""
+        Rstt.build_tagging_command.should == ("tree-tagger-german-utf8")
+      end
+    end # language finding and command building
+    
+    describe "output format" do
+      before(:each) do
+        @input = {lang: "de", content: "Das ist ein einfacher Dummy Satz"}
+      end
+      it "should be an array as output" do
+        Rstt.set_input @input
+        Rstt.preprocessing
+        Rstt.tagging.should be_a Array
+      end
+      it "should have correct count" do
+        Rstt.set_input @input
+        verfifier = @input[:content].split(" ").length
+        Rstt.preprocessing
+        Rstt.tagging
+        Rstt.tagged.should have(verfifier).things
+      end
+      
+      # testing output format, means: for each word in given input gives an array
+      # with: 1. input word, 2. tagging category, 3. lemma
+      # could be done over output length
+      it "should have correct output format" do
+        Rstt.set_input @input
+        Rstt.preprocessing
+        Rstt::LANGUAGES
+        Rstt.tagging.first.should have(3).things
+      end
+    end # output format
+    
+  end # tagging stages
+end

data/spec/spec_helper.rb

@@ -0,0 +1,17 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper.rb"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+
+require "rstt"
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+end

data/tmp/tmp.html

@@ -0,0 +1,146 @@
+<h2>und hier kommt dummy text</h2>
+<p>
+  <mark data-related='hamburg' class='glossary'>Hamburg</mark>
+  Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Nam cursus. Morbi ut mi. Nullam enim leo, egestas id, condimentum at, laoreet mattis, massa.
+  Sed eleifend nonummy diam. Praesent mauris ante, elementum et, bibendum at, posuere sit amet, nibh. Duis tincidunt lectus quis dui viverra vestibulum.
+  Suspendisse vulputate aliquam dui. Nulla elementum dui ut augue. Aliquam vehicula mi at mauris. Maecenas placerat, nisl at consequat rhoncus, sem nunc
+  gravida justo, quis eleifend arcu velit quis lacus. Morbi magna magna, tincidunt a, mattis non, imperdiet vitae, tellus. Sed odio est, auctor ac,
+  sollicitudin in, consequat vitae, orci. Fusce id felis. Vivamus sollicitudin metus eget eros.Pellentesque habitant morbi tristique senectus et netus et
+  malesuada fames ac turpis egestas. In posuere felis nec tortor. Pellentesque faucibus. Ut accumsan ultricies elit. Maecenas at justo id velit placerat
+  molestie. Donec dictum lectus non odio. Cras a ante vitae enim iaculis aliquam. Mauris nunc quam, venenatis nec, euismod sit amet, egestas placerat, est.
+  Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Cras id elit. Integer quis urna. Ut ante enim, dapibus malesuada,
+  fringilla eu, condimentum quis, tellus. Aenean porsttitor eros vel dolor. Donec convallis pede venenatis nibh. Duis quam. Nam eget lacus. Aliquam erat volutpat.
+  Quisque dignissim congue leo.
+</p>
+<figure>
+  <figcaption>
+    <span class='caption'>Abb. 1:</span>
+    <span class='content'>Und hier kommt die Caption zum Bild rein.</span>
+  </figcaption>
+</figure>
+<p>
+  <mark data-related='hamburg' class='glossary'>Hamburg</mark>
+  Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Nam cursus. Morbi ut mi. Nullam enim leo, egestas id, condimentum at, laoreet mattis, massa.
+  Sed eleifend nonummy diam. Praesent mauris ante, elementum et, bibendum at, posuere sit amet, nibh. Duis tincidunt lectus quis dui viverra vestibulum.
+  Suspendisse vulputate aliquam dui. Nulla elementum dui ut augue. Aliquam vehicula mi at mauris. Maecenas placerat, nisl at consequat rhoncus, sem nunc
+  gravida justo, quis eleifend arcu velit quis lacus. Morbi magna magna, tincidunt a, mattis non, imperdiet vitae, tellus. Sed odio est, auctor ac,
+  sollicitudin in, consequat vitae, orci. Fusce id felis. Vivamus sollicitudin metus eget eros.Pellentesque habitant morbi tristique senectus et netus et
+  malesuada fames ac turpis egestas. In posuere felis nec tortor. Pellentesque faucibus. Ut accumsan ultricies elit. Maecenas at justo id velit placerat
+  molestie. Donec dictum lectus non odio. Cras a ante vitae enim iaculis aliquam. Mauris nunc quam, venenatis nec, euismod sit amet, egestas placerat, est.
+  Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Cras id elit. Integer quis urna. Ut ante enim, dapibus malesuada,
+  fringilla eu, condimentum quis, tellus. Aenean porsttitor eros vel dolor. Donec convallis pede venenatis nibh. Duis quam. Nam eget lacus. Aliquam erat volutpat.
+  Quisque dignissim congue leo.
+</p>
+<h2>und hier kommt dummy text</h2>
+<p>
+  <mark data-related='hamburg' class='glossary'>Hamburg</mark>
+  Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Nam cursus. Morbi ut mi. Nullam enim leo, egestas id, condimentum at, laoreet mattis, massa.
+  Sed eleifend nonummy diam. Praesent mauris ante, elementum et, bibendum at, posuere sit amet, nibh. Duis tincidunt lectus quis dui viverra vestibulum.
+  Suspendisse vulputate aliquam dui. Nulla elementum dui ut augue. Aliquam vehicula mi at mauris. Maecenas placerat, nisl at consequat rhoncus, sem nunc
+  gravida justo, quis eleifend arcu velit quis lacus. Morbi magna magna, tincidunt a, mattis non, imperdiet vitae, tellus. Sed odio est, auctor ac,
+  sollicitudin in, consequat vitae, orci. Fusce id felis. Vivamus sollicitudin metus eget eros.
+</p>
+<p>
+  <mark data-related='hamburg' class='glossary'>Hamburg</mark>
+  Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Nam cursus. Morbi ut mi. Nullam enim leo, egestas id, condimentum at, laoreet mattis, massa.
+  Sed eleifend nonummy diam. Praesent mauris ante, elementum et, bibendum at, posuere sit amet, nibh. Duis tincidunt lectus quis dui viverra vestibulum.
+  Suspendisse vulputate aliquam dui. Nulla elementum dui ut augue. Aliquam vehicula mi at mauris. Maecenas placerat, nisl at consequat rhoncus, sem nunc
+  gravida justo, quis eleifend arcu velit quis lacus. Morbi magna magna, tincidunt a, mattis non, imperdiet vitae, tellus. Sed odio est, auctor ac,
+  sollicitudin in, consequat vitae, orci. Fusce id felis. Vivamus sollicitudin metus eget eros.Pellentesque habitant morbi tristique senectus et netus et
+  malesuada fames ac turpis egestas. In posuere felis nec tortor. Pellentesque faucibus. Ut accumsan ultricies elit. Maecenas at justo id velit placerat
+  molestie. Donec dictum lectus non odio. Cras a ante vitae enim iaculis aliquam. Mauris nunc quam, venenatis nec, euismod sit amet, egestas placerat, est.
+  Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Cras id elit. Integer quis urna. Ut ante enim, dapibus malesuada,
+  fringilla eu, condimentum quis, tellus. Aenean porsttitor eros vel dolor. Donec convallis pede venenatis nibh. Duis quam. Nam eget lacus. Aliquam erat volutpat.
+  Quisque dignissim congue leo.
+</p>
+<figure>
+  <figcaption>
+    <span class='caption'>Tab. 1:</span>
+    <span class='content'>Einige arithmetische Grundrechnungen (das kleine Ein-mal-Eins)</span>
+  </figcaption>
+  <table>
+    <thead>
+      <tr>
+        <th>#</th>
+        <th>#+#</th>
+        <th>#*#-#</th>
+        <th>#*#</th>
+        <th>#</th>
+        <th>#+#</th>
+        <th>#*#-#</th>
+        <th>#*#</th>
+        <th>#</th>
+        <th>#+#</th>
+        <th>#*#-#</th>
+        <th>#*#</th>
+        <th>#</th>
+        <th>#+#</th>
+        <th>#*#-#</th>
+        <th>#*#</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+        <td>1</td>
+        <td>2</td>
+        <td>0</td>
+        <td>1</td>
+        <td>1</td>
+        <td>2</td>
+        <td>0</td>
+        <td>1</td>
+        <td>1</td>
+        <td>2</td>
+        <td>0</td>
+        <td>1</td>
+        <td>1</td>
+        <td>2</td>
+        <td>0</td>
+        <td>1</td>
+      </tr>
+      <tr>
+        <td>2</td>
+        <td>4</td>
+        <td>2</td>
+        <td>4</td>
+        <td>2</td>
+        <td>4</td>
+        <td>2</td>
+        <td>4</td>
+        <td>2</td>
+        <td>4</td>
+        <td>2</td>
+        <td>4</td>
+        <td>2</td>
+        <td>4</td>
+        <td>2</td>
+        <td>4</td>
+      </tr>
+      <tr>
+        <td>3</td>
+        <td>6</td>
+        <td>6</td>
+        <td>9</td>
+        <td>3</td>
+        <td>6</td>
+        <td>6</td>
+        <td>9</td>
+        <td>3</td>
+        <td>6</td>
+        <td>6</td>
+        <td>9</td>
+        <td>3</td>
+        <td>6</td>
+        <td>6</td>
+        <td>9</td>
+      </tr>
+    </tbody>
+  </table>
+</figure>
+<div class='related' id='hamburg'>
+  na und hier kommen dann irgendwelche Erklärungen zu Hamburg hinein
+</div>
+<ol>
+  <li>foo</li>
+  <li>bar</li>
+</ol>

metadata

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification
+name: rstt
+version: !ruby/object:Gem::Version
+  version: 0.9.0
+  prerelease: 
+platform: ruby
+authors:
+- LeFnord
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-03-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70269725598760 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70269725598760
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: &70269725613740 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70269725613740
+- !ruby/object:Gem::Dependency
+  name: syntax
+  requirement: &70269725612840 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70269725612840
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: &70269725612400 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70269725612400
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: &70269725611800 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70269725611800
+- !ruby/object:Gem::Dependency
+  name: celluloid
+  requirement: &70269725611320 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70269725611320
+- !ruby/object:Gem::Dependency
+  name: slop
+  requirement: &70269725610860 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70269725610860
+description: another ruby wrapper for Stuttgarter Tree Tagger
+email:
+- pscholz.le@gmail.com
+executables:
+- rstt
+extensions: []
+extra_rdoc_files: []
+files:
+- .DS_Store
+- .gitignore
+- .rspec
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/rstt
+- lib/rstt.rb
+- lib/rstt/preprocess.rb
+- lib/rstt/version.rb
+- lib/tasks/config.thor
+- rstt.gemspec
+- spec/rstt_preprocessing_spec.rb
+- spec/rstt_spec.rb
+- spec/spec_helper.rb
+- tmp/tmp.html
+- todos.task
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+- bin
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: rstt
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: another ruby wrapper for Stuttgarter Tree Tagger
+test_files:
+- spec/rstt_preprocessing_spec.rb
+- spec/rstt_spec.rb
+- spec/spec_helper.rb
+has_rdoc: