rbtagger 0.0.1

data/lib/brill/tagger.rb

@@ -0,0 +1,225 @@
+require 'rule_tagger'
+
+module Brill
+  class Tagger
+    def initialize( lexicon, lexical_rules, contextual_rules )
+      @tagger = ::Tagger::BrillTagger.new
+      Brill::Tagger.load_lexicon(@tagger,lexicon)
+      Brill::Tagger.load_lexical_rules(@tagger,lexical_rules)
+      Brill::Tagger.load_contextual_rules(@tagger,contextual_rules)
+    end
+
+    # Tag a body of text
+    # returns an array like [[token,tag],[token,tag]...[token,tag]] 
+    #
+    def tag( text )
+      tokens = Brill::Tagger.tokenize( text )
+      tags = Brill::Tagger.tag_start( tokens )
+
+      @tagger.apply_lexical_rules( tokens, tags, [], 0 )
+      @tagger.default_tag_finish( tokens, tags )
+ 
+      # Brill uses these fake "STAART" tags to delimit the start & end of sentence.
+      tokens << "STAART" 
+      tokens << "STAART" 
+      tokens.unshift "STAART"
+      tokens.unshift "STAART"
+      tags << "STAART" 
+      tags << "STAART" 
+      tags.unshift "STAART"
+      tags.unshift "STAART"
+
+      @tagger.apply_contextual_rules( tokens, tags, 1 )
+
+      tags.shift
+      tags.shift
+      tokens.shift
+      tokens.shift
+      tags.pop
+      tags.pop
+      tokens.pop
+      tokens.pop
+
+      pairs = []
+      tokens.each_with_index do|t,i|
+        pairs << [t,tags[i]]
+      end
+      pairs
+    end
+  private
+    def self.lines( file )
+      lines = []
+      File.open(file,'r') do|f|
+        lines = f.readlines
+      end
+      lines
+    end
+    # load LEXICON 
+    def self.load_lexicon(tagger,lexicon)
+      lines = Brill::Tagger.lines(lexicon)
+      i = 0
+      count = lines.size
+      while i < count
+        line = lines[i]
+        #puts "line: #{line.inspect}:#{i.inspect}"
+        parts = line.split(/\s/)
+        #puts "word: #{word.inspect}, tags: #{tags.inspect}"
+        word = parts.first
+        tags = parts[1..-1]
+        tagger.add_to_lexicon(word,tags.first)
+        #puts "#{word} => #{tags.inspect}"
+        tags.each do|tag|
+          tagger.add_to_lexicon_tags("#{word} #{tag}")
+        end
+        i += 1
+      end
+    end
+
+    # load LEXICALRULEFILE 
+    def self.load_lexical_rules(tagger,rules)
+      lines = self.lines(rules)
+      i = 0
+      count = lines.size
+=begin
+  # original perl
+    chomp;
+    my @line = split or next;
+    $self->_add_lexical_rule($_);
+
+    if ($line[1] eq 'goodright') {
+      $self->_add_goodright($line[0]);
+    } elsif ($line[2] eq 'fgoodright') {
+      $self->_add_goodright($line[1]);
+    } elsif ($line[1] eq 'goodleft') {
+      $self->_add_goodleft($line[0]);
+    } elsif ($line[2] eq 'fgoodleft') {
+      $self->_add_goodleft($line[1]);
+    }
+=end
+      while i < count
+        line = lines[i].chomp
+        cols = line.split(/\s/)
+        next unless line.size > 0
+        tagger.add_lexical_rule(line)
+        if cols[1] == 'goodright'
+          tagger.add_goodright(cols[0])
+        elsif cols[2] == 'fgoodright'
+          tagger.add_goodright(cols[1])
+        elsif cols[1] == 'goodleft'
+          tagger.add_goodleft(cols[0])
+        elsif cols[2] == 'fgoodleft'
+          tagger.add_goodleft(cols[1])
+        end
+
+        i += 1
+      end
+    end
+
+    # load CONTEXTUALRULEFILE
+    def self.load_contextual_rules(tagger,rules)
+      lines = self.lines(rules)
+      i = 0
+      count = lines.size
+      while i < count
+        line = lines[i].chomp
+        next unless line.size > 0
+        tagger.add_contextual_rule(line);
+        i += 1
+      end
+    end
+
+    def self.tag_start(tokens)
+      tokens.map{|token| token.match(/^[A-Z]/) ? 'NNP' : 'NN' }
+    end
+
+    # this tokenize code is a port from perl
+    def self.tokenize(text)
+      # Normalize all whitespace
+      text = text.gsub(/\s+/,' ')
+
+      # translate some common extended ascii characters to quotes
+      text.gsub!(/#{145.chr}/,'`')
+      text.gsub!(/#{146.chr}/,"'")
+      text.gsub!(/#{147.chr}/,"``")
+      text.gsub!(/#{148.chr}/,"''")
+
+      # Attempt to get correct directional quotes
+      # s{\"\b} { `` }g;
+      text.gsub!(/\"\b/,' `` ')
+      # s{\b\"} { '' }g;
+      text.gsub!(/\b\"/," '' ")
+      #s{\"(?=\s)} { '' }g;
+      text.gsub!(/\"(?=\s)/," '' ")
+      #s{\"} { `` }g;
+      text.gsub!(/\"(?=\s)/," `` ")
+
+      # Isolate ellipses
+      # s{\.\.\.}   { ... }g;
+      text.gsub!(/\.\.\./,' ... ')
+
+
+      # Isolate any embedded punctuation chars
+      #   s{([,;:\@\#\$\%&])} { $1 }g;
+      text.gsub!(/([,;:\@\#\$\%&])/, ' \1 ')
+  
+      # Assume sentence tokenization has been done first, so split FINAL
+      # periods only.
+      # s/ ([^.]) \.  ([\]\)\}\>\"\']*) [ \t]* $ /$1 .$2 /gx;
+      text.gsub!(/ ([^.]) \.  ([\]\)\}\>\"\']*) [ \t]* $ /x, '\1 .\2 ')
+
+      # however, we may as well split ALL question marks and exclamation points,
+      # since they shouldn't have the abbrev.-marker ambiguity problem
+      #s{([?!])} { $1 }g;
+      text.gsub!(/([?!])/, ' \1 ')
+
+      # parentheses, brackets, etc.
+      #s{([\]\[\(\)\{\}\<\>])} { $1 }g;
+      text.gsub!(/([\]\[\(\)\{\}\<\>])/,' \1 ')
+
+      #s/(-{2,})/ $1 /g;
+      text.gsub!(/(-{2,})/,' \1 ')
+
+      # Add a space to the beginning and end of each line, to reduce
+      # necessary number of regexps below.
+      #s/$/ /;
+      text.gsub!(/$/," ")
+      #s/^/ /;
+      text.gsub!(/^/," ")
+
+      # possessive or close-single-quote
+      #s/\([^\']\)\' /$1 \' /g;
+      text.gsub!(/\([^\']\)\' /,%q(\1 ' ))
+
+      # as in it's, I'm, we'd
+      #s/\'([smd]) / \'$1 /ig;
+      text.gsub!(/\'([smd]) /i,%q( '\1 ))
+
+      #s/\'(ll|re|ve) / \'$1 /ig;
+      text.gsub!(/\'(ll|re|ve) /i,%q( '\1 ))
+      #s/n\'t / n\'t /ig;
+      text.gsub!(/n\'t /i,"  n't ")
+
+      #s/ (can)(not) / $1 $2 /ig;
+      text.gsub!(/ (can)(not) /i,' \1 \2 ')
+      #s/ (d\')(ye) / $1 $2 /ig;
+      text.gsub!(/ (d\')(ye) /i,' \1 \2 ')
+      #s/ (gim)(me) / $1 $2 /ig;
+      text.gsub!(/ (gim)(me) /i,' \1 \2 ')
+      #s/ (gon)(na) / $1 $2 /ig;
+      text.gsub!(/ (gon)(na) /i,' \1 \2 ')
+      #s/ (got)(ta) / $1 $2 /ig;
+      text.gsub!(/ (got)(ta) /i,' \1 \2 ')
+      #s/ (lem)(me) / $1 $2 /ig;
+      text.gsub!(/ (lem)(me) /i,' \1 \2 ')
+      #s/ (more)(\'n) / $1 $2 /ig;
+      text.gsub!(/ (more)(\'n) /i,' \1 \2 ')
+      #s/ (\'t)(is|was) / $1 $2 /ig;
+      text.gsub!(/ (\'t)(is|was) /i,' \1 \2 ')
+      #s/ (wan)(na) / $1 $2 /ig;
+      text.gsub!(/ (wan)(na) /i,' \1 \2 ')
+ 
+      text.split(/\s/)
+    end
+
+  end
+end

data/lib/rbtagger/version.rb

@@ -0,0 +1,9 @@
+module RbTagger #:nodoc:
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 0
+    TINY  = 1
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end

data/lib/rbtagger.rb

@@ -0,0 +1,6 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+module RbTagger
+  
+end

data/script/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# File: script/console
+irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+libs =  " -r irb/completion"
+# Perhaps use a console_lib to store any extra methods I may want available in the cosole
+# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
+libs <<  " -r #{File.dirname(__FILE__) + '/../lib/rbtagger.rb'}"
+puts "Loading rb-brill-tagger gem"
+exec "#{irb} #{libs} --simple-prompt"

data/script/destroy

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)

data/script/txt2html

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+
+GEM_NAME = 'rbtagger' # what ppl will type to install your gem
+RUBYFORGE_PROJECT = 'ruletagger'
+
+require 'rubygems'
+begin
+  require 'newgem'
+  require 'rubyforge'
+rescue LoadError
+  puts "\n\nGenerating the website requires the newgem RubyGem"
+  puts "Install: gem install newgem\n\n"
+  exit(1)
+end
+require 'redcloth'
+require 'syntax/convertors/html'
+require 'erb'
+require File.dirname(__FILE__) + "/../lib/#{GEM_NAME}/version.rb"
+
+version  = RbTagger::VERSION::STRING
+download = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
+
+def rubyforge_project_id
+  RubyForge.new.autoconfig["group_ids"][RUBYFORGE_PROJECT]
+end
+
+class Fixnum
+  def ordinal
+    # teens
+    return 'th' if (10..19).include?(self % 100)
+    # others
+    case self % 10
+    when 1: return 'st'
+    when 2: return 'nd'
+    when 3: return 'rd'
+    else    return 'th'
+    end
+  end
+end
+
+class Time
+  def pretty
+    return "#{mday}#{mday.ordinal} #{strftime('%B')} #{year}"
+  end
+end
+
+def convert_syntax(syntax, source)
+  return Syntax::Convertors::HTML.for_syntax(syntax).convert(source).gsub(%r!^<pre>|</pre>$!,'')
+end
+
+if ARGV.length >= 1
+  src, template = ARGV
+  template ||= File.join(File.dirname(__FILE__), '/../website/template.html.erb')
+else
+  puts("Usage: #{File.split($0).last} source.txt [template.html.erb] > output.html")
+  exit!
+end
+
+template = ERB.new(File.open(template).read)
+
+title = nil
+body = nil
+File.open(src) do |fsrc|
+  title_text = fsrc.readline
+  body_text_template = fsrc.read
+  body_text = ERB.new(body_text_template).result(binding)
+  syntax_items = []
+  body_text.gsub!(%r!<(pre|code)[^>]*?syntax=['"]([^'"]+)[^>]*>(.*?)</\1>!m){
+    ident = syntax_items.length
+    element, syntax, source = $1, $2, $3
+    syntax_items << "<#{element} class='syntax'>#{convert_syntax(syntax, source)}</#{element}>"
+    "syntax-temp-#{ident}"
+  }
+  title = RedCloth.new(title_text).to_html.gsub(%r!<.*?>!,'').strip
+  body = RedCloth.new(body_text).to_html
+  body.gsub!(%r!(?:<pre><code>)?syntax-temp-(\d+)(?:</code></pre>)?!){ syntax_items[$1.to_i] }
+end
+stat = File.stat(src)
+created = stat.ctime
+modified = stat.mtime
+
+$stdout << template.result(binding)