treat 0.1.1 → 0.1.2

data/lib/treat/inflectors/lemmatizers/e_lemma/elemma.c

@@ -1,68 +0,0 @@
-#include "wn.h"
-#include "wnconsts.h"
-#include "ruby.h"
-
-/*
-
-Copyright (C) 2004 UTIYAMA Masao <mutiyama@crl.go.jp>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABITreatY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-*/
-
-static VALUE
-parse(VALUE klass, VALUE rb_word, VALUE rb_pos)
-{
-  char *word;
-  char *POS = STR2CSTR(rb_pos);
-  char *lemma;
-  int pos;
-  VALUE retval = rb_ary_new();
-  
-  word = malloc(strlen(STR2CSTR(rb_word))+1);
-  if(!word){rb_raise(rb_eStandardError, "malloc failed.\n");}
-  strcpy(word, STR2CSTR(rb_word));
-  
-  if(strcmp(POS,"noun")==0){pos = NOUN;}
-  else if(strcmp(POS,"verb")==0){pos = VERB;}
-  else if(strcmp(POS,"adj")==0){pos = ADJ;}
-  else if(strcmp(POS,"adv")==0){pos = ADV;}
-  else{
-    rb_raise(rb_eStandardError, "%s should be (noun|verb|adj|adv)\n", POS);
-  }
-  if(is_defined(word, pos)){
-    /*printf("* %s found as is.\n", word);*/
-    rb_ary_push(retval, rb_str_new2(word));
-  }
-  if((lemma=morphstr(word, pos))!=NULL){
-    do {
-      if(is_defined(lemma, pos)){
-	/*printf("* %s => %s found.\n", word, lemma);*/
-	rb_ary_push(retval, rb_str_new2(lemma));
-      }
-    } while((lemma=morphstr(NULL, pos))!=NULL);
-  }
-  free(word);
-  return retval;
-}
-
-void
-Init_elemma()
-{
-  VALUE mod = rb_define_module("ELemma");
-  rb_define_module_function(mod, "parse", parse, 2);
-  if(wninit()){
-    rb_raise(rb_eStandardError, "Cannot open WordNet database\n");
-  }
-}

data/lib/treat/inflectors/lemmatizers/e_lemma/extconf.rb

@@ -1,6 +0,0 @@
-require 'mkmf'
-
-$CFLAGS = "-Wall -I/usr/local/WordNet-2.1/include/"
-$LOCAL_LIBS = "-L/usr/local/WordNet-2.1/lib -lwn"
-
-create_makefile("elemma")

data/lib/treat/inflectors/lemmatizers/e_lemma.rb

@@ -1,12 +0,0 @@
-module Treat
-  module Inflectors
-    module Lemmatizers
-      class ELemma
-        silently { require 'treat/inflectors/lemmatizers/elemma/elemma'}
-        def self.lemma(entity, options = nil)
-          ::ELemma::parse(word, entity.tag)
-        end
-      end
-    end
-  end
-end

data/lib/treat/resources/categories.rb

@@ -1,18 +0,0 @@
-module Treat
-  module Resources
-    class Categories
-      List = [
-        :adjective, :adverb, :noun, :verb, :interjection,
-        :clitic, :coverb, :conjunction, :determiner, :particle,
-        :preposition, :pronoun, :number, :symbol, :punctuation,     
-        :complementizer
-      ]
-      wttc = {}
-      Treat::Resources::Tags::AlignedWordTags.each_slice(2) do |desc, tags|
-        desc = desc.gsub(',', ' ,').split(' ')[0].downcase
-        tags.each { |tag| wttc[tag] = desc.intern }
-      end
-      WordTagToCategory = wttc
-    end
-  end
-end

data/lib/treat/resources/delegates.rb

@@ -1,96 +0,0 @@
-module Treat
-  module Resources
-    module Delegates
-      class English
-        Extractors = {
-          time: [:chronic],
-          topics: [:reuters],
-          topic_words: [:lda],
-          key_sentences: [:topics_frequency]
-        }
-        Processors = {
-          chunkers: [:txt],
-          parsers: [:enju, :stanford],
-          segmenters: [:tactful, :punkt, :stanford],
-          tokenizers: [:multilingual, :macintyre, :perl, :punkt, :tactful, :stanford]
-        }
-        Lexicalizers = {
-          category: [:from_tag],
-          linkages: [:naive],
-          synsets: [:wordnet, :rita_wn],
-          tag: [:brill, :lingua, :stanford]
-        }
-        Inflectors = {
-          conjugators: [:linguistics],
-          declensors: [:linguistics, :english],
-          lemmatizers: [:e_lemma],
-          stemmers: [:porter_c, :porter, :uea],
-          ordinal_words: [:linguistics],
-          cardinal_words: [:linguistics]
-        }
-      end
-      class German
-        Extractors = {}
-        Inflectors = {}
-        Lexicalizers = {
-          tag: [:stanford]
-        }
-        Processors = {
-          chunkers: [:txt],
-          parsers: [:stanford],
-          segmenters: [:tactful, :punkt, :stanford],
-          tokenizers: [:multilingual, :macintyre, :perl, :punkt, :tactful, :stanford]
-        }
-      end
-      class French
-        Extractors = {}
-        Inflectors = {}
-        Lexicalizers = {
-          tag: [:stanford]
-        }
-        Processors = {
-          chunkers: [:txt],
-          parsers: [:stanford],
-          segmenters: [:tactful, :punkt, :stanford],
-          tokenizers: [:multilingual, :macintyre, :perl, :punkt, :tactful, :stanford]
-        }
-      end
-      class Italian
-        Extractors = {}
-        Inflectors = {}
-        Lexicalizers = {}
-        Processors = {
-          chunkers: [:txt],
-          segmenters: [:tactful, :punkt, :stanford],
-          tokenizers: [:multilingual, :macintyre, :perl, :punkt, :tactful, :stanford]
-        }
-      end
-      class Arabic
-        Extractors = {}
-        Inflectors = {}
-        Lexicalizers = {
-          tag: [:stanford]
-        }
-        Processors = {
-          parsers: [:stanford]
-        }
-      end
-      class Chinese
-        Extractors = {}
-        Inflectors = {}
-        Lexicalizers = {
-          tag: [:stanford]
-        }
-        Processors = {}
-      end
-      class Xinhua
-        Extractors = {}
-        Inflectors = {}
-        Lexicalizers = {}
-        Processors = {
-          parsers: [:stanford]
-        }
-      end
-    end
-  end
-end

data/lib/treat/resources/edges.rb

@@ -1,8 +0,0 @@
-# http://nlp.stanford.edu/software/dependencies_manual.pdf
-
-=begin
-ENJU
-
-pred: noun_arg0, noun_arg1, noun_arg2, noun_arg12, it_arg1, there_arg0, quote_arg2, quote_arg12, quote_arg23, quote_arg123, poss_arg2, poss_arg12, aux_arg12, aux_mod_arg12, verb_arg1, verb_arg12, verb_arg123, verb_arg1234, verb_mod_arg1, verb_mod_arg12, verb_mod_arg123, verb_mod_arg1234, adj_arg1, adj_arg12, adj_mod_arg1, adj_mod_arg12, conj_arg1, conj_arg12, conj_arg123, coord_arg12, det_arg1, prep_arg12, prep_arg123, prep_mod_arg12, prep_mod_arg123, lgs_arg2, dtv_arg2, punct_arg1, app_arg12, lparen_arg123, rparen_arg0, comp_arg1, comp_arg12, comp_mod_arg1, relative_arg1, relative_arg12
-
-=end

data/lib/treat/resources/formats.rb

@@ -1,23 +0,0 @@
-module Treat
-  module Resources
-    module Format
-      
-      class XML
-        require 'nokogiri'
-        def self.validate(document_path, schema_path, root_element)
-          schema = Nokogiri::XML::Schema(File.read(schema_path))
-          document = Nokogiri::XML(File.read(document_path))
-          schema.validate(document.xpath("//#{root_element}").to_s)
-        end
-        validate('input.xml', 'schema.xdf', 'container').each do |error|
-          puts error.message
-        end
-      end
-      
-      class HTML < XML
-        
-      end
-      
-    end
-  end
-end

data/lib/treat/resources/languages.rb

@@ -1,86 +0,0 @@
-module Treat
-  module Resources
-    # Dictionnary of ISO-639-1, ISO-639-2 language codes,
-    # as well as their full text description in both
-    # English and French.
-    module Languages
-      ISO639_1 = 1
-      ISO639_2 = 2
-      # Describe a language code (ISO-639-1 or ISO-639-2)
-      # or its full text description in full French or English.
-      def self.describe(lang, desc_lang = :en)
-        raise "Must provide a non-nil language identifier to describe." if lang.nil?
-        lang = find(lang).to_s
-        if [:en, :eng, :english, :anglais].include?(desc_lang)
-          l = @@english_full.key(lang)
-        elsif [:fr, :fra, :french, :french].include?(desc_lang)
-          l = @@french_full.key(lang)
-        else
-          raise Treat::Exception,
-          "Unknown language to describe: #{desc_lang}."
-        end
-        not_found(lang) if l.nil?
-        l.intern
-      end
-      # Raise an error message when a language code
-      # or description is not found and suggest 
-      # possible misspellings.
-      def self.not_found(lang)
-        msg = "Language '#{lang}' does not exist."
-        all = @@iso639_2.keys + @@iso639_1.keys +
-              @@english_full.keys + @@french_full.keys
-        msg += did_you_mean?(all, lang)
-        raise Treat::Exception, msg
-      end
-      # Find a language by ISO-639-1 or ISO-639-2 code
-      # or full name (in English or French) and return
-      # the ISO-639-1 or ISO-639-2 language code as a 
-      # lowercase identifier.
-      def self.find(lang, rc = ISO639_2)
-        raise "Must provide a non-nil language identifier to describe." if lang.nil?
-        get_languages
-        lang = lang.to_s.downcase
-        if @@iso639_1.has_key?(lang)
-          return :"#{lang}" if rc == ISO639_1
-          return :"#{@@iso639_1[lang]}" if rc == ISO639_2
-        elsif @@iso639_2.has_key?(lang)
-          return :"#{lang}" if rc == ISO639_2
-          return :"#{@@iso639_2[lang]}" if rc == ISO639_1
-        elsif @@english_full.has_key?(lang)
-          return :"#{@@english_full[lang]}" if rc == ISO639_2
-          return :"#{@@iso639_2[@@english_full[lang]]}" if rc == ISO639_1
-        elsif @@french_full.has_key?(lang)
-          return :"#{@@french_full[lang]}" if rc == ISO639_2
-          return :"#{@@iso639_1[@@french_full[lang]]}" if rc == ISO639_2
-        else
-          not_found(lang)
-        end
-      end
-      @@loaded = false
-      # Get the languages from the dictionary.
-      def self.get_languages
-        return if @@loaded
-        @@iso639_1 = {}; @@iso639_2 = {};
-        @@english_full = {}; @@french_full = {}
-        languages = IO.readlines(Treat.lib + '/treat/resources/languages.txt')
-        languages.each do |language|
-          iso639_2, iso639_1, english_desc, french_desc =
-          language.split(',')
-          @@iso639_1[iso639_1] = iso639_2
-          @@iso639_2[iso639_2] = iso639_1
-          unless english_desc.nil?
-            english_desc.strip.downcase.split('|').each do |l|
-              @@english_full[l.downcase.strip] = iso639_2
-            end
-          end
-          unless french_desc.nil?
-            french_desc.strip.downcase.split('|').each do |l|
-              @@french_full[l.downcase.strip] = iso639_2
-            end
-          end
-        end
-        @@loaded = true
-      end
-    end
-  end
-end

data/lib/treat/resources.rb

@@ -1,10 +0,0 @@
-module Treat
-  module Resources
-    require 'treat/resources/delegates'
-    require 'treat/resources/dependencies'
-    require 'treat/resources/edges'
-    require 'treat/resources/languages'
-    require 'treat/resources/tags'
-    require 'treat/resources/categories'
-  end
-end

data/lib/treat/utilities.rb

@@ -1,127 +0,0 @@
-module Treat
-  # Provides utility functions used across the library.
-  module Utilities
-    # Require file utilities.
-    require 'fileutils'
-    # Returns the platform we are running on.
-    def self.platform
-      RUBY_PLATFORM.split("-")[1]
-    end
-    # Runs a block of code silently, i.e. without
-    # expressing warnings even in verbose mode.
-    # Rename to silence_streamsings.
-    def self.silently(&block)
-      warn_level = $VERBOSE
-      $VERBOSE = nil
-      result = block.call
-      $VERBOSE = warn_level
-      result
-    end
-    def self.silence_streams(*streams)
-      yield
-    end
-    # Create a temporary file which is deleted
-    # after execution of the block.
-    require 'tempfile'
-    def self.create_temp_file(ext, value = nil, &block)
-      tmp = Tempfile.new(['', ".#{ext.to_s}"], Treat.tmp)
-      tmp.puts(value) if value
-      block.call(tmp.path)
-    end
-    # A list of acronyms used in class names within
-    # the program. These do not CamelCase; they
-    # CAMELCASE.
-    @@acronyms = ['XML', 'HTML', 'YAML', 'UEA', 'LDA', 'PDF', 'GOCR', 'Treat'].join('|')
-    @@cc_cache = {}
-    # Convert un_camel_case to CamelCase.
-    def self.camel_case(o_phrase)
-      phrase = o_phrase.to_s.dup
-      return @@cc_cache[o_phrase] if @@cc_cache[o_phrase]
-      phrase.gsub!(/#{@@acronyms.downcase}[^a-z]+/) { |a| a.upcase }
-      phrase.gsub!(/^[a-z]|_[a-z]/) { |a| a.upcase }
-      phrase.gsub!('_', '')
-      @@cc_cache[o_phrase] = phrase
-      phrase
-    end
-    @@ucc_cache = {}
-    # Convert CamelCase to un_camel_case.
-    def self.un_camel_case(o_phrase)
-      phrase = o_phrase.to_s.dup
-      return @@ucc_cache[o_phrase] if @@ucc_cache[o_phrase]
-      phrase.gsub!(/#{@@acronyms}/) { |a| a.downcase.capitalize }
-      phrase.gsub!(/[A-Z]/) { |p| '_' + p.downcase  }
-      phrase = phrase[1..-1] if phrase[0] == '_'
-      @@ucc_cache[o_phrase] = phrase
-      phrase
-    end
-    # Return the levensthein distance between two stringsm
-    # taking into account the costs of insertion, deletion,
-    # and substitution. Stolen from:
-    # http://ruby-snippets.heroku.com/string/levenshtein-distance
-    def self.levenshtein(first, other, ins=1, del=1, sub=1)
-      return nil if first.nil? || other.nil?
-      dm = []
-      dm[0] = (0..first.length).collect { |i| i * ins}
-      fill = [0] * (first.length - 1)
-      for i in 1..other.length
-        dm[i] = [i * del, fill.flatten]
-      end
-      for i in 1..other.length
-        for j in 1..first.length
-          dm[i][j] = [
-            dm[i-1][j-1] + (first[i-1] == other[i-1] ? 0 : sub),
-            dm[i][j-1] + ins,
-            dm[i-1][j] + del
-          ].min
-        end
-      end
-      dm[other.length][first.length]
-    end
-    # Search the list to see if there are words
-    # similar to name. If yes, return a string
-    # saying "Did you mean ... ?"
-    def self.did_you_mean?(list, name)
-      msg = ''
-      sugg = []
-      list.each do |element|
-        l = levenshtein(element,name)
-        if  l > 0 && l < 2
-          sugg << element
-        end
-      end
-      unless sugg.empty?
-        if sugg.size == 1
-          msg += " Perhaps you meant '#{sugg[0]}' ?"
-        else
-          sugg_quote = sugg[0..-2].map {|x| '\'' + x + '\''}
-          msg += " Perhaps you meant #{sugg_quote.join(', ')}," +
-          " or '#{sugg[-1]}' ?"
-        end
-      end
-      msg
-    end
-    def self.caller_method(n = 3)
-      at = caller(n).first
-      /^(.+?):(\d+)(?::in `(.*)')?/ =~ at
-      :"#{Regexp.last_match[3]}"
-    end
-  end
-end
-
-# Make undefining constants publicly
-# available on any object.
-Object.module_eval do
-  def self.const_unset(const); Object.instance_eval { remove_const(const) }; puts const; end
-end
-
-# Make the most common utility functions available in the global scope.
-def create_temp_file(ext, value = nil, &block)
-  Treat::Utilities.create_temp_file(ext, value) { |f| block.call(f) }
-end
-def silence_streams(*streams); Treat::Utilities.silence_streams(*streams) { yield }; end
-def silently(&block); Treat::Utilities.silently { block.call }; end
-def cc(w); Treat::Utilities.camel_case(w); end
-def ucc(w); Treat::Utilities.un_camel_case(w); end
-def cl(n); n.to_s.split('::')[-1]; end
-def did_you_mean?(l, e); Treat::Utilities.did_you_mean?(l, e); end
-def caller_method(n = 3); Treat::Utilities.caller_method(n); end