pollex 0.0.3 → 0.1.0

data/CHANGELOG

@@ -1,3 +1,15 @@
+3/6/2013 version 0.1.0
+* New class: Pollex:Translator
+* New methods:
+	* Entry.terms        - parses the entry's description and returns an array
+	                       of English definitions for the entry
+	* Source.grammar     - outputs information about parsing descriptions
+	                       of the source's entries
+	* SemanticField.find - filters SemanticField.all by name
+* Including Pollex::Level
+* Fixing Entry#inspect
+* Error handling in PollexObject#inspect
+
 3/5/2013 version 0.0.3
 * Minor bug-fix, adding version and changelog
 

data/lib/pollex.rb

@@ -3,16 +3,24 @@ end
 
 require 'nokogiri'
 require 'lrucache'
+require 'cld'
+require 'json'
+
+require 'singleton'
+require 'open-uri'
+require 'pp'
 
 [
   'version',
   'pollex_class',
   'scraper',
+  'translator',
   'entry',
   'language',
   'reconstruction',
   'semantic_field',
-  'source'
+  'source',
+  'level'
 ].each do |file|
   require File.dirname(__FILE__) + "/pollex/#{file}.rb"
 end

data/lib/pollex/entry.rb

@@ -3,7 +3,7 @@ module Pollex
   class Entry < PollexObject
     extend PollexClass
 
-    attr_accessor :reflex, :description, :flag
+    attr_accessor :reflex, :description, :language_name, :source_code, :reconstruction_name, :flag
     attr_writer :reconstruction_name, :reconstruction_path
     attr_writer :language_name, :language_path
     attr_writer :source_code, :source_path
@@ -15,9 +15,40 @@ module Pollex
       @reconstruction_path
     end
 
+    # Processes the description of this entry and extracts a lits of
+    # definitions, translated into English if necessary.
+    # @return [Array<String>] definitions corresponding to this entry
+    def terms
+      string = @description
+      grammar = description_grammar
+
+      # trim last part of description, if necessary
+      if grammar[:trim_after]
+        string = string.split(grammar[:trim_after])[0]
+      end
+
+      # split into terms, remove any unnecessary expressions
+      terms = string.split(grammar[:dividers])
+                    .map {|t| t.sub(grammar[:trim_expressions], '')
+                               .strip
+                               .capitalize }
+                    .select {|t| t.match(/\w/) }
+
+      # attempt to translate to English if necessary
+      if grammar[:language] != 'en'
+        terms.map! {|t| Translator.instance.translate(t, grammar[:language], terms) }
+      end
+
+      terms
+    end
+
     # @return [Language] the Language corresponding to this entry
     def language
-      @language ||= Language.new(:name => @language_name, :path => @language_path)
+      if @language_path
+        @language ||= Language.new(:name => @language_name, :path => @language_path)
+      else
+        nil
+      end
     end
 
     # @return [(Source, nil)] the Source corresponding to this entry, if given
@@ -54,5 +85,17 @@ module Pollex
         [:flag, "td[3]/span[@class='flag']/text()"]
       ])
     end
+
+    private
+
+    # @return [Hash] grammatical information pertaining to the description of this
+    #   entry, used by Entry#terms
+    def description_grammar
+      if source
+        source.grammar
+      else
+        Source.new.grammar
+      end
+    end
   end
 end

data/lib/pollex/pollex_class.rb

@@ -19,7 +19,7 @@ module Pollex
     def inspect
       inspectables = self.class.inspectables
       if inspectables
-        "#<#{self.class}:0x#{object_id.to_s(16)} " + inspectables.map {|i| "@#{i}=\"#{send(i)}\""}.join(' ') + ">"
+        "#<#{self.class}:0x#{object_id.to_s(16)} " + inspectables.map {|i| "@#{i}=\"#{send(i) rescue nil}\""}.join(' ') + ">"
       else
         super
       end

data/lib/pollex/scraper.rb

@@ -1,6 +1,3 @@
-require 'singleton'
-require 'open-uri'
-
 module Pollex
   # Singleton object for scraping Pollex, caching the results, and extracting data.
   class Scraper

data/lib/pollex/semantic_field.rb

@@ -28,10 +28,19 @@ module Pollex
       ])
     end
 
-    # Counts the number of SemanticField within Pollex
-    # @return [Integer] number of SemanticField in Pollex
+    # Counts the number of SemanticFields within Pollex
+    # @return [Integer] number of SemanticFields in Pollex
     def self.count
       self.all.count
     end
+
+    # Looks up all SemanticFields matching a given name.
+    # @note Pollex has no built-in search for SemanticFields, so this method is
+    #   simply a filter over SemanticField.all.
+    # @param name [String] term to search for
+    # @return [Array<SemanticField>] array of SemanticFields matching the search term
+    def self.find(name)
+      self.all.select { |sf| sf.name.downcase.include?(name.downcase) }
+    end
   end
 end

data/lib/pollex/source.rb

@@ -13,6 +13,8 @@ module Pollex
       @entries ||= Scraper.instance.get_all(Entry, @path, [
         [:language_name, 'td[1]/a/text()'],
         [:language_path, 'td[1]/a/@href'],
+        [:source_code, nil, lambda {|x| @code}],
+        [:source_path, nil, lambda {|x| @path}],
         [:reflex, 'td[2]/text()'],
         [:description, 'td[3]/text()'],
         [:flag, "td[3]/span[@class='flag']/text()"]
@@ -38,6 +40,76 @@ module Pollex
       @count ||= @entries.count
     end
 
+    # Returns grammatical information for this source, used for
+    # intelligently parsing the descriptions of entries from this source
+    # @note Information is currently entered for all sources on
+    #   http://pollex.org.nz/source/ up to (and including)
+    #   Bse
+    # @return [Hash] grammatical information pertaining to the descriptions
+    #   of this sources' entries
+    # @see Entry#terms
+    def grammar
+      # first, assume reasonable defaults
+
+      language = 'en' # default language: English
+      dividers = /[,;]/ # default: split on comma and semicolon
+      trim_expressions = '' # default: don't trim any expressions
+      trim_after = nil # default: don't trim any trailing text
+
+      # now bring in source-specific information
+
+      if ['Cnt', 'Bxn'].include? @code
+        # Spanish-language sources
+        language = 'es'
+      elsif ['Aca', 'Bgn', 'Btn', 'Hmn', 'Rch'].include? @code
+        # French-language sources
+        language = 'fr'
+      end
+
+      if ['Aca', 'Bxn'].include? @code
+        # split by comma, semicolon, period
+        dividers = /(,|;|\. )/
+      elsif ['Atn', 'Bwh', 'Hmn'].include? @code
+        # don't split at all
+        dividers = '\n' # dividers = nil doesn't work
+      elsif ['Bgn', 'Bst', 'Brn'].include? @code
+        # split by period
+        dividers = '.'
+      elsif ['Bkr', 'Bgs'].include? @code
+        # split by comma, period
+        dividers = /(,|\. )/
+      elsif ['Bge', 'Bck'].include? @code
+        # split by semicolon
+        dividers = ';'
+      end
+
+      if ['McP', 'Dsn'].include? @code
+        # Trim all (parenthetical expressions)
+        trim_expressions = /\(.*\)/
+      elsif ['Cnt', 'Aca', 'Bse', 'Hmn'].include? @code
+        # Trim parenthetical expressions that are <= 4 chars or contain numbers
+        trim_expressions = /\((.{0,4}|.*[0-9].*)\)/
+      elsif ['Stz', 'Bck'].include? @code
+        # Trim parenthetical expressions that contain numbers
+        trim_expressions = /\(.*[0-9].*\)/
+      elsif ['Rsr'].include? @code
+        # Trim all "expressions in quotes"
+        trim_expressions = /".*"/
+      end
+
+      if ['Btl', 'Bck'].include? @code
+        # Trim everything after a period
+        trim_after = '.'
+      end
+
+      {
+        :language => language,
+        :dividers => dividers,
+        :trim_expressions => trim_expressions,
+        :trim_after => trim_after
+      }
+    end
+
     # Returns all Sources in Pollex.
     # @return [Array<Source>] array of Sources in Pollex
     def self.all
@@ -49,5 +121,11 @@ module Pollex
         [:reference, 'td[4]/text()']
       ])
     end
+
+    # Counts the number of Sources within Pollex
+    # @return [Integer] number of Sources in Pollex
+    def self.count
+      self.all.count
+    end
   end
 end

data/lib/pollex/translator.rb

@@ -0,0 +1,49 @@
+module Pollex
+  # Singleton object for translating descriptions into English.
+  class Translator
+    include Singleton
+
+    # Instantiates a cache of size 100 for storing translations.
+    def initialize()
+      @cache = LRUCache.new(:max_size => 2500, :default => nil)
+    end
+
+    # Translates a phrase into English using the free MyMemory API, and caches
+    # the result.
+    # @note MyMemory currently has a limit of 100 API requests per IP per day for
+    #   unregistered users.
+    # @param phrase [String] Phrase to be translated
+    # @param source_lang_code [String] Two-letter language code for the source language
+    # @param context [Array<String>] Adjoining phrases (optional)
+    # @result [String] Translated phrase
+    def translate(phrase, source_lang_code, context = nil)
+      context ||= [phrase]
+      if context.all? {|x| CLD.detect_language(x)[:code] == 'en'}
+        # we are reasonably sure that this phrase is already in English - no need to translate
+        phrase
+      else
+        # first, check the cache
+        key = [phrase, source_lang_code]
+        if @cache[key]
+          @cache[key]
+        else
+          # make a request to MyMemory
+          puts "Translating '#{phrase}' ..."
+          url = "http://mymemory.translated.net/api/get?q=#{URI::encode(phrase)}&langpair=#{source_lang_code}%7Cen"
+          results_json = open(url).read
+          result = JSON.parse(results_json)['responseData']['translatedText']
+
+          if result.include? 'MYMEMORY WARNING'
+            # translation failed - return original phrase
+            puts result
+            phrase
+          else
+            # translation succeeded - store into cache and return translated phrase
+            @cache[key] = result
+            result
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pollex/version.rb

@@ -1,3 +1,3 @@
 module Pollex
-  VERSION = '0.0.3'
+  VERSION = '0.1.0'
 end

data/pollex.gemspec

@@ -14,4 +14,6 @@ Gem::Specification.new do |s|
 
   s.add_dependency 'nokogiri'
   s.add_dependency 'lrucache'
+  s.add_dependency 'cld'
+  s.add_dependency 'json'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pollex
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-06 00:00:00.000000000 Z
+date: 2013-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -43,6 +43,38 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: cld
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Ruby wrapper for scraping pollex (the Polynesian Lexicon Project)
 email: alex.nisnevich@gmail.com
 executables: []
@@ -60,6 +92,7 @@ files:
 - lib/pollex/scraper.rb
 - lib/pollex/semantic_field.rb
 - lib/pollex/source.rb
+- lib/pollex/translator.rb
 - lib/pollex/version.rb
 - pollex.gemspec
 homepage: http://github.com/AlexNisnevich/pollex
@@ -82,7 +115,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Ruby wrapper for scraping pollex (the Polynesian Lexicon Project)