pollex 0.0.1 → 0.0.2

data/lib/pollex/entry.rb

@@ -1,4 +1,5 @@
 module Pollex
+  # A Pollex entry, corresponding to a reflex for a reconstruction, with a language and a source.
   class Entry < PollexObject
     extend PollexClass
 
@@ -8,14 +9,19 @@ module Pollex
     attr_writer :source_code, :source_path
     attr_inspector :reflex, :description, :language_name, :source_code, :reconstruction_name, :flag
 
+    # @return [(String, nil)] the path to this entry, if given
+    # @note In some Pollex listings, entries' paths are not listed.
     def path
       @reconstruction_path
     end
 
+    # @return [Language] the Language corresponding to this entry
     def language
       @language ||= Language.new(:name => @language_name, :path => @language_path)
     end
 
+    # @return [(Source, nil)] the Source corresponding to this entry, if given
+    # @note In some Pollex listings, entries' sources are not listed.
     def source
       if @source_path
         @source ||= Source.new(:code => @source_code, :path => @source_path)
@@ -24,6 +30,8 @@ module Pollex
       end
     end
 
+    # @return [(Reconstruction, nil)] the Reconstruction corresponding to this entry, if given
+    # @note In some Pollex listings, entries' reconstructions are not listed.
     def reconstruction
       if @reconstruction_path
         @reconstruction ||= Reconstruction.new(:protoform => @reconstruction_name, :path => @reconstruction_path)
@@ -32,6 +40,9 @@ module Pollex
       end
     end
 
+    # Looks up all Entries matching a given name.
+    # @param name [String] term to search for
+    # @return [Array<Entry>] array of Entries matching the search term
     def self.find(name)
       Scraper.instance.get_all(Entry, "/search/?field=entry&query=#{name}", [
         [:reflex, 'td[3]/text()'],

data/lib/pollex/language.rb

@@ -1,4 +1,5 @@
 module Pollex
+  # A Polynesian language with entries in Pollex.
   class Language < PollexObject
     extend PollexClass
 
@@ -6,6 +7,8 @@ module Pollex
     attr_writer :code, :count
     attr_inspector :name, :code, :count, :path
 
+    # Returns all Entries belonging to this Language
+    # @return [Array<Entry>] array of Entries belonging to this Language
     def entries
       @entries ||= Scraper.instance.get_all(Entry, @path, [
         [:reflex, 'td[2]/text()'],
@@ -18,16 +21,20 @@ module Pollex
      ])
     end
 
+    # @return [String] the Language's abbreviated code
     def code
       @code ||= @path.split('/')[2].upcase
     end
 
+    # @return [Integer] number of Entries belonging to this Language
     def count
       @count ||= Scraper.instance.get(@path, [
         [:count, "p[@class='count']/text()", lambda {|x| x.split(' ').first}]
-      ])[:count]
+      ])[:count].to_i
     end
 
+    # Returns all Languages in Pollex.
+    # @return [Array<Language>] array of Languages in Pollex
     def self.all
       @languages ||= Scraper.instance.get_all(Language, "/language/", [
         [:name, 'td[2]/a/text()'],
@@ -37,10 +44,15 @@ module Pollex
       ])
     end
 
+    # Counts the number of Languages within Pollex
+    # @return [Integer] number of Languages in Pollex
     def self.count
       self.all.count
     end
 
+    # Looks up all Languages matching a given name.
+    # @param name [String] term to search for
+    # @return [Array<Language>] array of Languages matching the search term
     def self.find(name)
       Scraper.instance.get_all(Language, "/search/?field=language&query=#{name}", [
         [:name, 'td[1]/a/text()'],

data/lib/pollex/level.rb

@@ -1,4 +1,5 @@
 module Pollex
+  # A level to which protoforms are reconstructed within Pollex.
   class Level < PollexObject
     extend PollexClass
 
@@ -6,6 +7,8 @@ module Pollex
     attr_writer :subgroup, :count
     attr_inspector :token, :subgroup, :count, :path
 
+    # Returns all Reconstructions at this Level
+    # @return [Array<Reconstruction>] array of Reconstructions at this Level
     def reconstructions
       @reconstructions ||= Scraper.instance.get_all(Reconstruction, @path, [
         [:path, 'td[1]/a/@href'],
@@ -14,18 +17,22 @@ module Pollex
       ])
     end
 
+    # @return the full name of this Level
     def subgroup
       @subgroup ||= Scraper.instance.get(@path, [
         [:subgroup, 'h1/text()', lambda {|x| x.split(' - ')[1]}]
       ])[:subgroup]
     end
 
+    # @return [Integer] number of Reconstructions at this Level
     def count
       @count ||= Scraper.instance.get(@path, [
         [:count, "p[@class='count']/text()", lambda {|x| x.split(' ').first}]
       ])[:count]
     end
 
+    # Returns all Levels in Pollex.
+    # @return [Array<Level>] array of Levels in Pollex
     def self.all
       @levels ||= Scraper.instance.get_all(Source, "/level/", [
         [:token, 'td[1]/a/text()'],
@@ -35,6 +42,8 @@ module Pollex
       ])
     end
 
+    # Counts the number of Levels within Pollex
+    # @return [Integer] number of Levels in Pollex
     def self.count
       self.all.count
     end

data/lib/pollex/pollex_class.rb

@@ -1,7 +1,9 @@
 module Pollex
-  # helper instance methods
+  # Superclass for Pollex objects providing helper instance methods.
   class PollexObject
-    # taken from https://github.com/neweryankee/nextbus/blob/master/lib/instantiate_with_attrs.rb
+    # Initializes objects with a hash of attributes.
+    # @see https://github.com/neweryankee/nextbus/blob/master/lib/instantiate_with_attrs.rb
+    # @author neweryankee
     def initialize(attrs={})
       super()
       attrs.each do |name, value|
@@ -11,6 +13,9 @@ module Pollex
       self
     end
 
+    # Overrides <tt>Object#inspect</tt> to only show the attributes defined
+    # by <tt>PollexClass#attr_inspector</tt>.
+    # @see PollexClass#attr_inspector
     def inspect
       inspectables = self.class.inspectables
       if inspectables
@@ -21,10 +26,13 @@ module Pollex
     end
   end
 
-  # helper class methods
+  # Provides helper class methods for Pollex classes.
   module PollexClass
     attr_reader :inspectables
 
+    # Defines the list of attributes whose values are displayed by <tt>PollexObject#inspect</tt>.
+    # @param *attrs [Array<Symbol>] array of attribute labels
+    # @see PollexObject#inspect
     def attr_inspector(*attrs)
       @inspectables = attrs
     end

data/lib/pollex/reconstruction.rb

@@ -1,10 +1,13 @@
 module Pollex
+  # A reconstructed protoform in Pollex.
   class Reconstruction < PollexObject
     extend PollexClass
 
     attr_accessor :path, :protoform, :description, :semantic_field
     attr_inspector :protoform, :description, :path
 
+    # Returns all Entries belonging to this Reconstruction
+    # @return [Array<Entry>] array of Entries belonging to this Reconstruction
     def entries
       @entries ||= Scraper.instance.get_all(Entry, @path, [
         [:reflex, 'td[2]/text()'],
@@ -19,12 +22,14 @@ module Pollex
      ], 1)
     end
 
+    # @return [String] the Reconstruction's description
     def description
       @description ||= Scraper.instance.get(@path, [
         [:description, "table[1]/tr[1]/td/text()"]
       ])[:description]
     end
 
+    # @return [Level] the Level corresponding to this Reconstruction
     def level
       unless @level
         level_parts = Scraper.instance.get(@path, [
@@ -36,18 +41,22 @@ module Pollex
       @level
     end
 
+    # @return [String] the Reconstruction's notes
     def notes
       @notes ||= Scraper.instance.get(@path, [
         [:notes, "table[1]/tr[3]/td/p/text()"]
       ])[:notes]
     end
 
+    # @return [Integer] number of Entries belonging to this Reconstruction
     def count
       @count ||= Scraper.instance.get(@path, [
         [:count, "p[@class='count']/text()", lambda {|x| x.split(' ').first}]
       ])[:count]
     end
 
+    # Returns all Reconstructions in Pollex.
+    # @return [Array<Reconstruction>] array of Reconstructions in Pollex
     def self.all
       @sources ||= Scraper.instance.get_all(Reconstruction, "/entry/", [
         [:path, 'td[2]/a/@href'],
@@ -56,12 +65,17 @@ module Pollex
       ])
     end
 
+    # Counts the number of Reconstruction within Pollex
+    # @return [Integer] number of Reconstruction in Pollex
     def self.count
       @count ||= Scraper.instance.get("/entry/", [
         [:count, "p[@class='count']/text()", lambda {|x| x.split(' ').first}]
       ])[:count]
     end
 
+    # Looks up all Reconstructions matching a given name.
+    # @param name [String] term to search for
+    # @return [Array<Reconstruction>] array of Reconstructions matching the search term
     def self.find(name)
       Scraper.instance.get_all(Reconstruction, "/search/?field=protoform&query=#{name}", [
         [:path, 'td[2]/a/@href'],

data/lib/pollex/scraper.rb

@@ -2,28 +2,53 @@ require 'singleton'
 require 'open-uri'
 
 module Pollex
+  # Singleton object for scraping Pollex, caching the results, and extracting data.
   class Scraper
     include Singleton
 
+    attr_accessor :verbose
+
+    # Instantiates a cache of size 100 for storing scraped pages.
     def initialize()
       @cache = LRUCache.new(:max_size => 100, :default => nil)
+      @verbose = false
     end
 
-    def open_from_cache(path)
+    # Opens the given Pollex page, either by retrieving it from the cache
+    # or by making a request with Nokogiri and then storing it in the cache.
+    # @param path [String] relative path from <tt>http://pollex.org.nz</tt>
+    # @return [Nokogiri::HTML::Document] the requested page, parsed with Nokogiri
+    def open_with_cache(path)
       if @cache[path]
-        puts "Opening cached contents of http://pollex.org.nz#{path} ..."
+        if @verbose
+          puts "Opening cached contents of http://pollex.org.nz#{path} ..."
+        end
         @cache[path]
       else
-        puts "Connecting to http://pollex.org.nz#{path} ..."
+        if @verbose
+          puts "Connecting to http://pollex.org.nz#{path} ..."
+        end
         page = Nokogiri::HTML(open("http://pollex.org.nz#{path}"))
         @cache[path] = page
         page
       end
     end
 
-    # gets arbitrary data from page by xpath, with optional post-processing
+    # Gets arbitrary data from a page, with optional post-processing.
+    # @param path [String] relative path from <tt>http://pollex.org.nz</tt>
+    # @param attr_infos [Array<Array<Symbol, String, (Proc, nil)>>] an array that,
+    #   for each element to be scraped, contains an array of:
+    #   * a key for the element
+    #   * the XPath to the element, from the <tt>div#content</tt> tag of the page
+    #   * (optionally) a Proc to be performed on the element's contents
+    # @return [Array<Symbol, String>] array of key-value pairs
+    # @example Return information about the level of a given reconstruction
+    #   Scraper.instance.get(@reconstruction_path, [
+    #     [:level_token, "table[1]/tr[2]/td/a/text()", lambda {|x| x.split(':')[0]}],
+    #     [:level_path, "table[1]/tr[2]/td/a/@href"]
+    #   ])
     def get(path, attr_infos)
-      page = open_from_cache(path)
+      page = open_with_cache(path)
       contents = page.css('#content')
 
       attrs = {}
@@ -39,10 +64,32 @@ module Pollex
       attrs
     end
 
-    # gets all elements from table by xpath, with optional post-processing
+    # Gets all elements from a table within a page, with optional post-processing.
+    # The results are returned as either an array of key-value pairs or as an
+    # array of objects, if a klass is specifed. If more than one page of results is
+    # found, the first page of results is returned as a PaginatedArray.
+    # @param klass [Class] (optional) class of objects to be instantiated
+    # @param path [String] relative path from <tt>http://pollex.org.nz</tt>
+    # @param attr_infos [Array<Array<Symbol, String, (Proc, nil)>>] an array that,
+    #   for each element to be scraped, contains an array of:
+    #   * a key for the element
+    #   * the XPath to the element, from a given table
+    #   * (optionally) a Proc to be performed on the element's contents
+    # @param table_num [Integer] the number of the table on the page to process
+    #   (default: 0 - that is, the first table on the page)
+    # @return [Array<klass>] if one page of results was found
+    # @return [PaginatedArray<klass>] if multiple pages of results were found
+    # @return [Array<Array<Symbol, String>>] if no klass is specified
+    # @example Return an array of all SemanticFields in Pollex
+    #   Scraper.instance.get_all(SemanticField, "/category/", [
+    #     [:id, 'td[1]/a/text()'],
+    #     [:path, 'td[1]/a/@href'],
+    #     [:name, 'td[2]/a/text()'],
+    #     [:count, 'td[3]/text()']
+    #   ])
     def get_all(klass, path, attr_infos, table_num = 0)
       puts "Connecting to http://pollex.org.nz#{path} ..."
-      page = Nokogiri::HTML(open("http://pollex.org.nz#{path}"))
+      page = open_with_cache(path)
 
       rows = page.css('table')[table_num].css('tr')
       objs = rows[1..-1].map do |row|
@@ -81,7 +128,7 @@ module Pollex
     end
   end
 
-  # array with a pointer to the next page of results
+  # Array with an optional pointer to the next page of results
   class PaginatedArray < Array
     attr_accessor :next_page, :query
 
@@ -93,6 +140,9 @@ module Pollex
       str
     end
 
+    # Returns the next page of results, if one exists
+    # @return PaginatedArray<@query[:klass]>
+    # @see Scraper#get_all
     def more
       if @next_page
         Scraper.instance.get_all(query[:klass], @next_page, query[:attr_infos], query[:table_num])

data/lib/pollex/semantic_field.rb

@@ -1,10 +1,13 @@
 module Pollex
+  # A semantic class containing a list of Pollex reconstructed protoforms.
   class SemanticField < PollexObject
     extend PollexClass
 
     attr_accessor :id, :name, :path, :count
     attr_inspector :id, :name, :count, :path
 
+    # Returns all Reconstructions corresponding to this SemanticField
+    # @return [Array<Reconstruction>] array of Reconstructions corresponding to this SemanticField
     def reconstructions
       @reconstructions ||= Scraper.instance.get_all(Reconstruction, @path, [
         [:path, 'td[1]/a/@href'],
@@ -14,6 +17,8 @@ module Pollex
       ])
     end
 
+    # Returns all SemanticFields in Pollex.
+    # @return [Array<SemanticField>] array of SemanticFields in Pollex
     def self.all
       @semantic_fields ||= Scraper.instance.get_all(SemanticField, "/category/", [
         [:id, 'td[1]/a/text()'],
@@ -23,6 +28,8 @@ module Pollex
       ])
     end
 
+    # Counts the number of SemanticField within Pollex
+    # @return [Integer] number of SemanticField in Pollex
     def self.count
       self.all.count
     end

data/lib/pollex/source.rb

@@ -1,4 +1,5 @@
 module Pollex
+  # A source of entries in Pollex.
   class Source < PollexObject
     extend PollexClass
 
@@ -6,6 +7,8 @@ module Pollex
     attr_writer :name, :reference, :count
     attr_inspector :code, :name, :reference, :count, :path
 
+    # Returns all Entries belonging to this Source
+    # @return [Array<Entry>] array of Entries belonging to this Source
     def entries
       @entries ||= Scraper.instance.get_all(Entry, @path, [
         [:language_name, 'td[1]/a/text()'],
@@ -16,22 +19,27 @@ module Pollex
      ])
     end
 
+    # @return [String] full name of this Source
     def name
       @name ||= Scraper.instance.get(@path, [
         [:name, 'h1/text()', lambda {|x| x.match('Entries from (.*) in Pollex-Online')[1]}]
       ])[:name]
     end
 
+    # @return [String] reference information for this Source
     def reference
       @reference ||= Scraper.instance.get(@path, [
         [:name, "p[@class='ref']/text()"]
       ])[:name]
     end
 
+    # @return [Integer] number of Entries belonging to this Source
     def count
       @count ||= @entries.count
     end
 
+    # Returns all Sources in Pollex.
+    # @return [Array<Source>] array of Sources in Pollex
     def self.all
       @sources ||= Scraper.instance.get_all(Source, "/source/", [
         [:code, 'td[1]/a/text()'],

data/pollex.gemspec

@@ -1,8 +1,8 @@
 Gem::Specification.new do |s|
   s.name        = 'pollex'
-  s.version     = '0.0.1'
+  s.version     = '0.0.2'
   s.date        = '2013-03-04'
-  s.summary     = "Ruby API for scraping pollex (the Polynesian Lexicon Project)"
+  s.summary     = "Ruby wrapper for scraping pollex (the Polynesian Lexicon Project)"
   s.description = ""
   s.authors     = ["Alex Nisnevich"]
   s.email       = 'alex.nisnevich@gmail.com'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pollex
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -58,7 +58,6 @@ files:
 - lib/pollex/scraper.rb
 - lib/pollex/semantic_field.rb
 - lib/pollex/source.rb
-- pollex-0.0.1.gem
 - pollex.gemspec
 homepage: http://github.com/AlexNisnevich/pollex
 licenses: []
@@ -80,8 +79,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
-summary: Ruby API for scraping pollex (the Polynesian Lexicon Project)
+summary: Ruby wrapper for scraping pollex (the Polynesian Lexicon Project)
 test_files: []
+has_rdoc: