pollex 0.0.1

data/lib/pollex.rb

@@ -0,0 +1,10 @@
+module Pollex
+end
+
+require 'nokogiri'
+require 'lrucache'
+
+['pollex_class', 'scraper', 'entry', 'language', 'reconstruction',
+      'semantic_field', 'source'].each do |file|
+  require File.dirname(__FILE__) + "/pollex/#{file}.rb"
+end

data/lib/pollex/entry.rb

@@ -0,0 +1,47 @@
+module Pollex
+  class Entry < PollexObject
+    extend PollexClass
+
+    attr_accessor :reflex, :description, :flag
+    attr_writer :reconstruction_name, :reconstruction_path
+    attr_writer :language_name, :language_path
+    attr_writer :source_code, :source_path
+    attr_inspector :reflex, :description, :language_name, :source_code, :reconstruction_name, :flag
+
+    def path
+      @reconstruction_path
+    end
+
+    def language
+      @language ||= Language.new(:name => @language_name, :path => @language_path)
+    end
+
+    def source
+      if @source_path
+        @source ||= Source.new(:code => @source_code, :path => @source_path)
+      else
+        nil
+      end
+    end
+
+    def reconstruction
+      if @reconstruction_path
+        @reconstruction ||= Reconstruction.new(:protoform => @reconstruction_name, :path => @reconstruction_path)
+      else
+        nil
+      end
+    end
+
+    def self.find(name)
+      Scraper.instance.get_all(Entry, "/search/?field=entry&query=#{name}", [
+        [:reflex, 'td[3]/text()'],
+        [:description, 'td[4]/text()'],
+        [:language_path, 'td[1]/a/@href'],
+        [:language_name, 'td[1]/a/text()'],
+        [:reconstruction_path, 'td[2]/a/@href'],
+        [:reconstruction_name, 'td[2]/a/text()', lambda {|x| x.split('.')[1..-1].join('.')}],
+        [:flag, "td[3]/span[@class='flag']/text()"]
+      ])
+    end
+  end
+end

data/lib/pollex/language.rb

@@ -0,0 +1,51 @@
+module Pollex
+  class Language < PollexObject
+    extend PollexClass
+
+    attr_accessor :name, :path
+    attr_writer :code, :count
+    attr_inspector :name, :code, :count, :path
+
+    def entries
+      @entries ||= Scraper.instance.get_all(Entry, @path, [
+        [:reflex, 'td[2]/text()'],
+        [:description, 'td[3]/text()'],
+        [:language_name, nil, lambda {|x| @name}],
+        [:language_path, nil, lambda {|x| @path}],
+        [:source_code, 'td[4]/a/text()'],
+        [:source_path, 'td[4]/a/@href'],
+        [:flag, "td[3]/span[@class='flag']/text()"]
+     ])
+    end
+
+    def code
+      @code ||= @path.split('/')[2].upcase
+    end
+
+    def count
+      @count ||= Scraper.instance.get(@path, [
+        [:count, "p[@class='count']/text()", lambda {|x| x.split(' ').first}]
+      ])[:count]
+    end
+
+    def self.all
+      @languages ||= Scraper.instance.get_all(Language, "/language/", [
+        [:name, 'td[2]/a/text()'],
+        [:path, 'td[1]/a/@href'],
+        [:code, 'td[1]/a/text()'],
+        [:count, 'td[3]/text()']
+      ])
+    end
+
+    def self.count
+      self.all.count
+    end
+
+    def self.find(name)
+      Scraper.instance.get_all(Language, "/search/?field=language&query=#{name}", [
+        [:name, 'td[1]/a/text()'],
+        [:path, 'td[1]/a/@href']
+      ])
+    end
+  end
+end

data/lib/pollex/level.rb

@@ -0,0 +1,42 @@
+module Pollex
+  class Level < PollexObject
+    extend PollexClass
+
+    attr_accessor :token, :path
+    attr_writer :subgroup, :count
+    attr_inspector :token, :subgroup, :count, :path
+
+    def reconstructions
+      @reconstructions ||= Scraper.instance.get_all(Reconstruction, @path, [
+        [:path, 'td[1]/a/@href'],
+        [:protoform, 'td[1]/a/text()'],
+        [:description, 'td[2]/text()']
+      ])
+    end
+
+    def subgroup
+      @subgroup ||= Scraper.instance.get(@path, [
+        [:subgroup, 'h1/text()', lambda {|x| x.split(' - ')[1]}]
+      ])[:subgroup]
+    end
+
+    def count
+      @count ||= Scraper.instance.get(@path, [
+        [:count, "p[@class='count']/text()", lambda {|x| x.split(' ').first}]
+      ])[:count]
+    end
+
+    def self.all
+      @levels ||= Scraper.instance.get_all(Source, "/level/", [
+        [:token, 'td[1]/a/text()'],
+        [:subgroup, 'td[2]/a/text()'],
+        [:path, 'td[2]/a/@href'],
+        [:count, 'td[3]/a/text()'],
+      ])
+    end
+
+    def self.count
+      self.all.count
+    end
+  end
+end

data/lib/pollex/pollex_class.rb

@@ -0,0 +1,32 @@
+module Pollex
+  # helper instance methods
+  class PollexObject
+    # taken from https://github.com/neweryankee/nextbus/blob/master/lib/instantiate_with_attrs.rb
+    def initialize(attrs={})
+      super()
+      attrs.each do |name, value|
+        setter = "#{name.to_s}=".to_sym
+        self.send(setter, value) if self.respond_to?(setter)
+      end
+      self
+    end
+
+    def inspect
+      inspectables = self.class.inspectables
+      if inspectables
+        "#<#{self.class}:0x#{object_id.to_s(16)} " + inspectables.map {|i| "@#{i}=\"#{send(i)}\""}.join(' ') + ">"
+      else
+        super
+      end
+    end
+  end
+
+  # helper class methods
+  module PollexClass
+    attr_reader :inspectables
+
+    def attr_inspector(*attrs)
+      @inspectables = attrs
+    end
+  end
+end

data/lib/pollex/reconstruction.rb

@@ -0,0 +1,73 @@
+module Pollex
+  class Reconstruction < PollexObject
+    extend PollexClass
+
+    attr_accessor :path, :protoform, :description, :semantic_field
+    attr_inspector :protoform, :description, :path
+
+    def entries
+      @entries ||= Scraper.instance.get_all(Entry, @path, [
+        [:reflex, 'td[2]/text()'],
+        [:description, 'td[3]/text()'],
+        [:language_name, 'td[1]/a/text()'],
+        [:language_path, 'td[1]/a/@href'],
+        [:source_code, 'td[4]/a/text()'],
+        [:source_path, 'td[4]/a/@href'],
+        [:reconstruction_name, nil, lambda {|x| @protoform}],
+        [:reconstruction_path, nil, lambda {|x| @path}],
+        [:flag, "td[3]/span[@class='flag']/text()"]
+     ], 1)
+    end
+
+    def description
+      @description ||= Scraper.instance.get(@path, [
+        [:description, "table[1]/tr[1]/td/text()"]
+      ])[:description]
+    end
+
+    def level
+      unless @level
+        level_parts = Scraper.instance.get(@path, [
+          [:token, "table[1]/tr[2]/td/a/text()", lambda {|x| x.split(':')[0]}],
+          [:path, "table[1]/tr[2]/td/a/@href"]
+        ])
+        @level = Level.new(:token => level_parts[:token], :path => level_parts[:path])
+      end
+      @level
+    end
+
+    def notes
+      @notes ||= Scraper.instance.get(@path, [
+        [:notes, "table[1]/tr[3]/td/p/text()"]
+      ])[:notes]
+    end
+
+    def count
+      @count ||= Scraper.instance.get(@path, [
+        [:count, "p[@class='count']/text()", lambda {|x| x.split(' ').first}]
+      ])[:count]
+    end
+
+    def self.all
+      @sources ||= Scraper.instance.get_all(Reconstruction, "/entry/", [
+        [:path, 'td[2]/a/@href'],
+        [:protoform, 'td[2]/a/text()'],
+        [:description, 'td[3]/text()']
+      ])
+    end
+
+    def self.count
+      @count ||= Scraper.instance.get("/entry/", [
+        [:count, "p[@class='count']/text()", lambda {|x| x.split(' ').first}]
+      ])[:count]
+    end
+
+    def self.find(name)
+      Scraper.instance.get_all(Reconstruction, "/search/?field=protoform&query=#{name}", [
+        [:path, 'td[2]/a/@href'],
+        [:protoform, 'td[2]/a/text()'],
+        [:description, 'td[3]/text()']
+      ])
+    end
+  end
+end

data/lib/pollex/scraper.rb

@@ -0,0 +1,104 @@
+require 'singleton'
+require 'open-uri'
+
+module Pollex
+  class Scraper
+    include Singleton
+
+    def initialize()
+      @cache = LRUCache.new(:max_size => 100, :default => nil)
+    end
+
+    def open_from_cache(path)
+      if @cache[path]
+        puts "Opening cached contents of http://pollex.org.nz#{path} ..."
+        @cache[path]
+      else
+        puts "Connecting to http://pollex.org.nz#{path} ..."
+        page = Nokogiri::HTML(open("http://pollex.org.nz#{path}"))
+        @cache[path] = page
+        page
+      end
+    end
+
+    # gets arbitrary data from page by xpath, with optional post-processing
+    def get(path, attr_infos)
+      page = open_from_cache(path)
+      contents = page.css('#content')
+
+      attrs = {}
+      attr_infos.each do |name, xpath, post_processor|
+        attrs[name] = ''
+        if xpath
+          attrs[name] = contents.at_xpath(xpath).to_s.strip
+        end
+        if post_processor
+          attrs[name] = post_processor.call(attrs[name])
+        end
+      end
+      attrs
+    end
+
+    # gets all elements from table by xpath, with optional post-processing
+    def get_all(klass, path, attr_infos, table_num = 0)
+      puts "Connecting to http://pollex.org.nz#{path} ..."
+      page = Nokogiri::HTML(open("http://pollex.org.nz#{path}"))
+
+      rows = page.css('table')[table_num].css('tr')
+      objs = rows[1..-1].map do |row|
+        attrs = {}
+        attr_infos.each do |name, xpath, post_processor|
+          attrs[name] = ''
+          if xpath
+            attrs[name] = row.at_xpath(xpath).to_s.strip
+          end
+          if post_processor
+            attrs[name] = post_processor.call(attrs[name])
+          end
+        end
+        attrs
+      end
+
+      # check if there is a "next" page
+      last_link = page.css('.pagination a').last()
+      if last_link and last_link.text()[0..3] == 'Next'
+        last_link_path = last_link.attributes()['href']
+        new_path = path.split('?')[0] + last_link_path
+
+        results = PaginatedArray.new()
+        results.query = {:klass => klass, :attr_infos => attr_infos, :table_num => table_num}
+        results.next_page = new_path
+        results.concat(objs.to_a) # merge rather than create new array
+      else
+        results = objs
+      end
+
+      if klass
+        results.map! {|x| klass.new(x) }
+      end
+
+      results
+    end
+  end
+
+  # array with a pointer to the next page of results
+  class PaginatedArray < Array
+    attr_accessor :next_page, :query
+
+    def inspect
+      str = super.inspect
+      if @next_page
+        str += "\nThere are more items available at #{@next_page}. Use _.more to get them."
+      end
+      str
+    end
+
+    def more
+      if @next_page
+        Scraper.instance.get_all(query[:klass], @next_page, query[:attr_infos], query[:table_num])
+      else
+        nil
+      end
+    end
+  end
+end

data/lib/pollex/semantic_field.rb

@@ -0,0 +1,30 @@
+module Pollex
+  class SemanticField < PollexObject
+    extend PollexClass
+
+    attr_accessor :id, :name, :path, :count
+    attr_inspector :id, :name, :count, :path
+
+    def reconstructions
+      @reconstructions ||= Scraper.instance.get_all(Reconstruction, @path, [
+        [:path, 'td[1]/a/@href'],
+        [:protoform, 'td[1]/a/text()'],
+        [:description, 'td[2]/text()'],
+        [:semantic_field, nil, lambda {|x| self}]
+      ])
+    end
+
+    def self.all
+      @semantic_fields ||= Scraper.instance.get_all(SemanticField, "/category/", [
+        [:id, 'td[1]/a/text()'],
+        [:path, 'td[1]/a/@href'],
+        [:name, 'td[2]/a/text()'],
+        [:count, 'td[3]/text()']
+      ])
+    end
+
+    def self.count
+      self.all.count
+    end
+  end
+end

data/lib/pollex/source.rb

@@ -0,0 +1,45 @@
+module Pollex
+  class Source < PollexObject
+    extend PollexClass
+
+    attr_accessor :code, :path
+    attr_writer :name, :reference, :count
+    attr_inspector :code, :name, :reference, :count, :path
+
+    def entries
+      @entries ||= Scraper.instance.get_all(Entry, @path, [
+        [:language_name, 'td[1]/a/text()'],
+        [:language_path, 'td[1]/a/@href'],
+        [:reflex, 'td[2]/text()'],
+        [:description, 'td[3]/text()'],
+        [:flag, "td[3]/span[@class='flag']/text()"]
+     ])
+    end
+
+    def name
+      @name ||= Scraper.instance.get(@path, [
+        [:name, 'h1/text()', lambda {|x| x.match('Entries from (.*) in Pollex-Online')[1]}]
+      ])[:name]
+    end
+
+    def reference
+      @reference ||= Scraper.instance.get(@path, [
+        [:name, "p[@class='ref']/text()"]
+      ])[:name]
+    end
+
+    def count
+      @count ||= @entries.count
+    end
+
+    def self.all
+      @sources ||= Scraper.instance.get_all(Source, "/source/", [
+        [:code, 'td[1]/a/text()'],
+        [:path, 'td[1]/a/@href'],
+        [:name, 'td[2]/a/text()'],
+        [:count, 'td[3]/text()'],
+        [:reference, 'td[4]/text()']
+      ])
+    end
+  end
+end

data/pollex.gemspec

@@ -0,0 +1,15 @@
+Gem::Specification.new do |s|
+  s.name        = 'pollex'
+  s.version     = '0.0.1'
+  s.date        = '2013-03-04'
+  s.summary     = "Ruby API for scraping pollex (the Polynesian Lexicon Project)"
+  s.description = ""
+  s.authors     = ["Alex Nisnevich"]
+  s.email       = 'alex.nisnevich@gmail.com'
+  s.homepage    = 'http://github.com/AlexNisnevich/pollex'
+
+  s.files       = `git ls-files`.split("\n")
+
+  s.add_dependency 'nokogiri'
+  s.add_dependency 'lrucache'
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: pollex
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Alex Nisnevich
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-03-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: lrucache
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email: alex.nisnevich@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/pollex.rb
+- lib/pollex/entry.rb
+- lib/pollex/language.rb
+- lib/pollex/level.rb
+- lib/pollex/pollex_class.rb
+- lib/pollex/reconstruction.rb
+- lib/pollex/scraper.rb
+- lib/pollex/semantic_field.rb
+- lib/pollex/source.rb
+- pollex-0.0.1.gem
+- pollex.gemspec
+homepage: http://github.com/AlexNisnevich/pollex
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Ruby API for scraping pollex (the Polynesian Lexicon Project)
+test_files: []