freescrape 0.0.9

data/History.txt

@@ -0,0 +1,7 @@
+=== 0.0.9 / 2008-09-28
+
+* Initial release.
+* Can request items from freebase.com using either a URL, an Item GUID or an
+  Item name.
+* Preserves tags and other freebase links.
+

data/Manifest.txt

@@ -0,0 +1,14 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+lib/freebase.rb
+lib/freebase/item_link.rb
+lib/freebase/category.rb
+lib/freebase/item.rb
+lib/freebase/freebase.rb
+lib/freebase/version.rb
+tasks/spec.rb
+spec/item_spec.rb
+spec/freebase_spec.rb
+spec/spec_helper.rb

data/README.txt

@@ -0,0 +1,35 @@
+= FreeBase
+
+* http://freebase.rubyforge.org/
+* Postmodern (postmodern.mod3@gmail.com)
+
+== DESCRIPTION:
+
+A web-scraping interface to freebase.com, the open and shared database of the
+world's knowledge.
+
+== FEATURES/PROBLEMS:
+
+* Can request items from freebase.com using either a URL, an Item GUID or an
+  Item name.
+* Preserves tags and other freebase links.
+
+== EXAMPLES:
+
+  require 'freebase'
+
+  Freebase.item('Aphex Twin')
+  # => #<Freebase::Item:0xb73fdba0 ...>
+
+  Freebase.item('http://www.freebase.com/view/guid/9202a8c04000641f8000000003ac957f')
+  # => #<Freebase::Item:0xb73fe3dc ...>
+
+== REQUIREMENTS:
+
+* Hpricot
+* WWW::Mechanize
+
+== INSTALL:
+
+  $ sudo gem install freescrape
+

data/Rakefile

@@ -0,0 +1,15 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+require './tasks/spec.rb'
+require './lib/freebase/version.rb'
+
+Hoe.new('freescrape', Freebase::VERSION) do |p|
+  p.rubyforge_name = 'freebase'
+  p.remote_rdoc_dir = ''
+  p.developer('Postmodern Modulus III', 'postmodern.mod3@gmail.com')
+  p.extra_deps = ['hpricot', 'mechanize']
+end
+
+# vim: syntax=Ruby

data/lib/freebase.rb

@@ -0,0 +1,3 @@
+require 'freebase/item'
+require 'freebase/freebase'
+require 'freebase/version'

data/lib/freebase/category.rb

@@ -0,0 +1,33 @@
+module Freebase
+  class Category
+
+    # Name of the category
+    attr_reader :name
+
+    # Metadata associated with the category
+    attr_reader :metadata
+
+    #
+    # Creates a new Category object with the specified _name_.
+    #
+    def initialize(name)
+      @name = name
+      @metadata = {}
+    end
+
+    #
+    # Returns the metadata with the specified _name_.
+    #
+    def [](name)
+      @metadata[name]
+    end
+
+    #
+    # Returns the name of the category in +String+ form.
+    #
+    def to_s
+      @name.to_s
+    end
+
+  end
+end

data/lib/freebase/freebase.rb

@@ -0,0 +1,185 @@
+require 'freebase/item'
+require 'freebase/item_link'
+
+require 'www/mechanize'
+require 'hpricot'
+require 'open-uri'
+
+module Freebase
+  # Common proxy port
+  COMMON_PROXY_PORT = 8080
+
+  # Default language
+  DEFAULT_LANGUAGE = :en
+
+  #
+  # Returns the +Hash+ of proxy information.
+  #
+  def Freebase.proxy
+    @@free_base_proxy ||= {
+      :host => nil,
+      :port => COMMON_PROXY_PORT,
+      :user => nil,
+      :password => nil
+    }
+  end
+
+  #
+  # Creates a HTTP URI based from the given _proxy_info_ hash. The
+  # _proxy_info_ hash defaults to Web.proxy, if not given.
+  #
+  # _proxy_info_ may contain the following keys:
+  # <tt>:host</tt>:: The proxy host.
+  # <tt>:port</tt>:: The proxy port. Defaults to COMMON_PROXY_PORT,
+  #                  if not specified.
+  # <tt>:user</tt>:: The user-name to login as.
+  # <tt>:password</tt>:: The password to login with.
+  #
+  def Freebase.proxy_uri(proxy_info=Freebase.proxy)
+    if Freebase.proxy[:host]
+      return URI::HTTP.build(:host => Freebase.proxy[:host],
+                             :port => Freebase.proxy[:port],
+                             :userinfo => "#{Freebase.proxy[:user]}:#{Freebase.proxy[:password]}",
+                             :path => '/')
+    end
+  end
+  
+  #
+  # Returns the supported Freebase User-Agent Aliases.
+  #
+  def Freebase.user_agent_aliases
+    WWW::Mechanize::AGENT_ALIASES
+  end
+
+  #
+  # Returns the Freebase User-Agent
+  #
+  def Freebase.user_agent
+    @@free_base_user_agent ||= Freebase.user_agent_aliases['Windows IE 6']
+  end
+
+  #
+  # Sets the Freebase User-Agent to the specified _agent_.
+  #
+  def Freebase.user_agent=(agent)
+    @@free_base_user_agent = agent
+  end
+
+  #
+  # Sets the Freebase User-Agent using the specified user-agent alias
+  # _name_.
+  # 
+  def Freebase.user_agent_alias=(name)
+    @@free_base_user_agent = Freebase.user_agent_aliases[name.to_s]
+  end
+
+  #
+  # Opens the _uri_ with the given _options_. The contents of the _uri_
+  # will be returned.
+  #
+  # _options_ may contain the following keys:
+  # <tt>:user_agent_alias</tt>:: The User-Agent Alias to use.
+  # <tt>:user_agent</tt>:: The User-Agent String to use.
+  # <tt>:proxy</tt>:: A +Hash+ of proxy information which may
+  #                   contain the following keys:
+  #                   <tt>:host</tt>:: The proxy host.
+  #                   <tt>:port</tt>:: The proxy port.
+  #                   <tt>:user</tt>:: The user-name to login as.
+  #                   <tt>:password</tt>:: The password to login with.
+  #
+  #   Freebase.open_uri('http://www.hackety.org/')
+  #
+  #   Freebase.open_uri('http://tenderlovemaking.com/',
+  #     :user_agent_alias => 'Linux Mozilla')
+  #   Freebase.open_uri('http://www.wired.com/',
+  #     :user_agent => 'the future')
+  #
+  def Freebase.open_uri(uri,options={})
+    headers = {}
+
+    if options[:user_agent_alias]
+      headers['User-Agent'] = WWW::Mechanize::AGENT_ALIASES[options[:user_agent_alias]]
+    elsif options[:user_agent]
+      headers['User-Agent'] = options[:user_agent]
+    elsif Freebase.user_agent
+      headers['User-Agent'] = Freebase.user_agent
+    end
+
+    proxy = (options[:proxy] || Freebase.proxy)
+    if proxy[:host]
+      headers[:proxy] = Freebase.proxy_uri(proxy)
+    end
+
+    return Kernel.open(uri,headers)
+  end
+
+  #
+  # Similar to Freebase.open_uri but returns an Hpricot document.
+  #
+  def Freebase.open_page(uri,options={})
+    Hpricot(Freebase.open_uri(uri,options))
+  end
+
+  #
+  # Creates a new WWW::Mechanize agent with the given _options_.
+  #
+  # _options_ may contain the following keys:
+  # <tt>:user_agent_alias</tt>:: The User-Agent Alias to use.
+  # <tt>:user_agent</tt>:: The User-Agent string to use.
+  # <tt>:proxy</tt>:: A +Hash+ of proxy information which may
+  #                   contain the following keys:
+  #                   <tt>:host</tt>:: The proxy host.
+  #                   <tt>:port</tt>:: The proxy port.
+  #                   <tt>:user</tt>:: The user-name to login as.
+  #                   <tt>:password</tt>:: The password to login with.
+  #
+  #   Freebase.web_agent
+  #
+  #   Freebase.web_agent(:user_agent_alias => 'Linux Mozilla')
+  #   Freebase.web_agent(:user_agent => 'Google Bot')
+  #
+  def Freebase.web_agent(options={},&block)
+    agent = WWW::Mechanize.new
+
+    if options[:user_agent_alias]
+      agent.user_agent_alias = options[:user_agent_alias]
+    elsif options[:user_agent]
+      agent.user_agent = options[:user_agent]
+    elsif Freebase.user_agent
+      agent.user_agent = Freebase.user_agent
+    end
+
+    proxy = (options[:proxy] || Freebase.proxy)
+    if proxy[:host]
+      agent.set_proxy(proxy[:host],proxy[:port],proxy[:user],proxy[:password])
+    end
+
+    block.call(agent) if block
+    return agent
+  end
+
+  #
+  # Returns the language to access Freebase with.
+  #
+  def Freebase.language
+    @@free_base_language ||= DEFAULT_LANGUAGE
+  end
+
+  #
+  # Sets the language to access Freebase with to the _new_language_.
+  #
+  def Freebase.language=(new_language)
+    @@free_base_language = new_language.to_sym
+  end
+
+  #
+  # Returns the Item with the specified _descriptor_, which can be either
+  # a URI to freebase.com, an Item GUID or an Item name.
+  #
+  #   Freebase.item('Aphex Twin')
+  #   # => #<Freebase::Item:0xb73fdba0 ...>
+  #
+  def Freebase.item(descriptor)
+    Item.from(descriptor)
+  end
+end

data/lib/freebase/item.rb

@@ -0,0 +1,171 @@
+require 'freebase/item_link'
+require 'freebase/category'
+
+require 'uri'
+
+module Freebase
+  class Item
+
+    # URL of the item
+    attr_reader :url
+
+    # Name of the item
+    attr_accessor :name
+
+    # Description of the item
+    attr_accessor :summary
+
+    # Categories the item is in
+    attr_reader :categories
+
+    #
+    # Creates a new Item object with the specified _name_ and the given
+    # _options_.
+    #
+    def initialize(url,options={})
+      @url = url
+      @name = options[:name]
+      @summary = options[:summary]
+
+      @categories = {}
+    end
+
+    #
+    # Returns the Item object with the specified _descriptor_, which can
+    # be either a URI to freebase.com, an Item GUID or an Item name.
+    #
+    def Item.from(descriptor)
+      descriptor = descriptor.to_s
+
+      if descriptor =~ /^[0-9a-f]+$/
+        return Item.guid(descriptor)
+      elsif descriptor =~ /^http(s)?:\/\/(www\.)?freebase.com\/view/
+        return Item.from_url(descriptor)
+      else
+        return Item.named(descriptor)
+      end
+    end
+
+    #
+    # Returns the Item object with the specified _name_.
+    #
+    def Item.named(name)
+      name = name.split(' ').map { |word|
+        word.downcase
+      }.join('_')
+
+      return Item.from_url("http://www.freebase.com/view/#{Freebase.language}/#{name}")
+    end
+
+    #
+    # Returns the Item object with the specified _guid_.
+    #
+    def Item.guid(guid)
+      Item.from_url("http://www.freebase.com/view/guid/#{guid}")
+    end
+
+    #
+    # Creates the Item at the specified _url_.
+    #
+    #   Item.from_url('http://www.freebase.com/view/guid/9202a8c04000641f800000000301146f')
+    #   # => #<Freebase::Item:0xb73fdba0 ...>
+    #
+    def Item.from_url(url)
+      url = URI(url.to_s)
+      page = Freebase.open_page(url)
+      new_item = Item.new(url)
+
+      content = page.at('#content_main')
+
+      new_item.name = content.at('#title//h1').inner_text.strip
+      new_item.summary = content.at('#title/div.article-container/div.article').inner_html.strip
+
+      extract_value = lambda { |elem|
+        if (item_link = elem.at('a.pv'))
+          link_url = new_item.url.merge(item_link['href'])
+
+          ItemLink.new(item_link.inner_text.strip, link_url)
+        elsif elem.at('a.detail-view').nil?
+          text = elem.inner_text.strip
+
+          if text.empty?
+            nil
+          else
+            text
+          end
+        end
+      }
+
+      content.search('div.domainsboxes//div.domainbox//div.typebox-container') do |domainbox|
+        category_name = domainbox.at('//div.typebox-column-title/a').inner_text
+        new_category = Category.new(category_name)
+
+        domainbox.search('//div.prop-typebox') do |field|
+          field_name = field.at('//span.prop-title').inner_text
+          field_content = field.at('//div.prop-content')
+
+          field_value = nil
+
+          if (table = field_content.at('table.prop-table'))
+            field_value = []
+
+            column_names = table.search('tr/th/div.prop-table-cell').map do |div|
+              div.inner_text.strip
+            end
+
+            table.search('tr[td]') do |row|
+              field_row = {}
+              index = 0
+
+              row.search('td') do |cell|
+                if (value = extract_value.call(cell))
+                  field_row[column_names[index]] = value
+                end
+
+                index += 1
+              end
+
+              field_value << field_row unless field_row.empty?
+            end
+          elsif (list = field_content.at('ul.prop-list'))
+            field_value = []
+
+            list.search('li.prop-list-item') do |list_item|
+              if (value = extract_value.call(list_item))
+                field_value << value
+              end
+            end
+          end
+
+          new_category.metadata[field_name] = field_value
+        end
+
+        new_item.categories[new_category.name] = new_category
+      end
+
+      return new_item
+    end
+
+    #
+    # Returns the category names of the item.
+    #
+    def category_names
+      @categories.keys
+    end
+
+    #
+    # Returns the Category with the specified _name_ of the item.
+    #
+    def [](name)
+      @categories[name]
+    end
+
+    #
+    # Returns the name of the item.
+    #
+    def to_s
+      @name.to_s
+    end
+
+  end
+end

data/lib/freebase/item_link.rb

@@ -0,0 +1,26 @@
+module Freebase
+  class ItemLink
+
+    # Title of the item
+    attr_reader :title
+
+    # URL of the item
+    attr_reader :url
+
+    #
+    # Creates a new ItemLink with the specified _title_ and _url_.
+    #
+    def initialize(title,url)
+      @title = title
+      @url = url
+    end
+
+    #
+    # Returns the title of the item-link in +String+ form.
+    #
+    def to_s
+      @title.to_s
+    end
+
+  end
+end

data/lib/freebase/version.rb

@@ -0,0 +1,3 @@
+module Freebase
+  VERSION = '0.0.9'
+end

data/spec/freebase_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+
+describe Freebase do
+  it "should have a version" do
+    Freebase.const_get('VERSION').should_not be_nil
+  end
+
+  it "should have a default language" do
+    Freebase.language.should_not be_nil
+  end
+
+  it "should have a default User-Agent string" do
+    Freebase.user_agent.should_not be_nil
+  end
+
+  it "should return an item from a given URL" do
+    @item = Freebase.item('http://www.freebase.com/view/en/squarepusher/')
+    @item.should_not be_nil
+    @item.name.should == 'Squarepusher'
+  end
+
+  it "should return an item from a given GUID" do
+    @item = Freebase.item('9202a8c04000641f8000000000184c7a')
+    @item.should_not be_nil
+    @item.name.should == 'Conflict'
+  end
+
+  it "should return an item for a given title" do
+    @item = Freebase.item('Aphex Twin')
+    @item.should_not be_nil
+    @item.name.should == 'Aphex Twin'
+  end
+end

data/spec/item_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+describe Item do
+  before(:all) do
+    @item = Item.from_url('http://www.freebase.com/view/en/aphex_twin')
+  end
+
+  it "should have a URL" do
+    @item.url.should_not be_nil
+  end
+
+  it "should have a name" do
+    @item.name.should_not be_nil
+    @item.name.should_not be_empty
+  end
+
+  it "should have categories" do
+    @item.categories.should_not be_empty
+  end
+
+  it "should have category names" do
+    @item.category_names.should_not be_empty
+  end
+
+  it "should have metadata for each category" do
+    @item.categories.each_value do |category|
+      category.metadata.should_not be_empty
+
+      category.metadata.each_value do |data|
+        data.should_not be_nil
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+gem 'rspec', '>=1.1.3'
+require 'spec'
+
+require 'freebase'
+
+include Freebase

data/tasks/spec.rb

@@ -0,0 +1,7 @@
+require 'spec/rake/spectask'
+
+desc "Run all specifications"
+Spec::Rake::SpecTask.new(:spec) do |t|
+  t.libs += ['lib', 'spec']
+  t.spec_opts = ['--colour', '--format', 'specdoc']
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification 
+name: freescrape
+version: !ruby/object:Gem::Version 
+  version: 0.0.9
+platform: ruby
+authors: 
+- Postmodern Modulus III
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-09-29 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: mechanize
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.7.0
+    version: 
+description: A web-scraping interface to freebase.com, the open and shared database of the world's knowledge.
+email: 
+- postmodern.mod3@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- README.txt
+files: 
+- History.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- lib/freebase.rb
+- lib/freebase/item_link.rb
+- lib/freebase/category.rb
+- lib/freebase/item.rb
+- lib/freebase/freebase.rb
+- lib/freebase/version.rb
+- tasks/spec.rb
+- spec/item_spec.rb
+- spec/freebase_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://freebase.rubyforge.org/
+post_install_message: 
+rdoc_options: 
+- --main
+- README.txt
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: freebase
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: A web-scraping interface to freebase.com, the open and shared database of the world's knowledge.
+test_files: []
+