semantic_extraction 0.1.1 → 0.2.0

data/README.rdoc

@@ -4,22 +4,26 @@ Extract meaningful information from unstructured text with Ruby.
 
 Using a variety of APIs (Yahoo Term Extractor and Alchemy are currently supported), semantic_extraction can automatically return a collection of keywords for an arbitrary block of text. If you use Alchemy, it can also return named entities.
 
+A brief walkthrough:
+
+  $ require 'rubygems'
+  $ require 'semantic_extraction'
+  $ SemanticExtraction.alchemy_api_key = 'YOUR_API_KEY_HERE'
+  $ SemanticExtraction.find_keywords("http://chrisvannoy.com/2010/03/10/introducing_semantic_extraction/")
+  $ ["Knight News Challenge", "Yahoo Term Extractor", "obscure gem", "soon-to-be twitter employee", "handle serving gems", "API providers", "Alchemy API", "unstructured text", "earliest steps", "Rails 3-compatible version", "structured data", "early stage", "death threats", "github", "final aside", "awesome piece", "default choice", "HTTP communication", "Indianapolis Star", "Feel free"]
+
 == The APIs in use
 
-* [Yahoo Term Extractor](http://developer.yahoo.com/search/content/V1/termExtraction.html)
+* {Yahoo Term Extractor}[http://developer.yahoo.com/search/content/V1/termExtraction.html]
 
-* [Alchemy API](http://www.alchemyapi.com/api/)
+* {Alchemy API}[http://www.alchemyapi.com/api/]
 
 == Upcoming To-Dos
 
-* Add support for [OpenCalais](http://www.opencalais.com/documentation/opencalais-documentation)
+* Add support for {OpenCalais}[http://www.opencalais.com/documentation/opencalais-documentation]
 
 * Flesh out the rest of the Alchemy API
 
-* Make it possible to dynamically pick with API to use (so its possible to use multiple APIs in the same app)
-
-* Make it less fugly.
-
 * Tests, tests and more tests.
 
 == Note on Patches/Pull Requests

data/Rakefile

@@ -10,7 +10,8 @@ begin
     gem.email = "chris@chrisvannoy.com"
     gem.homepage = "http://github.com/dummied/semantic_extraction"
     gem.authors = ["Chris Vannoy"]
-    gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
+    gem.add_development_dependency "shoulda", ">= 0"
+    gem.add_development_dependency "fakeweb"
     gem.add_dependency "ruby_tubesday"
     gem.add_dependency "nokogiri"
     gem.add_dependency "extlib"

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.2.0

data/lib/semantic_extraction.rb

@@ -2,10 +2,35 @@ require 'ruby_tubesday'
 require 'nokogiri'
 require 'extlib'
 require 'ostruct'
+require 'active_support/core_ext/module/attribute_accessors'
 
 module SemanticExtraction
   
+
+  mattr_accessor :preferred_extractor
+  mattr_accessor :alchemy_api_key
+  mattr_accessor :yahoo_api_key
+  mattr_accessor :valid_extractors
+  mattr_accessor :requires_api_key
+  
+  self.valid_extractors = ["yahoo", "alchemy"]
+  self.requires_api_key = ["yahoo", "alchemy"]
+  
+  # By default, we assume you want to use Alchemy.
+  # To override, just set SemanticExtraction.preferred_extractor somewhere and define the appropriate api_key.
+  def self.preferred_extractor=(value)
+    if self.valid_extractors.include?(value)
+      @@preferred_extractor = value
+    else
+      raise NotSupportedExtractor
+    end
+  end
+  
+  self.preferred_extractor = "alchemy" if self.preferred_extractor.blank?
+  
+  
   # Screw it. Hard-code time!
+  require 'semantic_extraction/utility_methods'
   require 'semantic_extraction/extractors/yahoo'
   require 'semantic_extraction/extractors/alchemy'
   
@@ -16,53 +41,30 @@ module SemanticExtraction
   # This will become more important when we start mapping out all of the other features in the Alchemy API
   class NotSupportedExtraction < StandardError; end
   
-  # By default, we assume you want to use Alchemy.
-  # To override, just set SemanticExtraction::PREFERRED_EXTRACTOR somewhere.
-  def self.preferred_extractor
-    defined?(PREFERRED_EXTRACTOR) ? PREFERRED_EXTRACTOR : "alchemy"
-  end
+  # Thrown when you attempt to set the preferred extractor to an extractor we don't yet support.
+  class NotSupportedExtractor < StandardError; end
   
-  HTTP = RubyTubesday.new
-  
-  # Will return an array of keywords gleaned from the text you pass in.
-  # Both Yahoo and Alchemy will handle a block of text, but Alchemy can also handle a plain URL.  
-  def self.find_keywords(text)
-    klass = SemanticExtraction.const_get(self.preferred_extractor.capitalize)
-    if klass.respond_to?(:find_keywords) && defined?(self.preferred_extractor.upcase + "_API_KEY")
-      return klass.find_keywords(text)
-    elsif !klass.respond_to?(:find_keywords)
+  def self.find_generic(typer, args)
+    if self.is_valid?(typer)
+      return @@klass.send(typer.to_sym, args)
+    elsif !@@klass.respond_to?(typer.to_sym)
       raise NotSupportedExtraction
     else
       raise MissingApiKey
     end
   end
   
-  # Will return an array of OpenStruct representing the named entities from the text.
-  # At the moment, Alchemy is the only one to support this.
-  # Down the road, we'll add in OpenCalais and others.
-  def self.find_entities(text)
-    klass = SemanticExtraction.const_get(self.preferred_extractor.capitalize)
-    if klass.respond_to?(:find_entities) && defined?(self.preferred_extractor.upcase + "_API_Key")
-      return klass.find_entities(text)
-    elsif !klass.respond_to?(:find_entities)
-      raise NotSupportedExtraction
-    else
-      raise MissingApiKey
-    end
+  def self.method_missing(method, args)
+    find_generic(method.to_sym, args)
   end
   
-  # Posts the url to the API.
-  def self.post(url, target, calling_param, api_key, api_param="apikey".to_sym)
-    HTTP.post(url, :params => {calling_param => target, api_param => api_key} )
-  end
   
-  # Checks to see if a string is a URL.
-  # This is really dumb at the moment, and will likely be refactored in future releases.
-  def self.is_url?(link)
-    if link[0..3] == "http"
-      return true
+  def self.is_valid?(method)
+    @@klass = SemanticExtraction.const_get(self.preferred_extractor.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase })
+    if self.requires_api_key.include? self.preferred_extractor
+      (@@klass.respond_to?(method) && defined?(self.send((preferred_extractor + "_api_key").to_sym)) && !(self.send((preferred_extractor + "_api_key").to_sym)).empty?) ? true : false
     else
-      return false
+      @@klass.respond_to?(method)
     end
   end
   

data/lib/semantic_extraction/extractors/alchemy.rb

@@ -1,41 +1,58 @@
 module SemanticExtraction
-  class Alchemy
+  module Alchemy
+    include SemanticExtraction::UtilityMethods
     STARTER = "http://access.alchemyapi.com/calls/"
         
     def self.find_keywords(text)
-      prefix = (SemanticExtraction.is_url?(text) ? "url" : "text")
+      prefix = is_url?(text) ? "url" : "text"
       endpoint = (prefix == "url" ? "URL" : "Text") + "GetKeywords"
       url = STARTER + prefix + "/" + endpoint
-      raw = SemanticExtraction.post(url, text, prefix, SemanticExtraction::ALCHEMY_API_KEY)
+      raw = post(url, text, prefix)
+      output_keywords(raw)
+    end
+    
+    def self.output_keywords(raw)
       h = Nokogiri::XML(raw)
-      if (h/"keywords keyword")
-        keywords = []
-        (h/"keywords keyword").each do |p|
-          keywords << p.text
-        end
+      keywords = []
+      (h/"keywords keyword").each do |p|
+        keywords << p.text
       end
       return keywords
     end
     
     def self.find_entities(text)
-      prefix = (SemanticExtraction.is_url?(text) ? "url" : "text")
+      prefix = is_url?(text) ? "url" : "text"
       endpoint = (prefix == "url" ? "URL" : "Text") + "GetRankedNamedEntities"
       url = STARTER + prefix + "/" + endpoint
-      raw = SemanticExtraction.post(url, text, prefix, SemanticExtraction::ALCHEMY_API_KEY)
+      raw = post(url, text, prefix)
+      output_entities(raw)
+    end
+    
+    def self.output_entities(raw)
       h = Nokogiri::XML(raw)
-      if (h/"entities entity")
-        entities = []
-        (h/"entities entity").each do |p|
-          hashie = Hash.from_xml(p.to_s)["entity"]
-          typer = hashie.delete("type")
-          if typer
-            hashie["entity_type"] = typer
-          end
-          entities << OpenStruct.new(hashie)
+      entities = []
+      (h/"entities entity").each do |p|
+        hashie = Hash.from_xml(p.to_s)["entity"]
+        typer = hashie.delete("type")
+        if typer
+          hashie["entity_type"] = typer
         end
+        entities << OpenStruct.new(hashie)
       end
       return entities
     end
-
+    
+    def self.extract_text(text)
+      prefix = is_url?(text) ? "url" : "html"
+      endpoint = (prefix == "url" ? "URL" : "HTML") + "GetText"
+      url = STARTER + prefix + "/" + endpoint
+      raw = post(url, text, prefix)      
+      output_text(raw)
+    end
+    
+    def self.output_text(raw)
+      h = Nokogiri::XML(raw)
+      return (h/"text").first.inner_html
+    end
   end
 end

data/lib/semantic_extraction/extractors/yahoo.rb

@@ -1,16 +1,19 @@
 module SemanticExtraction
-  class Yahoo
+  module Yahoo
+    include SemanticExtraction::UtilityMethods
     STARTER = "http://api.search.yahoo.com/ContentAnalysisService/V1/termExtraction"
-        
+      
     def self.find_keywords(text)
       prefix = 'context'
-      raw = SemanticExtraction.post(STARTER, text, prefix, SemanticExtraction::YAHOO_API_KEY, :appid)
+      raw = SemanticExtraction.post(STARTER, text, prefix, :appid)
+      self.output_keywords(raw)
+    end
+  
+    def self.output_keywords(raw)
       h = Nokogiri::XML(raw)
-      if (h/"Result")
-        keywords = []
-        (h/"Result").each do |p|
-          keywords << p.text
-        end
+      keywords = []
+      (h/"Result").each do |p|
+        keywords << p.text
       end
       return keywords
     end

data/lib/semantic_extraction/utility_methods.rb

@@ -0,0 +1,26 @@
+module SemanticExtraction
+  module UtilityMethods
+    
+    def self.included(includer)
+      includer.module_eval do
+
+        # Posts the url to the API.
+        def self.post(url, target, calling_param, api_param="apikey".to_sym)
+          RubyTubesday.new.post(url, :params => {calling_param => target, api_param => (SemanticExtraction.send((SemanticExtraction.preferred_extractor + "_api_key").to_sym))} )
+        end
+
+        # Checks to see if a string is a URL.
+        # This is really dumb at the moment, and will likely be refactored in future releases.
+        def self.is_url?(link)
+          if link[0..3] == "http"
+            return true
+          else
+            return false
+          end
+        end
+      end
+    end
+    
+    
+  end
+end

data/semantic_extraction.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{semantic_extraction}
-  s.version = "0.1.1"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Chris Vannoy"]
-  s.date = %q{2010-03-10}
+  s.date = %q{2010-07-06}
   s.description = %q{Using a variety of APIs (Yahoo term Extractor and Alchemy are currently supported), semantic_extraction can automatically return a collection of keywords for an arbitrary block of text. If using Alchemy, it can also return named entities.}
   s.email = %q{chris@chrisvannoy.com}
   s.extra_rdoc_files = [
@@ -26,17 +26,22 @@ Gem::Specification.new do |s|
      "lib/semantic_extraction.rb",
      "lib/semantic_extraction/extractors/alchemy.rb",
      "lib/semantic_extraction/extractors/yahoo.rb",
+     "lib/semantic_extraction/utility_methods.rb",
      "semantic_extraction.gemspec",
+     "test/api_extractor.rb",
      "test/helper.rb",
+     "test/our_extractor.rb",
      "test/test_semantic_extraction.rb"
   ]
   s.homepage = %q{http://github.com/dummied/semantic_extraction}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.6}
+  s.rubygems_version = %q{1.3.7}
   s.summary = %q{Extract meaningful information from unstructured text with Ruby}
   s.test_files = [
-    "test/helper.rb",
+    "test/api_extractor.rb",
+     "test/helper.rb",
+     "test/our_extractor.rb",
      "test/test_semantic_extraction.rb"
   ]
 
@@ -44,19 +49,22 @@ Gem::Specification.new do |s|
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
-    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<fakeweb>, [">= 0"])
       s.add_runtime_dependency(%q<ruby_tubesday>, [">= 0"])
       s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
       s.add_runtime_dependency(%q<extlib>, [">= 0"])
     else
-      s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<fakeweb>, [">= 0"])
       s.add_dependency(%q<ruby_tubesday>, [">= 0"])
       s.add_dependency(%q<nokogiri>, [">= 0"])
       s.add_dependency(%q<extlib>, [">= 0"])
     end
   else
-    s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<fakeweb>, [">= 0"])
     s.add_dependency(%q<ruby_tubesday>, [">= 0"])
     s.add_dependency(%q<nokogiri>, [">= 0"])
     s.add_dependency(%q<extlib>, [">= 0"])

data/test/api_extractor.rb

@@ -0,0 +1,17 @@
+module SemanticExtraction
+  module ApiExtractor
+    include SemanticExtraction::UtilityMethods
+    
+    SemanticExtraction.valid_extractors << "api_extractor"
+    SemanticExtraction.requires_api_key << "api_extractor"
+    
+    SemanticExtraction.module_eval("mattr_accessor :api_extractor_api_key")
+    
+    SemanticExtraction.api_extractor_api_key = "bogus"
+    
+    def self.find_keywords(text)
+      return []
+    end
+      
+  end
+end

data/test/our_extractor.rb

@@ -0,0 +1,12 @@
+module SemanticExtraction
+  module OurExtractor
+    include SemanticExtraction::UtilityMethods
+    
+    SemanticExtraction.valid_extractors << "our_extractor"
+    
+    def self.find_keywords(text)
+      return []
+    end
+      
+  end
+end

data/test/test_semantic_extraction.rb

@@ -1,7 +1,30 @@
 require 'helper'
+require 'our_extractor'
+require 'api_extractor'
 
 class TestSemanticExtraction < Test::Unit::TestCase
-  should "probably rename this file and start testing for real" do
-    flunk "hey buddy, you should probably rename this file and start testing for real"
+  should "correctly identify a url in is_url?" do
+    assert_equal SemanticExtraction.is_url?("http://www.indystar.com"), true
+    assert_equal SemanticExtraction.is_url?("I am a cheeky monkey"), false
   end
+  
+  should "throw error when trying to set an invalid extractor" do
+    begin
+      SemanticExtraction.preferred_extractor = "bullshit"
+    rescue StandardError => err
+      assert_equal err.class.to_s, "SemanticExtraction::NotSupportedExtractor"
+    end
+  end
+  
+  should "be able to define new extractors without api keys" do
+    SemanticExtraction.preferred_extractor = "our_extractor"
+    assert_equal true, SemanticExtraction.is_valid?(:find_keywords)
+  end
+  
+  should "be able to define new extractors with api keys" do
+    SemanticExtraction.preferred_extractor = "api_extractor"
+    assert_equal true, SemanticExtraction.requires_api_key.include?("api_extractor")
+    assert_equal true, SemanticExtraction.is_valid?(:find_keywords)
+  end
+  
 end

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: semantic_extraction
 version: !ruby/object:Gem::Version 
+  hash: 23
   prerelease: false
   segments: 
   - 0
-  - 1
-  - 1
-  version: 0.1.1
+  - 2
+  - 0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Chris Vannoy
@@ -14,57 +15,79 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-03-10 00:00:00 -05:00
+date: 2010-07-06 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: thoughtbot-shoulda
+  name: shoulda
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
   type: :development
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: ruby_tubesday
+  name: fakeweb
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
-  type: :runtime
+  type: :development
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: nokogiri
+  name: ruby_tubesday
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: extlib
+  name: nokogiri
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
   type: :runtime
   version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: extlib
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id005
 description: Using a variety of APIs (Yahoo term Extractor and Alchemy are currently supported), semantic_extraction can automatically return a collection of keywords for an arbitrary block of text. If using Alchemy, it can also return named entities.
 email: chris@chrisvannoy.com
 executables: []
@@ -84,8 +107,11 @@ files:
 - lib/semantic_extraction.rb
 - lib/semantic_extraction/extractors/alchemy.rb
 - lib/semantic_extraction/extractors/yahoo.rb
+- lib/semantic_extraction/utility_methods.rb
 - semantic_extraction.gemspec
+- test/api_extractor.rb
 - test/helper.rb
+- test/our_extractor.rb
 - test/test_semantic_extraction.rb
 has_rdoc: true
 homepage: http://github.com/dummied/semantic_extraction
@@ -97,26 +123,32 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Extract meaningful information from unstructured text with Ruby
 test_files: 
+- test/api_extractor.rb
 - test/helper.rb
+- test/our_extractor.rb
 - test/test_semantic_extraction.rb