sem_extractor 0.0.1 → 0.0.2

data/README.rdoc

@@ -1,18 +1,30 @@
-= sem_extractor
+== SemExtractor
+
 SemExtractor is made to have in a single place, wrappers for most of the semantic librairies:
 - Zemanta
 - Semantic Hacker from Textwise
 - Yahoo Boss
+- OpenCalais
+
+= H2 Please tell me if there are more API's to include!
+
+<em>After using Term Extraction gem, I happened to need the score of the different tags I got from the different APIS + I wanted to use Nokogiri for performance concerns.
+I thank alexrabarts, because his work gave me the idea to create my first gem</em>
 
-Please tell me if there are more API's to include!
+== Installation
 
-After using Term Extraction gem, I happened to need the score of the different tags I got from the different APIS + I wanted to use Nokogiri for performance concerns.
+To install (I strongly recommend you use RVM):
+gem install sem_extractor
+
+
+== Usage
 Most of the methods below retrieve a hash with 'name' and 'score'
 
 Initialize:
 - yahoo = SemExtractor::Yahoo.new(:api_key => your_key, :context => your_text)
 - zemanta = SemExtractor::Zemanta.new(:api_key => your_key, :context => your_text)
 - sem = SemExtractor::Textwise.new(:api_key => your_key, :context => your_text_or_url)
+- calais = SemExtractor::Calais.new(:api_key => CALAIS, :context => REQUEST)
 
 Get info:
 - yahoo.terms
@@ -21,6 +33,9 @@ Get info:
 - sem.terms
 - sem.categories
 - sem.filter  #filters the useful content of a web page, retrieves text
+- calais.terms
+- calais.categories
+- calais.geos
 
 == Note on Patches/Pull Requests
  
@@ -32,6 +47,9 @@ Get info:
   (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
 * Send me a pull request. Bonus points for topic branches.
 
+== Todo
+I've to create the tests for this gem... :)
+
 == Copyright
 
 Copyright (c) 2010 apneadiving. See LICENSE for details.

data/Rakefile

@@ -5,8 +5,8 @@ begin
   require 'jeweler'
   Jeweler::Tasks.new do |gem|
     gem.name = "sem_extractor"
-    gem.summary = %Q{Extracts data from semantics API like zemanta, textwise and yahoo}
-    gem.description = %Q{Extracts data from semantics API like zemanta, textwise and yahoo}
+    gem.summary = %Q{Extracts data from semantics APIs: zemanta, textwise, opencalais and yahoo}
+    gem.description = %Q{Extracts data from semantics APIs: zemanta, textwise, opencalais and yahoo}
     gem.email = "apnea.diving.deep@gmail.com"
     gem.homepage = "http://github.com/apneadiving/sem_extractor"
     gem.authors = ["apneadiving"]

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.0.2

data/lib/apis/calais.rb

@@ -0,0 +1,49 @@
+class SemExtractor
+  class Calais < SemExtractor
+    
+     def initialize(options={})
+       self.set(options)
+        Nokogiri::XML(remote_xml).xpath('//rdf:Description').map { |h|  
+          node_type = h.xpath('rdf:type').first['resource']
+          if node_type.include?('/type/cat/')
+            @categories << { "name" => h.xpath('c:categoryName').first.content, "score"=>h.xpath('c:score').first.content}
+          elsif node_type.include?('/type/em/')
+            nationality = h.xpath('c:nationality').first.nil? ? 'N/A' : h.xpath('c:nationality').first.content
+            @terms << { "name" => h.xpath('c:name').first.content, "score" => nil, "nationality" => nationality }
+          elsif node_type.include?('/type/sys/InstanceInfo')
+            #nothing to do, no info to take
+          elsif node_type.include?('/type/sys/RelevanceInfo')
+            # I assume here, Open Calais will keep on giving information in the proper order, seems fair :)
+            @terms.last["score"] = h.xpath('c:relevance').first.content
+          elsif node_type.include?('/Geo/')
+            @geos <<{ "name" => h.xpath('c:name').first.content } 
+          end
+          }
+      end
+
+     def uri
+       URI.parse(gateway + '?' + URI.escape(post_params.collect{ |k, v| "#{k}=#{v}" }.join('&')))
+     end
+
+    def post_params
+      {
+        'licenseID' => @api_key,
+        'content'   => @context
+      }
+    end
+
+  private
+      def gateway
+        'http://api.opencalais.com/enlighten/rest/'
+      end
+
+      def remote_xml
+        begin
+          Net::HTTP.get_response((uri)).body
+        rescue => e
+          $stderr.puts "Couldn't fetch from API: #{e.message}" if $VERBOSE
+          nil
+        end
+      end
+  end
+end

data/lib/apis/textwise.rb

@@ -1,9 +1,11 @@
-require 'nokogiri'
-require 'open-uri'
-require 'uri'
-
 class SemExtractor
   class Textwise < SemExtractor
+    #I kept here methods instead of variables because,each one leads to a http call
+   
+    def initialize(options={})
+      self.set(options)
+    end
+    
     def terms
       @options = { 'content' => @context }
       get_entity
@@ -24,7 +26,6 @@ class SemExtractor
     def match
       @type = 'match/rsscombined'
       @options = {'content' => @context }
-      puts remote_xml
     end
     
     def get_entity

data/lib/apis/yahoo.rb

@@ -1,15 +1,8 @@
-require 'nokogiri'
-require 'open-uri'
-require 'uri'
-
 class SemExtractor
  class Yahoo < SemExtractor
-    def terms
-      begin
-        Nokogiri::XML(remote_xml).css('Result').map { |h| {"name" => h.content} }
-      rescue
-        []
-      end
+    def initialize(options={})
+     self.set(options)
+     @terms = Nokogiri::XML(remote_xml).css('Result').map { |h| {"name" => h.content} }
     end
 
     def uri

data/lib/apis/zemanta.rb

@@ -1,24 +1,13 @@
-require 'nokogiri'
-require 'net/http'
-require 'uri'
-
 class SemExtractor
   class Zemanta < SemExtractor
     
-    def terms
-      begin
-        @categories = Nokogiri::XML(remote_xml).css('category').map { |h|  {"score" => h.css('confidence').first.content, "name" => h.css('name').first.content} }
-        Nokogiri::XML(remote_xml).css('keyword').map { |h|  {"score" => h.css('confidence').first.content, "name" => h.css('name').first.content} }
-      rescue
-        []
-      end
+    def initialize(options={})
+      self.set(options)
+      xml = remote_xml
+      @categories = Nokogiri::XML(xml).css('category').map { |h|  {"score" => h.css('confidence').first.content, "name" => h.css('name').first.content} }
+      @terms = Nokogiri::XML(xml).css('keyword').map { |h|  {"score" => h.css('confidence').first.content, "name" => h.css('name').first.content} }
     end
     
-    def categories
-      terms if @categories == nil
-      return @categories
-    end
-
     def uri
       URI.parse(gateway)
     end

data/lib/sem_extractor.rb

@@ -1,13 +1,18 @@
+require 'nokogiri'
+require 'uri'
+require 'net/http'
+require 'open-uri'
+
 class SemExtractor
-  attr_accessor :context, :api_key, :categories
+  attr_accessor :context, :api_key, :categories, :terms, :geos
 
-  def initialize(options={})
+  def set(options={})
     @context = options[:context]
     @api_key = options[:api_key]
     @type = options[:type]
-    @categories = nil
+    @categories, @terms, @geos, @relations = Array.new, Array.new, Array.new, Array.new
   end
 
 end
 
-%w{yahoo zemanta textwise}.each{|t| require "apis/#{t}"}
+%w{yahoo zemanta textwise calais}.each{|t| require "apis/#{t}"}

data/sem_extractor.gemspec

@@ -5,12 +5,12 @@
 
 Gem::Specification.new do |s|
   s.name = %q{sem_extractor}
-  s.version = "0.0.1"
+  s.version = "0.0.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["apneadiving"]
   s.date = %q{2010-10-02}
-  s.description = %q{Extracts data from semantics API like zemanta, textwise and yahoo}
+  s.description = %q{Extracts data from semantics APIs: zemanta, textwise, opencalais and yahoo}
   s.email = %q{apnea.diving.deep@gmail.com}
   s.extra_rdoc_files = [
     "LICENSE",
@@ -22,11 +22,13 @@ Gem::Specification.new do |s|
      "Rakefile",
      "VERSION",
      "lib/.DS_Store",
+     "lib/apis/calais.rb",
      "lib/apis/textwise.rb",
      "lib/apis/yahoo.rb",
      "lib/apis/zemanta.rb",
      "lib/sem_extractor.rb",
      "pkg/sem_extractor-0.0.0.gem",
+     "pkg/sem_extractor-0.0.1.gem",
      "sem_extractor.gemspec",
      "test/helper.rb",
      "test/test_sem_extractor.rb"
@@ -35,7 +37,7 @@ Gem::Specification.new do |s|
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
   s.rubygems_version = %q{1.3.7}
-  s.summary = %q{Extracts data from semantics API like zemanta, textwise and yahoo}
+  s.summary = %q{Extracts data from semantics APIs: zemanta, textwise, opencalais and yahoo}
   s.test_files = [
     "test/helper.rb",
      "test/test_sem_extractor.rb"

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: sem_extractor
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 27
   prerelease: false
   segments: 
   - 0
   - 0
-  - 1
-  version: 0.0.1
+  - 2
+  version: 0.0.2
 platform: ruby
 authors: 
 - apneadiving
@@ -32,7 +32,7 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id001
-description: Extracts data from semantics API like zemanta, textwise and yahoo
+description: "Extracts data from semantics APIs: zemanta, textwise, opencalais and yahoo"
 email: apnea.diving.deep@gmail.com
 executables: []
 
@@ -47,11 +47,13 @@ files:
 - Rakefile
 - VERSION
 - lib/.DS_Store
+- lib/apis/calais.rb
 - lib/apis/textwise.rb
 - lib/apis/yahoo.rb
 - lib/apis/zemanta.rb
 - lib/sem_extractor.rb
 - pkg/sem_extractor-0.0.0.gem
+- pkg/sem_extractor-0.0.1.gem
 - sem_extractor.gemspec
 - test/helper.rb
 - test/test_sem_extractor.rb
@@ -88,7 +90,7 @@ rubyforge_project:
 rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
-summary: Extracts data from semantics API like zemanta, textwise and yahoo
+summary: "Extracts data from semantics APIs: zemanta, textwise, opencalais and yahoo"
 test_files: 
 - test/helper.rb
 - test/test_sem_extractor.rb