calais 0.0.5 → 0.0.6

data/History.txt

@@ -1,3 +1,6 @@
+== 0.0.6
+* fully implemented 3.1 API
+
 == 0.0.5
 
 * fixed error where classes weren't being required in the proper order on Ubuntu (reported by Jon Moses)

data/Manifest.txt

@@ -6,12 +6,14 @@ Rakefile
 init.rb
 lib/calais.rb
 lib/calais/client.rb
-lib/calais/name.rb
-lib/calais/relationship.rb
 lib/calais/response.rb
+spec/calais/client_spec.rb
+spec/calais/response_spec.rb
 spec/calais_spec.rb
 spec/fixtures/.gitignore
-spec/fixtures/bicycles_austrailia.xml
+spec/fixtures/bicycles_australia.response.json
+spec/fixtures/bicycles_australia.response.rdf
+spec/fixtures/bicycles_australia.xml
 spec/fixtures/calais.yml.sample
 spec/fixtures/slovenia_euro.xml
 spec/helper.rb

data/README.txt

@@ -6,8 +6,8 @@ A Ruby interface to the Open Calais Web Service (http://opencalais.com)
 * Accepts documents in text/plain, text/xml and text/html format.
 * Basic access to the Open Calais API's Enlighten action.
 	* Output is RDF representation of input document.
-* Single function ability to tag a document and receive a response in RDF format, names in the document, and their relationships.
-
+* Single function ability to extract names, entities and geographies from given text.
+  
 == Synopsis
 
 This is a very basic wrapper to the Open Calais API. It uses the POST endpoint and currently supports the Enlighten action. Here's a simple call:
@@ -17,16 +17,17 @@ This is a very basic wrapper to the Open Calais API. It uses the POST endpoint a
 This is the easiest way to get the RDF-formated response from the OpenCalais service.
 
 If you want to do something more fun like getting all sorts of fun information about a document, you can try this:
-
+  
   Calais.process_document(:content => "The government of the United Kingdom has given corporations like fast food chain McDonald's the right to award high school qualifications to employees who complete a company training program.", :content_type => :text, :license_id => LICENSE_ID)
 
-This will return an object containing the RDF representation of the text, the names in the text, and any relationships that exist there.
+This will return an object containing information extracted from the RDF response.
 
 == Requirements
 
 * Ruby 1.8.5 or better
   * Uses the following standard libraries: digest/sha1, net/http, yaml, cgi
 * libxml2 / libxml-ruby
+* curb, libcurl
 
 == Install
 
@@ -37,3 +38,6 @@ You can install the Calais gem via Rubygems (gem install calais) or by building
 Abhay Kumar
 http://opensynapse.net
 
+== Acknowledgements
+
+* Paul Legato (http://www.economaton.com/). Help all around with the new response processor and implementation of the latest API.

data/Rakefile

@@ -19,7 +19,7 @@ end
 
 desc "Run all specs"
 Spec::Rake::SpecTask.new do |t|
-  t.spec_files = FileList["spec/*_spec.rb"].sort
+  t.spec_files = FileList["spec/**/*_spec.rb"].sort
   t.spec_opts = ["--options", "spec/spec.opts"]
 end
 

data/lib/calais.rb

@@ -2,55 +2,58 @@ require 'digest/sha1'
 require 'net/http'
 require 'cgi'
 require 'iconv'
+require 'set'
 
 require 'rubygems'
 require 'xml/libxml'
+require 'json'
+require 'curb'
 
 $KCODE = "UTF8"
 require 'jcode'
 
 $:.unshift File.expand_path(File.dirname(__FILE__)) + '/calais'
 
-require 'name'
-require 'relationship'
-require 'response'
 require 'client'
+require 'response'
 
 module Calais
-  POST_URL = "http://api.opencalais.com"
-  
-  AVAILABLE_OUTPUT_FORMATS = {
-    :rdf => "XML/RDF"
-  }
-  DEFAULT_OUTPUT_FORMAT = :rdf
-  
+  REST_ENDPOINT = "http://api.opencalais.com/enlighten/rest/"
+  BETA_REST_ENDPOINT = "http://beta.opencalais.com/enlighten/rest/"
+
   AVAILABLE_CONTENT_TYPES = {
-    :xml => "TEXT/XML",
-    :html => "TEXT/HTML",
-    :text => "TEXT/TXT"
+    :xml => 'text/xml',
+    :text => 'text/txt',
+    :html => 'text/html',
+    :raw => 'text/raw'
   }
-  DEFAULT_CONTENT_TYPE = :xml
-  
-  DEFAULT_SUBMITTER = "calais.rb"
-  
-  AVAILABLE_METHODS = {
-    :enlighten => "/enlighten/calais.asmx/Enlighten"
+
+  AVAILABLE_OUTPUT_FORMATS = {
+    :rdf => 'xml/rdf',
+    :simple => 'text/simple',
+    :microformats => 'text/microformats',
+    :json => 'application/json'
   }
-  
+
+  KNOWN_ENABLES = ['GenericRelations']
+  KNOWN_DISCARDS = ['er/Company', 'er/Geo']
+
   MAX_RETRIES = 5
-  
+  HTTP_TIMEOUT = 60
+  MIN_CONTENT_SIZE = 100
+  MAX_CONTENT_SIZE = 100_000
+
   class << self
-    def enlighten(*args, &block) Client.new(*args, &block).call(:enlighten) end
+    def enlighten(*args, &block); Client.new(*args, &block).enlighten; end
 
-    def process_document(*args, &block) 
-      data, error = Calais.enlighten(*args, &block)
-      process_data(data, error)
+    def process_document(*args, &block)
+      client = Client.new(*args, &block)
+      client.output_format = :rdf
+      Response.new(client.enlighten)
     end
-
-    def process_data(data, error=nil) Response.new(data, error) end
   end
 end
 
 module Calais
-  VERSION = '0.0.5'
-end
+  VERSION = '0.0.6'
+end

data/lib/calais/client.rb

@@ -1,47 +1,110 @@
 module Calais
   class Client
-    attr_accessor :license_id
+    # base attributes of the call
     attr_accessor :content
-    attr_accessor :content_type, :output_format
-    attr_accessor :allow_distribution, :allow_search, :submitter, :external_id
+    attr_accessor :license_id
+
+    # processing directives
+    attr_accessor :content_type, :output_format, :reltag_base_url, :calculate_relevance, :omit_outputting_original_text
+    attr_accessor :metadata_enables, :metadata_discards
+
+    # user directives
+    attr_accessor :allow_distribution, :allow_search, :external_id, :submitter
+
     attr_accessor :external_metadata
-    
+
+    attr_accessor :use_beta
+
     def initialize(options={}, &block)
       options.each {|k,v| send("#{k}=", v)}
       yield(self) if block_given?
     end
-    
-    def call(method)
-      method = method.intern unless method.is_a?(Symbol)
-      raise ArgumentError.new("Unknown method: #{method}") unless AVAILABLE_METHODS.keys.include? method
-      
+
+    def enlighten
       post_args = {
         "licenseID" => @license_id,
         "content" => Iconv.iconv('UTF-8//IGNORE', 'UTF-8',  "#{@content} ").first[0..-2],
         "paramsXML" => params_xml
       }
-      
-      url = URI.parse(POST_URL + AVAILABLE_METHODS[method])
-      resp, data = Net::HTTP.post_form(url, post_args)
-      
-      return resp.is_a?(Net::HTTPOK) ? data : [data, "API Error: #{resp}"]
+
+      @client ||= Curl::Easy.new
+      @client.url = @use_beta ? BETA_REST_ENDPOINT : REST_ENDPOINT
+      @client.timeout = HTTP_TIMEOUT
+
+      post_fields = post_args.map {|k,v| Curl::PostField.content(k, v) }
+
+      do_request(post_fields)
     end
-    
+
+    def params_xml
+      check_params
+
+      params_node = XML::Node.new('c:params')
+      params_node['xmlns:c'] = 'http://s.opencalais.com/1/pred/'
+      params_node['xmlns:rdf'] = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#'
+
+      processing_node = XML::Node.new('c:processingDirectives')
+      processing_node['c:contentType'] = AVAILABLE_CONTENT_TYPES[@content_type] if @content_type
+      processing_node['c:outputFormat'] = AVAILABLE_OUTPUT_FORMATS[@output_format] if @output_format
+      processing_node['c:reltagBaseURL'] = @reltag_base_url.to_s if @reltag_base_url
+
+      processing_node['c:enableMetadataType'] = @metadata_enables.join(';') unless @metadata_enables.empty?
+      processing_node['c:discardMetadata'] = @metadata_discards.join(';') unless @metadata_discards.empty?
+      processing_node['c:omitOutputtingOriginalText'] = 'true' if @omit_outputting_original_text
+
+      user_node = XML::Node.new('c:userDirectives')
+      user_node['c:allowDistribution'] = @allow_distribution.to_s unless @allow_distribution.nil?
+      user_node['c:allowSearch'] = @allow_search.to_s unless @allow_search.nil?
+      user_node['c:externalID'] = @external_id.to_s if @external_id
+      user_node['c:submitter'] = @submitter.to_s if @submitter
+
+      params_node << processing_node
+      params_node << user_node
+
+      if @external_metadata
+        external_node = XML::Node.new('c:externalMetadata')
+        external_node << @external_metadata
+        params_node << external_node
+      end
+
+      params_node.to_s
+    end
+
     private
-      def params_xml
-        content_type = @content_type && AVAILABLE_CONTENT_TYPES.keys.include?(@content_type) ? AVAILABLE_CONTENT_TYPES[@content_type] : AVAILABLE_CONTENT_TYPES[DEFAULT_CONTENT_TYPE]
-        output_format = @output_format && AVAILABLE_OUTPUT_FORMATS.keys.include?(@output_format) ? AVAILABLE_OUTPUT_FORMATS[@output_format] : AVAILABLE_OUTPUT_FORMATS[DEFAULT_OUTPUT_FORMAT]
-        allow_distribution = @allow_distribution ? "true" : "false"
-        allow_search = @allow_search ? "true" : "false"
-        submitter = @submitter || DEFAULT_SUBMITTER
-        external_id = @external_id || Digest::SHA1.hexdigest(@content.inspect)
-        external_metadata = @external_metadata || ""
-        
-        xml  = %[<c:params xmlns:c="http://s.opencalais.com/1/pred/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">]
-        xml += %[<c:processingDirectives c:contentType="#{content_type}" c:outputFormat="#{output_format}"></c:processingDirectives>]
-        xml += %[<c:userDirectives c:allowDistribution="#{allow_distribution}" c:allowSearch="#{allow_search}" c:externalID="#{external_id}" c:submitter="#{submitter}"></c:userDirectives>]
-        xml += %[<c:externalMetadata>#{external_metadata}</c:externalMetadata>]
-        xml += %[</c:params>]
+      def check_params
+        raise 'missing content' if @content.nil? || @content.empty?
+
+        content_length = @content.length
+        raise 'content is too small' if content_length < MIN_CONTENT_SIZE
+        raise 'content is too large' if content_length > MAX_CONTENT_SIZE
+
+        raise 'missing license id' if @license_id.nil? || @license_id.empty?
+
+        raise 'unknown content type' unless AVAILABLE_CONTENT_TYPES.keys.include?(@content_type) if @content_type
+        raise 'unknown output format' unless AVAILABLE_OUTPUT_FORMATS.keys.include?(@output_format) if @output_format
+
+        %w[calculate_relevance allow_distribution allow_search].each do |variable|
+          value = self.send(variable)
+          unless NilClass === value || TrueClass === value || FalseClass === value
+            raise "expected a boolean value for #{variable} but got #{value}"
+          end
+        end
+
+        @metadata_enables ||= []
+        unknown_enables = Set.new(@metadata_enables) - KNOWN_ENABLES
+        raise "unknown metadata enables: #{unknown_enables.to_ainspect}" unless unknown_enables.empty?
+
+        @metadata_discards ||= []
+        unknown_discards = Set.new(@metadata_discards) - KNOWN_DISCARDS
+        raise "unknown metadata discards: #{unknown_discards.to_ainspect}" unless unknown_discards.empty?
+      end
+
+      def do_request(post_fields)
+        unless @client.http_post(post_fields)
+          raise 'unable to post to api endpoint'
+        end
+
+        @client.body_str
       end
   end
 end

data/lib/calais/response.rb

@@ -1,77 +1,195 @@
 module Calais
   class Response
-    attr_reader :rdf, :names, :relationships, :error
-    
-    def initialize(raw, error=nil)
-      @error = error
-      @names = []
-      @relationships = []
-
-      parse_raw(raw)
-      return if @error
-      
-      parse_names
-      parse_relationships
+    MATCHERS = {
+      :docinfo => 'DocInfo',
+      :docinfometa => 'DocInfoMeta',
+      :defaultlangid => 'DefaultLangId',
+      :doccat => 'DocCat',
+      :entities => 'type/em/e',
+      :relations => 'type/em/r',
+      :geographies => 'type/er',
+      :instances => 'type/sys/InstanceInfo',
+      :relevances => 'type/sys/RelevanceInfo',
+    }
+
+    attr_accessor :submitter_code, :signature, :language, :submission_date, :request_id, :doc_title, :doc_date
+    attr_accessor :hashes, :entities, :relations, :geographies, :categories
+
+    def initialize(rdf_string)
+      @raw_response = rdf_string
+
+      @hashes = []
+      @entities = []
+      @relations = []
+      @geographies = []
+      @relevances = {} # key = String hash, val = Float relevance
+      @categories = []
+
+      extract_data
+    end
+
+    class Entity
+      attr_accessor :hash, :type, :attributes, :relevance, :instances
+    end
+
+    class Relation
+      attr_accessor :hash, :type, :attributes, :instances
     end
-    
-    Name::TYPES.each_pair do |method_name, type|
-      define_method method_name.to_sym do
-        @names.map {|name| name if name.type == type }.compact
+
+    class Geography
+      attr_accessor :name, :hash, :attributes
+    end
+
+    class Category
+      attr_accessor :name, :score
+    end
+
+    class Instance
+      attr_accessor :prefix, :exact, :suffix, :offset, :length
+
+      # Makes a new Instance object from an appropriate LibXML::XML::Node.
+      def self.from_node(node)
+        instance = self.new
+        instance.prefix = node.find_first("c:prefix").content
+        instance.exact = node.find_first("c:exact").content
+        instance.suffix = node.find_first("c:suffix").content
+        instance.offset = node.find_first("c:offset").content.to_i
+        instance.length = node.find_first("c:length").content.to_i
+
+        instance
       end
     end
-    
+
+    class CalaisHash
+      attr_accessor :value
+
+      def self.find_or_create(hash, hashes)
+        if !selected = hashes.select {|h| h.value == hash }.first
+          selected = self.new
+          selected.value = hash
+          hashes << selected
+        end
+
+        selected
+      end
+    end
+
     private
-      def parse_raw(raw)
-        @libxml = XML::Parser.string(XML::Parser.string(raw).parse.root.child.content).parse
-        @rdf = @libxml.to_s
-        @error = @libxml.find("/Error/Exception").first.content rescue @error
+      def extract_data
+        doc = XML::Parser.string(@raw_response).parse
+
+        doc.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '#{MATCHERS[:docinfometa]}')]/..").each do |node|
+          @language = node['language']
+          @submission_date =  DateTime.parse node['submissionDate']
+
+          attributes = extract_attributes(node.find("*[contains(name(), 'c:')]"))
+
+          @signature = attributes.delete('signature')
+          @submitter_code = attributes.delete('submitterCode')
+
+          node.remove!
+        end
+
+        doc.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '#{MATCHERS[:docinfo]}')]/..").each do |node|
+          @request_id = node['calaisRequestID']
+
+          attributes = extract_attributes(node.find("*[contains(name(), 'c:')]"))
+
+          @doc_title = attributes.delete('docTitle')
+          @doc_date = Date.parse attributes.delete('docDate')
+
+          node.remove!
+        end
+
+        @categories = doc.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '#{MATCHERS[:doccat]}')]/..").map do |node|
+          category = Category.new
+          category.name = node.find_first("c:categoryName").content
+          category.score = node.find_first("c:score").content.to_f
+
+          node.remove!
+          category
+        end
+
+        @relevances = doc.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '#{MATCHERS[:relevances]}')]/..").inject({}) do |acc, node|
+          subject_hash = node.find_first("c:subject")[:resource].split('/')[-1]
+          acc[subject_hash] = node.find_first("c:relevance").content.to_f
+
+          node.remove!
+          acc
+        end
+
+        @entities = doc.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '#{MATCHERS[:entities]}')]/..").map do |node|
+          extracted_hash = node['about'].split('/')[-1] rescue nil
+
+          entity = Entity.new
+          entity.hash = CalaisHash.find_or_create(extracted_hash, @hashes)
+          entity.type = extract_type(node)
+          entity.attributes = extract_attributes(node.find("*[contains(name(), 'c:')]"))
+
+          entity.relevance = @relevances[extracted_hash]
+          entity.instances = extract_instances(doc, extracted_hash)
+
+          node.remove!
+          entity
+        end
+
+        @relations = doc.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '#{MATCHERS[:relations]}')]/..").map do |node|
+          extracted_hash = node['about'].split('/')[-1] rescue nil
+
+          relation = Relation.new
+          relation.hash = CalaisHash.find_or_create(extracted_hash, @hashes)
+          relation.type = extract_type(node)
+          relation.attributes = extract_attributes(node.find("*[contains(name(), 'c:')]"))
+          relation.instances = extract_instances(doc, extracted_hash)
+
+          node.remove!
+          relation
+        end
+
+        @geographies = doc.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '#{MATCHERS[:geographies]}')]/..").map do |node|
+          attributes = extract_attributes(node.find("*[contains(name(), 'c:')]"))
+
+          geography = Geography.new
+          geography.name = attributes.delete('name')
+          geography.hash = attributes.delete('subject')
+          geography.attributes = attributes
+
+          node.remove!
+          geography
+        end
+
+        doc.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '#{MATCHERS[:defaultlangid]}')]/..").each { |node| node.remove! }
+        doc.root.find("./*").each { |node| node.remove! }
+
+        return
       end
-      
-      def parse_names
-        @names = @libxml.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '/em/e/')]/..").map do |n|
-          name = n.find_first("c:name").content
-          type = n.find_first("rdf:type").properties.to_a.assoc("resource").last.split('/').last
-          hash = n.properties.to_a.assoc("about").last.split("/").last
-          
-          locations = @libxml.root.find("rdf:Description/c:subject[contains(@rdf:resource, '#{hash}')]/..").map do |n2|
-            start = n2.find_first("c:offset").content.to_i
-            Range.new(start, start+n2.find_first("c:length").content.to_i)
-          end
-          
-          Name.new(
-            :name => name,
-            :hash => hash,
-            :type => type,
-            :locations => locations
-          )
+
+      def extract_instances(doc, hash)
+        doc.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '#{MATCHERS[:instances]}')]/..").select do |instance_node|
+          instance_node.find_first("c:subject")[:resource].split("/")[-1] == hash
+        end.map do |instance_node|
+          instance = Instance.from_node(instance_node)
+          instance_node.remove!
+
+          instance
         end
       end
-      
-      def parse_relationships
-        @libxml.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '/em/r')]/..").each do |n|
-          hash = n.properties.to_a.assoc("about").last.split("/").last
-          type = n.find_first("rdf:type").properties.to_a.assoc("resource").last.split('/').last
-          
-          metadata = {}
-          
-          n.to_a.each do |n2|
-            next if n2.name == "type" or n2.comment?
-            resource = n2.properties.to_a.assoc("resource")
-            metadata[n2.name] = resource ? Name.find_in_names(resource.last.split("/").last, @names) : n2.content.strip
-          end
-          
-          locations = @libxml.root.find("rdf:Description/c:subject[contains(@rdf:resource, '#{hash}')]/..").map do |n2|
-            start = n2.find_first("c:offset").content.to_i
-            Range.new(start, start+n2.find_first("c:length").content.to_i)
-          end
-          
-          
-          @relationships << Relationship.new(
-            :type => type,
-            :hash => hash,
-            :metadata => metadata,
-            :locations => locations
-          )
+
+      def extract_type(node)
+        node.find("*[name()='rdf:type']")[0]['resource'].split('/')[-1]
+      rescue
+        nil
+      end
+
+      def extract_attributes(nodes)
+        nodes.inject({}) do |hsh, node|
+          value = if node['resource']
+              extracted_hash = node['resource'].split('/')[-1] rescue nil
+              CalaisHash.find_or_create(extracted_hash, @hashes)
+            else
+              node.content
+            end
+          hsh.merge(node.name => value)
         end
       end
   end