dover_to_calais 0.2.0 → 0.2.1

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    OWJmOWEyMGFjNDk2ZjZiODYyNjQ1NDM2YjM0YjMyNzQ1MmUzZjg3MA==
+    MmI4YTNmZGZkYTViNDVlMzVmNGRiMGM3YTFlYzQ5YTc1ZTdmMDU3ZA==
   data.tar.gz: !binary |-
-    MTllMDRiOTNlNDg2Y2FiNmY1MmQyMjAyMzViNWJiZWFmN2ZjYTY3ZA==
+    M2U0YjI4ZTJhM2E4M2U4MGM2N2E2OWZkYjUwYzExM2YyOWExZDYxNg==
 SHA512:
   metadata.gz: !binary |-
-    OGI3NDU4YWU1YzllMjBiNTVlMmU3NzNhMDUzYmNhNWYzODY0ZTE3MDQzZmMx
-    NmZiMDMxZjYzMTI3ZTdkYWU5MGNiNDc3ZTE2ZTRjYThjYjc5ZDQxNjFlZjU0
-    MGRmZWY5ZGM4NTAwYjAyZTEyZmY5M2I5MDdjNDA4NWQ1MDE4MDM=
+    ZjA0NDgwNjM4MzQxZDYxZmMwNjFmYzg2MTc4MjI1MDY0ZmU2ZGI2ZWZmNTY4
+    MjU5OTBhZWVkY2YyYzEyODhiOTI4NWI3NGQwZjI4NTg4ZWNiNmVlZWQwZjAx
+    NGJjYjc5YzdhMWJmN2UyODQ1YzM0MGUxY2VkOGU4YmQ5NjU3OTc=
   data.tar.gz: !binary |-
-    ODUyZGFhN2JhYjdjZDAyNmMxMTNhZjY0MjJhNWQ5YjU2OTY0OTQyNmU4MDkz
-    NjljZTU1NDUzYTRhN2I2MTA3MmQ3MTM3MDYxODUyMjgzOGVkYTYzNzY4MjA1
-    YTgyZDNkNjE3YjI1NWJiYTJkMzNjN2RiYzEzN2M1MWNmYzFhMzU=
+    YjY3NDk2ODY1YjE4ZjM3NDMzZWQyMzc5Njc3YzRiODgyODYxNjJjMGY3YmUz
+    MDJiNzQzMTZhZjQxYWJjOWIxMjIyZjU3YjE3YmE5MDVmMjk2YjA3Y2QxZmQz
+    MmQ1YjY0ZTljNGQ0NGRkODU4NTg0ZWIzYzM4YWRmZGIxYTgxNDM=

data/Rakefile

@@ -3,5 +3,10 @@ require 'cucumber'
 require 'cucumber/rake/task'
 
 Cucumber::Rake::Task.new(:features) do |t|
-  t.cucumber_opts = "features --format pretty API_KEY=#{ENV['API_KEY']}"
+  if ENV['TAGS'].empty?
+    t.cucumber_opts = "features --format pretty API_KEY=#{ENV['API_KEY']}"
+  else
+    t.cucumber_opts = "features --format pretty API_KEY=#{ENV['API_KEY']} --tags #{ENV['TAGS']}"
+  end
+  
 end

data/dover_to_calais.gemspec

@@ -21,6 +21,10 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "eventmachine", "~> 1.0", ">= 1.0.3"
   spec.add_runtime_dependency "em-http-request", "~> 1.1"
   spec.add_runtime_dependency "yomu", "~> 0.1", ">= 0.1.9"
+  spec.add_runtime_dependency "json", "~> 1.5", ">= 1.5.5"
+  spec.add_runtime_dependency "ohm", "~> 2.0", ">= 2.0.0"
+  spec.add_runtime_dependency "ohm-contrib", "~> 2.0", ">= 2.0.0"
+  spec.add_runtime_dependency "em-throttled_queue", "~> 1.1", ">= 1.1.0"
 
 
   spec.files         = `git ls-files`.split($/)

data/features/data_mining.feature

@@ -0,0 +1,10 @@
+  Feature: Ability to detect relationships between entities and events
+
+  Background:
+    Given the file 'test_file_1.txt' is successfully processed with the rich output  
+
+    @rich_output
+    Scenario:  Filter an entity with the rich output format
+    When I filter the response on {:entity => 'Event', :value => 'Meeting'}
+    Then The output should be an error
+  

data/features/data_sources.feature

@@ -1,7 +1,11 @@
-Feature: Able to handle wide range of data formats as input
-  Scenario Outline:  Processing various data-source formats
+Feature: Able to handle wide range of data formats as input 
+
+
+  @simple_output
+  Scenario Outline:  Processing various data-source formats (Simple Format)
     Given  the file <input>
-    When DoverToCalais processes this file
+    When the Output format is set to 'Text/Simple'
+    And DoverToCalais processes this file
     Then the output should have no errors
 
   Examples:
@@ -12,3 +16,19 @@ Feature: Able to handle wide range of data formats as input
   |test_file_1.pdf|
   |test_file_1.rtf|
   |test_file_1.txt|
+
+  @rich_output
+  Scenario Outline:  Processing various data-source formats (Rich Format)
+    Given  the file <input>
+    When the Output format is set to 'Application/JSON'
+    And DoverToCalais processes this file
+    Then the output should have no errors
+
+  Examples:
+  | input |
+  |test_file_1.doc |
+  |test_file_1.html|
+  |test_file_1.odt|
+  |test_file_1.pdf|
+  |test_file_1.rtf|
+  |test_file_1.txt|

data/features/filtering.feature

@@ -1,24 +1,30 @@
+@simple_output
 Feature: Ability to select certain OpenCalais entities based on certain conditions
 
   Background:
-    Given the file 'test_file_1.txt' is successfully processed
+    Given the file 'test_file_1.txt' is successfully processed with the simple output
 
 
-  Scenario:  Select all entities with a specific name
+  Scenario:  Filter all entities with a specific name
     When I filter on {:entity => 'EmailAddress'}
     Then the output should have 2 entries
     And All entries should be named 'EmailAddress'
 
-  Scenario:  Select an entity with a specific value
+  Scenario:  Filter an entity with a specific value
     When I filter on {:entity => 'Event', :value => 'Meeting'}
     Then the output should have 1 entries
     And All entries should be named 'Event'
     And All entries should have the value 'Meeting'
 
 
-  Scenario:  Select an entity only if another entity with a specific value exists in the data source
+  Scenario:  Filter an entity only if another entity with a specific value exists in the data source
     When I filter on {:entity => 'Person', :given => {:entity => 'Event', :value => 'Meeting'}}
     Then the output should have 2 entries
     And All entries should be named 'Person'
     And One entry should have the value 'Roger Kay'
-    And One entry should have the value 'David Bailey'
+    And One entry should have the value 'David Bailey'
+
+  Scenario:  Filter all entities with a specific name
+    When I filter on {:entity => 'EmailAddress'}
+    Then the output should have 2 entries
+    And All entries should be named 'EmailAddress'

data/features/step_definitions/data_mining_steps.rb

@@ -0,0 +1,32 @@
+require 'eventmachine'
+require 'em-http-request'
+require 'yomu'
+require 'rspec'
+require_relative '../../lib/dover_to_calais'
+#require_relative './filtering_steps.rb'
+
+
+# N.B Cucumber must be run with the Environment variable 'API_KEY' set
+# to the OpenCalais API Key value.
+
+Given(/^the file '(\w+\.\w{3,4})' is successfully processed with the rich output$/) do |file|
+
+  steps %{
+        Given  the file #{file}
+        When the Output format is set to 'Application/JSON'
+        And DoverToCalais processes this file
+        Then the output should have no errors
+        }
+
+end
+
+When(/^I filter the response on ({.+})$/) do  |f|
+  @filtered_output = @output.filter(eval(f))
+
+end
+
+Then(/^The output should be an error$/) do
+  @filtered_output.match(/^ERR:\s/).should_not be_nil
+end
+
+

data/features/step_definitions/data_sources_steps.rb

@@ -4,27 +4,28 @@ require 'eventmachine'
 require 'em-http-request'
 require 'yomu'
 require 'rspec'
-require File.expand_path('../../../lib/dover_to_calais', __FILE__)
-
+require_relative '../../lib/dover_to_calais'
 
 # N.B Cucumber must be run with the Environment variable 'API_KEY' set
 # to the OpenCalais API Key value.
 
 
+
 Given(/^the file (\w+\.\w{3,4})$/) do |arg1|
   puts arg1
  @input = Dir.pwd + '/test/' + arg1
   @output = nil
+  
 end
 
 
 
 When(/^DoverToCalais processes this file$/) do
   EM.run {
-
     DoverToCalais::API_KEY =  ENV['API_KEY']
+    
     d1 =  DoverToCalais::Dover.new(@input)
-    d1.analyse_this
+    d1.analyse_this(@output_format)
     d1.to_calais do |response|
       @output = response
       EM.stop
@@ -33,6 +34,15 @@ When(/^DoverToCalais processes this file$/) do
   }
 end
 
+When(/^the Output format is set to 'Text\/Simple'$/) do
+  @output_format = nil
+end
+
+When(/^the Output format is set to 'Application\/JSON'$/) do
+  @output_format = :rich
+end
+
+
 
 
 Then(/^the output should have no errors$/) do

data/features/step_definitions/filtering_steps.rb

@@ -3,27 +3,31 @@ require 'eventmachine'
 require 'em-http-request'
 require 'yomu'
 require 'rspec'
-require File.expand_path('../../../lib/dover_to_calais', __FILE__)
-
+#require File.expand_path('../../../lib/dover_to_calais', __FILE__)
+require_relative '../../lib/dover_to_calais'
 
 # N.B Cucumber must be run with the Environment variable 'API_KEY' set
 # to the OpenCalais API Key value.
 
 
 
-Given(/^the file '(\w+\.\w{3,4})' is successfully processed$/) do |file|
+Given(/^the file '(\w+\.\w{3,4})' is successfully processed with the simple output$/) do |file|
 
   steps %{
         Given  the file #{file}
-        When DoverToCalais processes this file
+        When the Output format is set to 'Text/Simple'
+        And DoverToCalais processes this file
         Then the output should have no errors
         }
 
 end
 
 
-When(/^I filter on ({.+})/) do  |filter|
-  @filtered_output = @output.filter(eval(filter))
+
+
+
+When(/^I filter on ({.+})$/) do  |f|
+  @filtered_output = @output.filter(eval(f))
 
 end
 

data/lib/dover_to_calais.rb

@@ -5,12 +5,22 @@ require 'nokogiri'
 require 'eventmachine'
 require 'em-http-request'
 require 'yomu'
+require 'json'
+require "dover_to_calais/models"  #gem lib file
+require 'ohm'
 
 
 module DoverToCalais
 
 
   PROXY = nil
+  REDIS = "redis://127.0.0.1:6379/6"
+
+  def self.flushdb
+    Ohm.redis = Redic.new(REDIS)
+    Ohm.redis.call "FLUSHDB"
+  end
+
 
   # The ResponseItem structure holds all potential text and attribute values of an OpenCalais
   # XML Simple format element.
@@ -66,65 +76,361 @@ module DoverToCalais
   #     nil if none occurred
   #
   class ResponseData
-    attr_reader :error
 
+
+
+    class Entity< Struct.new(:type, :name, :ref)
+
+      def to_hash
+        a_hash = {}
+        self.each_pair do |attr, value|
+          a_hash[attr] =  value
+        end
+        a_hash
+      end
+
+    end
+
+    class GenericRelation< Struct.new(:subject, :verb, :object, :detection)
+
+    end #class
+
+    class Event
+
+      attr_reader :entities
+
+      def initialize(events_hash)
+        # @entities = entity_hash
+        events_hash.each do |k,v|
+          unless k.eql?('_typeGroup') || k.eql?('instances') || k.eql?('_typeReference')
+            k = 'type' if k.eql?('_type') #don't like the underscore
+            ## create and initialize an instance variable for this key/value pair
+            self.instance_variable_set("@#{k}", v)
+            ## create the getter that returns the instance variable
+            self.class.send(:define_method, k, proc{self.instance_variable_get("@#{k}")})
+            ## create the setter that sets the instance variable
+            self.class.send(:define_method, "#{k}=", proc{|v| self.instance_variable_set("@#{k}", v)})
+          end
+        end #block
+      end #method
+
+      def each_pair
+        self.instance_variables.each do |a|
+          yield a, self.instance_variable_get(a)
+        end
+      end
+
+      def [](attrib_name)
+        self.instance_variables.each do |a|
+          if "@#{a}" == attrib_name
+            self.instance_variable_get(a)
+          end
+        end
+      end
+
+      public :each_pair
+
+    end #class
+
+
+
+
+    attr_reader :error, :entities_store, :events_store, :generic_relations_store, :freds
     # creates a new ResponseData object, passing the name of the data source to be processed
     #
-    # @param xml_data [ Nokogiri::XML::NodeSet, nil] the xml data returned by OpenCalais
+    # @param response_data [ Nokogiri::XML::NodeSet, Hash, nil] the XML or JSON data returned by OpenCalais
     # @param error [ String, nil] an error description if the OpenCalais call has failed
-    def initialize(xml_data = nil, error = nil)
-      if xml_data
-        @raw = xml_data
+    def initialize(response_data = nil, error = nil)
+      if response_data.class.to_s == "Nokogiri::XML::Document"
+        @xml_data = response_data
+      elsif response_data.class.to_s == "Hash"
+        @json_data = response_data
+        prepare_data(response_data)
+
+
       else
         @error = error
       end
+
     end
 
     # Returns  the response data as an XML string or an error, if one has occurred.
     #
     # @return [String] an XML string
     def to_s
-      @raw ?  @raw.to_s : @error
+      if @xml_data
+        @xml_data.to_s
+      elsif @json_data
+        @json_data.to_s
+      else
+        @error
+      end
+    end
+
+
+
+
+    # The method will first create three Hash instance variables, where it will store the
+    # Entities, Generic Relations and Events -respectively- from the OpenCalais response.
+    # The key on each Hash instance variable will be the OpenCalais ID and the value will
+    # be the values_hash for that ID.
+    # Secondly, the method will iterate through each Entity, find all of it's related
+    # Relations and Events and store them -in a relational manner- in Redis, via Ohm.
+    #
+    # Only applicable with the JSON (rich) output format
+    #
+    # @param Hash the OpenCalais JSON response, as a Hash
+    # @return an GenericRelation Struct if a match is found, nil otherwise
+
+    def prepare_data(results_hash)
+
+      @entities_store = {}
+      @generic_relations_store = {}
+      @events_store = {}
+      # find all Entities in response
+      @entities_store = results_hash.select{|key, hash| hash["_typeGroup"] == "entities"}
+      # find all GenericRelations in response
+      @generic_relations_store = results_hash.select{|key, hash| hash["_typeGroup"] == "relations" &&
+      hash["_type"] == "GenericRelations"}
+      # find all Events in response
+      @events_store = results_hash.select{|key, hash| hash["_typeGroup"] == "relations" &&
+      hash["_type"] != "GenericRelations"}
+
+      Ohm.redis = Redic.new(REDIS)
+
+
+      #for each Entity find all related Relations and Events and store them to Ohm/Redis
+      @entities_store.each_pair do |k, v|
+
+        entity_set = EntityModel.find(calais_id: k)
+
+        if entity_set.size > 0 #entity already exists in store
+          entity = entity_set.first
+          k = entity.calais_id
+        else #entity doesn't exist in store
+          entity = EntityModel.create(:name => v['name'], :type => v['_type'], :calais_id => k)
+          entity.save
+        end #if
+
+
+        #get all referenced relations
+        find_in_relations(k).each do |obj|
+
+          found_rel = get_relation(obj[0])
+          if found_rel
+
+            found_rel.subject = convert_to_hash(found_rel.subject)
+            found_rel.object = convert_to_hash(found_rel.object)
+
+            relation = EntityModel::RelationModel.create(:subject => found_rel.subject,
+            :object => found_rel.object,
+            :verb => found_rel.verb,
+            :detection => found_rel.detection,
+            :calais_id => obj[0])
+            entity.relations.add(relation)
+          end #if
+        end #each
+        #get all referenced events
+        find_in_events(k).each do |obj|
+          found_event = get_event(obj[0])
+          attribs = {}
+          if found_event
+
+            found_event.each_pair do |key, val|
+
+              key = key.to_s.slice(1, key.length-1)
+              attribs[key] = val
+
+            end #block
+
+            event = EntityModel::EventModel.create(:calais_id => obj[0], :info_hash => attribs)
+            entity.events.add(event)
+
+          end #if
+
+        end #each
+      end #each_pair
+    end #method
+
+
+
+    # Coverts an attribute to an appropriate Hash
+    #
+    # Only applicable with the JSON (rich) output format
+    #
+    # @param [String, DoverToCalais::ResponseData::Entity, Hash] an object
+    # @return a Hash value
+    def convert_to_hash(an_attribute)
+      h = {}
+      if an_attribute.class.to_s.eql?('String')
+        h[:name] = an_attribute
+      end
+
+      if an_attribute.class.to_s.eql?('DoverToCalais::ResponseData::Entity')
+        h = an_attribute.to_hash
+      end
+
+      if an_attribute.class.to_s.eql?('Hash')
+        h = an_attribute
+      end
+
+      h
+    end #method
+
+
+
+    # Retrieves the entity with the specified key (OpenCalais ID)
+    #
+    # Only applicable with the JSON (rich) output format
+    #
+    # @param String the OpenCalais ID
+    # @return an Entity Struct if a match is found, nil otherwise
+    def get_entity(ref_key)
+      if @entities_store.has_key?(ref_key)
+        Entity.new(@entities_store[ref_key]['_type'],  @entities_store[ref_key]['name'], ref_key)
+      else
+        nil
+      end
     end
 
 
-    # Filters the response object to extract relevant data.
+    # Retrieves the relation with the specified key (OpenCalais ID). The method will also
+    # de-reference any of its attributes that refer to other entities via an OpenCalais ID
+    # and will replace the references with the appropriate Entity structure, if applicable
+    #
+    # Only applicable with the JSON (rich) output format
+    #
+    # @param String the OpenCalais ID
+    # @return an GenericRelation Struct if a match is found, nil otherwise
+    def get_relation(ref_key)
+      if @generic_relations_store.key?(ref_key)
+
+        if @generic_relations_store[ref_key]['relationsubject']
+          gr_subject = @generic_relations_store[ref_key]['relationsubject'].match('^http://d.opencalais.com') ?
+          get_entity(@generic_relations_store[ref_key]['relationsubject']) :
+          @generic_relations_store[ref_key]['relationsubject']
+        else
+          gr_subject = 'N/A'
+        end
+
+
+        if @generic_relations_store[ref_key]['relationobject']
+          gr_object = @generic_relations_store[ref_key]['relationobject'].match('^http://d.opencalais.com') ?
+          get_entity(@generic_relations_store[ref_key]['relationobject']) :
+          @generic_relations_store[ref_key]['relationobject']
+        else
+          gr_object = 'N/A'
+        end
+
+        GenericRelation.new(gr_subject,
+        @generic_relations_store[ref_key]['verb'],
+        gr_object,
+        @generic_relations_store[ref_key]['instances'][0]['exact'] ||= 'N/A')
+      else
+        nil
+      end
+    end
+
+    def get_event(ref_key)
+
+      dereferenced_events = {}
+
+      if @events_store.key?(ref_key)
+
+        @events_store[ref_key].each do |k, v|
+
+          if v.class.to_s.eql?("String") && v.match('^http://d.opencalais.com')
+            dereferenced_events[k] = get_entity(v).to_hash
+          elsif v.class.to_s.eql?("String") && !v.match('^http://d.opencalais.com')
+            h = {}
+            h['name'] = v
+            dereferenced_events[k] = h
+          elsif v.class.to_s.eql?("Array")
+            h = {}
+            h['name'] = v[0]['exact']
+            dereferenced_events[k] = h
+          end
+        end
+
+        Event.new(dereferenced_events)
+      else
+        nil
+      end
+    end
+
+
+    # Selects a Hash of generic relations, where the relations' subject or object attributes
+    # match the specified OpenCalais ID.
+    #
+    # Only applicable with the JSON (rich) output format
+    #
+    # @param String the OpenCalais ID
+    # @return a Hash with the selected matches
+    def find_in_relations(ref_key)
+      @generic_relations_store.select{|key, hash| (hash["relationsubject"] == ref_key) ||
+      (hash["relationobject"] == ref_key) }
+
+    end
+
+    # Selects a Hash of events, where the events' key matches the
+    # specified OpenCalais ID.
+    #
+    # Only applicable with the JSON (rich) output format
+    #
+    # @param String the OpenCalais ID
+    # @return a Hash with the selected matches
+    def find_in_events(ref_key)
+      @events_store.select{|key, hash| hash.has_value?(ref_key) }
+    end
+
+
+
+    # Filters the xml response object to extract relevant data.
     #
     # @param params [Hash] a filter Hash (see code samples)
     # @return [Array[ResponseItem]] a list of relevant response items
     def filter(params)
+      unless  @xml_data
+        return 'ERR: filter method only works with xml-based output!'
+
+      end
+
       result = Array.new
       begin
-        if @raw
+        if @xml_data
 
           if params[:given]
-            found = @raw.xpath("//#{params[:given][:entity]}[contains(text(), #{params[:given][:value].inspect})]")
+            found = @xml_data.xpath("//#{params[:given][:entity]}[contains(text(), #{params[:given][:value].inspect})]")
             if found.size > 0
-              @raw.xpath("//#{params[:entity]}[contains(text(), #{params[:value].inspect})]").each do |node|
+              @xml_data.xpath("//#{params[:entity]}[contains(text(), #{params[:value].inspect})]").each do |node|
                 result <<  create_response_item(node)
               end
             end
           else  # no conditional
-            @raw.xpath("//#{params[:entity]}[contains(text(), #{params[:value].inspect})]").each do |node|
+            @xml_data.xpath("//#{params[:entity]}[contains(text(), #{params[:value].inspect})]").each do |node|
               result <<  create_response_item(node)
             end
           end
 
           return result
         else # no xml data
-          puts 'ERR: no valid xml data!'
+          return 'ERR: no valid xml data!'
 
         end #if
 
       rescue  Exception=>e
-        puts "ERR: #filter:  #{e}"
+        return "ERR: #filter:  #{e}"
 
       end
 
-      #return result
+      return result
 
     end  #method
 
+    # Creates a Response Item from an xml node.
+    #
+    # @param node [Nokogiri::XML::Node] an XML node
+    # @return [ResponseItem] a response item object
     def create_response_item(node)
       node_relevance =  node.attribute('relevance').text.to_f if node.has_attribute?('relevance')
       node_count =  node.attribute('count').text.to_i if node.has_attribute?('count')
@@ -133,17 +439,18 @@ module DoverToCalais
       node_orig_value =  node.xpath('originalValue').text if node.name.eql?('SocialTag')
 
       ResponseItem.new(node.name,
-                       node.text,
-                       node_relevance,
-                       node_count,
-                       node_normalized,
-                       node_importance,
-                       node_orig_value )
+      node.text,
+      node_relevance,
+      node_count,
+      node_normalized,
+      node_importance,
+      node_orig_value )
 
     end
 
     public :filter
-    private :create_response_item
+    private :create_response_item, :prepare_data, :convert_to_hash, :find_in_relations, :find_in_events,
+            :get_event, :get_entity
 
   end #class
 
@@ -211,13 +518,23 @@ module DoverToCalais
 
     end #method
 
+
+
+
+
     # Gets the source text parsed. If the parsing is successful, the data source is POSTed to OpenCalais
     # via an EventMachine request and a callback is set to manage the OpenCalais response.
-    # All Dover object callbacks are then called with the request result yielded to them. 
+    # All Dover object callbacks are then called with the request result yielded to them.
     #
     # @param N/A
     # @return a {Class ResponseData} object
-    def analyse_this
+    def analyse_this(output_format=nil)
+
+      if output_format
+        @output_format = 'application/json'
+      else
+        @output_format = 'Text/Simple'
+      end
 
       @document = get_src_data(@data_src)
       begin
@@ -230,19 +547,19 @@ module DoverToCalais
 
 
           if DoverToCalais::PROXY &&
-              DoverToCalais::PROXY.class.eql?('Hash') &&
-              DoverToCalais::PROXY.keys[0].eql?(:proxy)
+            DoverToCalais::PROXY.class.eql?('Hash') &&
+            DoverToCalais::PROXY.keys[0].eql?(:proxy)
 
             connection_options = connection_options.merge(DoverToCalais::PROXY)
           end
 
           request_options = {
-              :body => @document.to_s,
-              :head => {
-                  'x-calais-licenseID' => DoverToCalais::API_KEY,
-                  :content_type => 'TEXT/RAW',
-                  :enableMetadataType => 'GenericRelations,SocialTags',
-                  :outputFormat => 'Text/Simple'}
+            :body => @document.to_s,
+            :head => {
+              'x-calais-licenseID' => DoverToCalais::API_KEY,
+              :content_type => 'TEXT/RAW',
+              :enableMetadataType => 'GenericRelations,SocialTags',
+            :outputFormat => @output_format}
           }
 
           http = EventMachine::HttpRequest.new(CALAIS_SERVICE, connection_options ).post request_options
@@ -251,20 +568,34 @@ module DoverToCalais
           http.callback do
 
             if http.response_header.status == 200
-              http.response.match(/<OpenCalaisSimple>/) do |m|
-                response = Nokogiri::XML('<OpenCalaisSimple>' + m.post_match)  do |config|
-                  #strict xml parsing, disallow network connections
-                  config.strict.nonet
-                end #block
-              end #block
-
-              result =   response ?
-                        ResponseData.new(response, nil) :
-                        ResponseData.new(nil,'ERR: cannot find <OpenCalaisSimple> tag in response data - source invalid?')
-            else #non-200 response header
-              result = ResponseData.new nil, 
-                            "ERR: OpenCalais service responded with #{http.response_header.status} - response body: '#{http.response}'"
-            end 
+              if @output_format == 'Text/Simple'
+                http.response.match(/<OpenCalaisSimple>/) do |m|
+                  response = Nokogiri::XML('<OpenCalaisSimple>' + m.post_match)  do |config|
+                    #strict xml parsing, disallow network connections
+                    config.strict.nonet
+                  end #block
+                end
+              else #@output_format == 'application/json'
+                response = JSON.parse(http.response) #response should now be a Hash
+
+              end #if
+
+              case response.class.to_s
+              when 'NilClass'
+                result = ResponseData.new(nil,'ERR: cannot parse response data - source invalid?')
+              when 'Nokogiri::XML::Document'
+                result = ResponseData.new(response, nil)
+              when 'Hash'
+                result = ResponseData.new(response, nil)
+              else
+                result = ResponseData.new(nil,'ERR: cannot parse response data - unrecognized format!')
+              end
+
+
+            else #non-200 response
+              result = ResponseData.new nil,
+              "ERR: OpenCalais service responded with #{http.response_header.status} - response body: '#{http.response}'"
+            end
 
             @callbacks.each { |c| c.call(result) }
 
@@ -287,6 +618,7 @@ module DoverToCalais
     end  #method
 
 
+
     alias_method :analyze_this, :analyse_this
     public :to_calais, :analyse_this
     private :get_src_data

data/lib/dover_to_calais/models.rb

@@ -0,0 +1,46 @@
+require 'ohm'
+require 'ohm/contrib'
+
+module DoverToCalais
+  class EntityModel < Ohm::Model
+    attribute :name
+    attribute :type
+    attribute :calais_id
+    set :relations, :RelationModel
+    set :events, :EventModel
+
+
+    index :name
+    index :type
+    index :calais_id
+
+    def validate
+      assert_present :name
+      assert_present :type
+      assert_present :calais_id
+    end
+
+    class RelationModel < Ohm::Model
+      include Ohm::DataTypes
+
+      attribute :subject, Type::Hash
+      attribute :object, Type::Hash
+      attribute :verb
+      attribute :detection
+      attribute :calais_id
+
+      index :subject
+
+    end #class
+
+    class EventModel < Ohm::Model
+      include Ohm::DataTypes
+
+      attribute :calais_id
+      attribute :info_hash, Type::Hash
+
+    end #class
+
+  end #class
+
+end

data/lib/dover_to_calais/ontology.rb

@@ -1,23 +1,3 @@
-#  ontology.rb
-#  
-#  Copyright 2013 Fred <fred@fred-Veriton-X270>
-#  
-#  This program is free software; you can redistribute it and/or modify
-#  it under the terms of the GNU General Public License as published by
-#  the Free Software Foundation; either version 2 of the License, or
-#  (at your option) any later version.
-#  
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#  
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software
-#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-#  MA 02110-1301, USA.
-#  
-#  
 
 
 module CalaisOntology
@@ -67,25 +47,38 @@ CALAIS_EVENTS = %w(
     Alliance
     AnalystEarningsEstimate
     AnalystRecommendation
+    ArmedAttack
+    ArmsPurchaseSale
     Arrest
     Bankruptcy
     BonusSharesIssuance
     BusinessRelation
     Buybacks
+    CandidatePosition
     CompanyAccountingChange
+    CompanyAffiliates
+    CompanyCompetitor
+    CompanyCustomer
     CompanyEarningsAnnouncement
     CompanyEarningsGuidance
+    CompanyEmployeesNumber
     CompanyExpansion
     CompanyForceMajeure
+    CompanyFounded
     CompanyInvestment
     CompanyLaborIssues
     CompanyLayoffs
     CompanyLegalIssues
+    CompanyLocation
     CompanyListingChange
     CompanyMeeting
     CompanyNameChange
+    CompanyProduct
     CompanyReorganization
     CompanyRestatement
+    CompanyTechnology
+    CompanyUsingProduct
+    CompanyTicker
     ConferenceCall
     Conviction
     CreditRating
@@ -94,19 +87,32 @@ CALAIS_EVENTS = %w(
     DiplomaticRelations
     Dividend
     EmploymentChange
+    EmploymentRelation
     EnvironmentalIssue
+    EquityFinancing
     Extinction
+    FamilyRelation
     FDAPhase
     Indictment
+    IndicesChanges
     IPO
     JointVenture
     ManMadeDisaster
     Merger
+    MilitaryAction
     MovieRelease
     MusicAlbumRelease
     NaturalDisaster
     PatentFiling
     PatentIssuance
+    PersonAttributes
+    PersonCareer
+    PersonCommunication
+    PersonEducation
+    PersonEmailAddress
+    PersonLocation
+    PersonParty
+    PersonRelation
     PersonTravel
     PoliticalEndorsement
     PoliticalRelationship
@@ -114,14 +120,15 @@ CALAIS_EVENTS = %w(
     ProductIssues
     ProductRecall
     ProductRelease
-    SocialTags
+    Quotation
     SecondaryIssuance
-    PersonCommunication
     StockSplit
     Trial
     VotingResult
 )
 
+
+
 CALAIS_TOPICS = %w(
     Business_Finance
     Disaster_Accident

data/lib/dover_to_calais/version.rb

@@ -1,3 +1,3 @@
 module DoverToCalais
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dover_to_calais
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Fred Heath
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-10 00:00:00.000000000 Z
+date: 2014-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -78,6 +78,86 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.1.9
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.5
+- !ruby/object:Gem::Dependency
+  name: ohm
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: ohm-contrib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: em-throttled_queue
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -164,11 +244,14 @@ files:
 - README.md
 - Rakefile
 - dover_to_calais.gemspec
+- features/data_mining.feature
 - features/data_sources.feature
 - features/filtering.feature
+- features/step_definitions/data_mining_steps.rb
 - features/step_definitions/data_sources_steps.rb
 - features/step_definitions/filtering_steps.rb
 - lib/dover_to_calais.rb
+- lib/dover_to_calais/models.rb
 - lib/dover_to_calais/ontology.rb
 - lib/dover_to_calais/version.rb
 - test/test_file_1.doc
@@ -202,8 +285,10 @@ signing_key:
 specification_version: 4
 summary: An easy-to-use wrapper round the OpenCalais semantic analysis web service.
 test_files:
+- features/data_mining.feature
 - features/data_sources.feature
 - features/filtering.feature
+- features/step_definitions/data_mining_steps.rb
 - features/step_definitions/data_sources_steps.rb
 - features/step_definitions/filtering_steps.rb
 - test/test_file_1.doc