semaphore_classification 0.1.0

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010 Gemini SBS, LLC
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,156 @@
+= Semaphore Classification
+
+Ruby wrapper around the Semaphore Classification Server (CS) API.
+
+== Usage
+
+Before you can classify documents, you must set the path to your CS:
+
+  Semaphore::Client.set_realm(<uri_to_classification_server>)
+  
+To classify documents:
+
+  Semaphore::Client.classify(<uri_to_document>, [options])
+
+=== Semaphore::Client.classify() Options
+
+==== :document_uri (required)
+
+This may use the following protocols (FTP, FTPS, HTTP, HTTPS). For example: http://mybucket.s3.amazonaws.com/some_file.pdf
+Supported document types include: Microsoft Office files, Lotus files, OpenOffice files, PDFs, WordPerfect docs, HTML docs and most other common file formats.
+The document type will be automatically identified by the CS.
+
+==== :title (optional)
+
+*Value:* String
+  
+The title in the request is used mainly for classification of documents held by a content management system.
+
+*Default:* none
+
+==== :alternate_body (optional)
+
+*Value:* String
+
+This will be used to classify on if the document fails to be retrieved by the CS for some reason.
+
+*Default:* none
+
+==== :article_mode (optional)
+
+*Value:* :single or :multi
+
+:single will process the document in 1 large chunk.  This will mean that evidence from all parts of the document are considered at the same time.  Depending on the design of the rulenet this may increase the chance of mis-classifications.  Singlearticle may also require large amounts of memory (or if this is restricted, large amounts of time) due to the size of evidence tables which have to be evaluated.
+
+:multi will attempt to split the document into "articles" so that the rules only consider evidence within an article and then clustering is applied to calculate which categories are representative for the document as a whole rather than simply for an article.
+
+*Default:* :multi
+
+==== :debug (optional)
+
+*Value:* true or false
+
+Will return the article(s) as well as rule matches in the response. Useful for troubleshooting, but results in large responses.
+
+*Default:* false
+
+==== :clustering_type (optional)
+
+*Value:* [:all, :average, :average_scored_only, :common_scored_only, :common, :rms_scored_only, :rms, :none]
+
+Clustering type specifies the type of calculation to use in deriving the document level scores from the article scores. This only applies to multiarticle style classifications.
+
+*Default:* :rms_scored_only
+
+==== :clustering_threshold (optional)
+
+*Value:* [0-100]
+
+The clustering threshold is only used in multiarticle mode. When the clustering algorithm is selected, the result is checked against this threshold and a score is only promoted to document level if it is >= this value.
+
+*Default:* 48
+
+==== :threshold (optional)
+
+*Value:* [0-100]
+
+The threshold is used to decide at what level of significance a category rule will fire. 
+
+The score (or significance if you prefer) varies between 0 and 100 sometimes this is displayed as 0.00 - 1.00 depending on whether it is used for integer calculations (0-100) or for statistical floating point operations (0.00 - 1.00 ie a normalised value is generally better here).
+
+*Default:* 48
+
+==== :language (optional)
+
+*Value:* [:english, :english_marathon_stemmer, :english_morphological_stemmer, :english_morph_and_derivational_stemmer, :french, :italian, :german, :spanish, :dutch, :portuguese, :danish, :norwegian, :swedish, :arabic]
+
+Note: for Standard Language processing only English has multiple stemmers available - The other languages supported only have Marathon stemmer available.
+
+*Default:* :english_marathon_stemmer
+
+==== :generated_keys (optional)
+
+*Value:* true or false
+
+Using generated keys will mean that all rules will have a unique key (which is simply the index of the rule in the rulenet). 
+
+*Default:* true
+
+==== :min_avg_article_page_size (optional)
+
+*Value:* Decimal
+
+The minimum average article page size is only relevant in multi article mode
+
+For documents which contain page information (ie not html and other continuous formats) the count of pages in the document is used to check whether automatic splitting has provided a sensible result.  If the number of articles made multiplied by this value is greater than the count of pages in the document then CS will assume that the splitting does not make sense for this document and will revert back to a single article.
+
+The idea is that this gives an easy to use approximate measure for checking splitting - ie a min average article page size of 1 means that on average we want 1 article to be bigger than a single page so if a document of 10 pages splits into 20 articles then we probably have a bad statistical split so classifying as a single article will give better results
+
+*Default:* 1.0
+
+==== :character_cutoff (optional)
+
+*Values:* FixNum
+
+The character count cutoff is a mechanism for avoiding errors or lengthy classification times on large documents. 
+
+if the corpus of documents that is to be classified is likely to include junk (eg automatically generated log files from SQL servers etc) then a cutoff can make sense.
+
+A value of 0 means no cutoff action is performed.
+
+The cutoff defines the approximate size (in characters) after which CS will stop parsing the data.  The value is used as an approximation so various parsers implement this behaviour in different manners - eg pdf documents are cutoff at the next page boundary when this limit is reached, word documents are cut off after the next complete word etc.
+
+Measuring in characters appears to be the most sensible unit here since this cutoff is applied fairly early on in the processing of a document - at this point CS may not yet know what language the document is in so possibly cannot assume that the language is space seperated words - let alone count the number of sentences or paragraphs yet.
+
+Other multi document type parsing systems (for example the simple parser included with google search appliance) often have a cutoff value (which is generally not configurable) defined in terms of the file size - however a word document with many embedded pictures may have a very large physical size but a relatively small number of characters so would be able to be classified perfectly well - hence the choice of a cutoff defined in terms of number of characters.
+
+Generally this value is set high enough that any reasonable document (ie one produced by a person) will be fully considered so a value of 1/2 a million is realistic - automatically generated text files which cause lengthy classification times often have more than this characters but the information is very rarely of any use to an end user.
+
+*Default:* 500000
+
+==== :document_score_limit (optional)
+
+*Value:* FixNum
+
+The document level score limit is a mechanism for restricting the document level classifications to the top-N results only.
+
+This is generally not a good idea since the confidence of a classification is meant to provide an absolute measure of the confidence of a particular classification across documents.  That is a higher confidence means that the document is more likely to be "about" or "mainly concerned with" the particular topic.  So if document 1 classifies category "A" with a confidence of 0.75 whilst document 2 has a confidence of 0.6 for category "A" then document 1 should be returned before document 2 (if this is a search style installation rather than a conformance checking).  If whilst processing document 1 the classification of "A" is discarded since document 1 has too many higher confidence classifications then the results across the corpus are skewed and search will not be as accurate.
+
+However some systems have rather strict limits on the number of "tags" or "meta information" which a particular document is allowed to contain - when CS is integrated with one of these systems it is probably better to allow CS pick the top-N rather than having this code at the integration layer.
+
+Currently the implementation is pretty simplistic (will return N or less scores sorted by the confidence) so could easily be implemented in the integration layer but it is possible that further work could go here so that CS could check particular categories or classes of categories in a specific rulenet defined manner so that "important" classes of categorisations (though with a low confidence) are not excluded by large numbers of higher confidence classifications in some less important class of rules.
+
+*Default:* 0
+
+== Dependencies
+
+* {nokogiri}[http://github.com/tenderlove/nokogiri] (HTML, XML, SAX and Reader parser)
+* {curb}[http://github.com/taf2/curb] (Ruby wrapper around the great {libcurl}[http://curl.haxx.se/])
+
+== Copyright
+
+Copyright (c) 2010 Gemini SBS. See LICENSE for details.
+
+== Authors
+
+* {Mauricio Gomes}[http://github.com/mgomes]

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/semaphore.rb

@@ -0,0 +1,2 @@
+# So you can require "semaphore" instead of "semaphore_classification"
+require "semaphore_classification"

data/lib/semaphore_classification/client.rb

@@ -0,0 +1,52 @@
+module Semaphore
+
+  class Client
+    
+    LANGUAGES = { :english => "en", :english_marathon_stemmer => "en1", :english_morphological_stemmer => "en2", :english_morph_and_derivational_stemmer => "en3", 
+                  :french => "fr", :italian => "it", :german => "de", :spanish => "es", :dutch => "nl", :portuguese => "pt", :danish => "da", :norwegian => "no", 
+                  :swedish => "sv", :arabic => "ar" 
+                  }
+                  
+    CLUSTERING_TYPES = { :all => "ALL", :average => "AVERAGE_INCLUDING_EMPTY", :average_scored_only => "AVERAGE", :common_scored_only => "COMMON", 
+                         :common => "COMMON_INCLUDING_EMPTY", :rms_scored_only => "RMS", :rms => "RMS_INCLUDING_EMPTY", :none => "NONE"
+                         }
+    
+    @@default_options = { :title => "", :alternate_body => "", :debug => false, :clustering_type => CLUSTERING_TYPES[:rms_scored_only], :clustering_threshold => 48,
+                          :threshold => 48, :language => LANGUAGES[:english_marathon_stemmer], :generated_keys => true, :min_avg_article_page_size => 1.0, 
+                          :character_cutoff => 500000, :document_score_limit => 0, :article_mode => :multi
+                          }
+    @@connection = nil
+
+    class << self
+      
+      def set_realm(realm, proxy=nil)
+        @@connection = Connection.new(realm, proxy)
+      end
+
+      def classify(document_uri, *args)
+        options = extract_options!(args)
+        options[:document_uri] = document_uri
+        
+        result = post @@default_options.merge(options)
+      end
+      
+    private
+    
+      def post(data)
+        raise RealmNotSpecified if @@connection.nil?
+        @@connection.post data
+      end
+    
+      def extract_options!(args)
+        if args.last.is_a?(Hash)
+          return args.pop
+        else
+          return {}
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/semaphore_classification/connection.rb

@@ -0,0 +1,94 @@
+module Semaphore
+
+  class Connection
+
+    attr_reader :realm
+
+    def initialize(realm, proxy=nil)
+      @realm = realm
+      @proxy = proxy
+    end
+
+    def post(data)
+      request :post, data
+    end
+    
+  private
+
+    def request(method, data)
+      response = send_request method, construct_document(data)
+
+      deconstruct_document(response)
+    end
+
+    def send_request(method, data)
+      RestClient.proxy = @proxy unless @proxy.nil?
+      
+      begin
+        response = RestClient.post @realm, :XML_INPUT => data, :multipart => true
+      rescue => e
+        raise_errors(e.response)
+      end
+
+      response
+    end
+    
+    def construct_document(data)
+      builder = Nokogiri::XML::Builder.new do |xml|
+        xml.request(:op => "#{ data[:debug] ? 'TEST' : 'CLASSIFY' }") {
+          xml.document {
+            xml.title data[:title] unless data[:title].empty?
+            xml.path data[:document_uri] 
+            xml.body data[:alternate_body] unless data[:alternate_body].empty?
+            case data[:article_mode]
+              when :multi
+                xml.multiarticle
+              when :single
+                xml.singlearticle
+            end
+            xml.feedback if data[:debug]
+            xml.use_generated_keys if data[:generated_keys]
+            xml.clustering(:type => data[:clustering_type], :threshold => data[:clustering_threshold])
+            xml.language data[:language]
+            xml.threshold data[:threshold]
+            xml.min_average_article_pagesize data[:min_avg_article_page_size]
+            xml.char_count_cutoff data[:character_cutoff]
+            xml.document_score_limit data[:document_score_limit]
+          }
+        }
+      end
+      
+      builder.to_xml
+    end
+    
+    def deconstruct_document(response)
+      data = Array.new
+      
+      if !response.body.empty?
+        begin
+          doc = Nokogiri::XML.parse(response)
+          doc.xpath('//META').each do |node|
+            data << { :term => node['value'], :key => node['key'], :score => node['score'] } if node['name'] == "Generic"
+          end
+        rescue
+          raise DecodeError, "content: <#{response.body}>"
+        end
+      end
+      
+      data.uniq
+    end
+
+    def raise_errors(response)
+      case response.code
+        when 500
+          raise ServerError, "Semaphore Classification Server had an internal error. #{response.description}\n\n#{response.body}"
+        when 502..503
+          raise Unavailable, response.description
+        else
+          raise SemaphoreError, response.description
+      end
+    end
+
+  end
+
+end

data/lib/semaphore_classification.rb

@@ -0,0 +1,19 @@
+require 'uri'
+require 'nokogiri'
+require 'rest_client'
+
+require 'semaphore_classification/connection'
+require 'semaphore_classification/client'
+
+module Semaphore
+  VERSION = File.read(File.join(File.dirname(__FILE__), '..', 'VERSION'))
+  
+  class SemaphoreError < StandardError; end
+  class InsufficientArgs < SemaphoreError; end
+  class Unauthorized < SemaphoreError; end
+  class NotFound < SemaphoreError; end
+  class ServerError < SemaphoreError; end
+  class Unavailable < SemaphoreError; end
+  class DecodeError < SemaphoreError; end
+  class RealmNotSpecified < SemaphoreError; end
+end

metadata

@@ -0,0 +1,122 @@
+--- !ruby/object:Gem::Specification 
+name: semaphore_classification
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Mauricio Gomes
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-08-27 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 113
+        segments: 
+        - 1
+        - 4
+        - 3
+        - 1
+        version: 1.4.3.1
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rest-client
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 15
+        segments: 
+        - 1
+        - 6
+        - 0
+        version: 1.6.0
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 13
+        segments: 
+        - 1
+        - 2
+        - 9
+        version: 1.2.9
+  type: :development
+  version_requirements: *id003
+description: Ruby wrapper around the Semaphore Classification Server API.
+email: mauricio@geminisbs.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- LICENSE
+- README.rdoc
+- VERSION
+- lib/semaphore.rb
+- lib/semaphore_classification.rb
+- lib/semaphore_classification/client.rb
+- lib/semaphore_classification/connection.rb
+has_rdoc: true
+homepage: http://github.com/geminisbs/semaphore_classification
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Ruby wrapper around the Semaphore Classification Server
+test_files: []
+