company-mapping 0.1.0

data/lib/company/mapping/tfidf/tf/normalized_term_frequency.rb

@@ -0,0 +1,27 @@
+module Company
+  module Mapping
+
+# NormalizedTermFrequency inherits from TermFrequency class. It calculates Term Frequency as
+# logarithmically scaled frequency: tf(t,d) = 1 + log(f(t,d)), or zero if
+# f(t,d) is zero.
+
+    class NormalizedTermFrequency < TermFrequency
+
+      def calculate(text)
+        _rawTF = rawFrequency(text)
+        _logTF = Hash.new
+
+        _rawTF.each do |key, value|
+          _logTF[key] = 1.0 + Math.log(value)
+        end
+
+        return _logTF
+      end
+
+      def info
+        return "Logarithmically scaled term frequency: tf(t,d) = 1 + log(f(t,d)), or zero if ft,d is zero";
+      end
+    end
+
+  end
+end

data/lib/company/mapping/tfidf/tf/term_frequency.rb

@@ -0,0 +1,37 @@
+module Company
+  module Mapping
+
+    class TermFrequency
+
+      def initialize(tokenizer)
+        @tokenizer = tokenizer
+      end
+
+      #Calculates the raw term frequency given the contents of the document.
+      def calculate(text)
+        return rawFrequency(text)
+      end
+
+      def info
+        return "Raw term frequency (number of times a token appears in a given string - document)"
+      end
+
+      protected
+      def rawFrequency(contents)
+        _tokens = @tokenizer.tokenize(contents)
+        _tf = Hash.new
+
+        _tokens.each {
+            |_token|
+          if (!_tf.has_key?(_token))
+            _tf[_token] = 1
+          else
+            _tf[_token] = _tf[_token] + 1
+          end
+        }
+        return _tf
+      end
+    end
+
+  end
+end

data/lib/company/mapping/tfidf/tfidf.rb

@@ -0,0 +1,81 @@
+module Company
+  module Mapping
+
+#TFIDF class implements Term Frequency Inverse Document Frequency statistic.
+    class TFIDF
+      attr_accessor :tf, :idf
+
+      def initialize(corpus)
+        @corpus = corpus
+      end
+
+      #Calculates the tf-idf weights in the given corpus
+      def calculate
+        @tfidf = Hash.new
+
+        if (@idf==nil)
+          @idf = InverseDocumentFrequency.new(@corpus)
+        end
+
+        if (@tf==nil)
+          _tokenizer = BasicTokenizer.new
+          @tf = NormalizedTermFrequency.new(_tokenizer)
+        end
+
+        @idf_weights = @idf.calculate
+
+        @corpus.each {
+            |doc|
+
+          _termfreq = @tf.calculate(doc.contents)
+
+          _tfidf_weights = Hash.new
+
+          _termfreq.each do |term, tf|
+            _weight = tf * @idf_weights[term]
+            _tfidf_weights[term] = _weight
+          end
+
+          @tfidf[doc.id] = _tfidf_weights
+        }
+        return @tfidf
+      end
+
+      #Calculates tfidf weights of new incoming document without importing the document in the corpus and re-calculating the tf-idf weights for the entire corpus
+      def calculate_tfidf_weights_of_new_document(new_doc)
+        _termfreq = @tf.calculate(new_doc.contents)
+
+        _tfidf_weights = Hash.new
+
+        _termfreq.each do |term, tf|
+          if (@idf_weights.has_key? term)
+            _weight = tf * @idf_weights[term]
+          else
+            _weight = tf * @idf.maxIDF
+          end
+          _tfidf_weights[term] = _weight
+        end
+        @tfidf[new_doc.id] = _tfidf_weights
+        return @tfidf
+      end
+
+      #Calculates tf-idf similarity between two given documents. It is actually
+      #the calculated Cosine Similarity by using tf*idf weights.
+      def similarity(doc1_id, doc2_id)
+        if (@tfidf==nil)
+          calculate
+        end
+
+        _cosine_similarity = CosineSimilarity.new
+        return _cosine_similarity.calculate(@tfidf[doc1_id], @tfidf[doc2_id])
+      end
+
+      def info
+        " term frequency–inverse document frequency, is a numerical "
+        +"statistic that is intended to reflect how important a word "
+        +"is to a document in a collection or corpus"
+      end
+    end
+
+  end
+end

data/lib/company/mapping/vector_similarity/cosine_similarity.rb

@@ -0,0 +1,47 @@
+module Company
+  module Mapping
+
+#Implements Cosine Similarity between two non zero vectors and it measures the cosine of the angle between them.
+    class CosineSimilarity
+
+      #Calculates cosine similarity between two documents. The documents are expressed as vectors of tokens (bag of words model).
+      def calculate(doc1, doc2)
+        return (dotProduct(doc1, doc2) / (Math.sqrt(d(doc1)) * Math.sqrt(d(doc2)))).round(4)
+      end
+
+      def info
+        "Implements Cosine Similarity between two non zero vectors and it measures the cosine of the angle between them.\n"
+        +"More info about Cosine Similarity can be found: https://en.wikipedia.org/wiki/Cosine_similarity";
+      end
+
+      protected
+      #Calculated the dot product between the two document vectors. The dot product is an algebraic operation
+      # that takes two equal-length sequences of numbers (usually coordinate vectors) and returns a single number.
+      def dotProduct(doc1, doc2)
+        _common_tokens = findCommonTokens(doc1.keys, doc2.keys)
+        _dot_product = 0.0
+
+        _common_tokens.each do |token|
+          _dot_product += doc2[token] * doc1[token]
+        end
+        return _dot_product
+      end
+
+      #Calculates the magnitude of a vector document
+      def d(doc)
+        _d = 0.0
+        doc.keys.each do |term|
+          _d += doc[term]**2.0
+        end
+        return _d
+      end
+
+      #returns the set of common tokens between two document vectors
+      def findCommonTokens(doc1_tokens, doc2_tokens)
+        common_tokens = Set.new doc1_tokens
+        return common_tokens.intersection(Set.new doc2_tokens)
+      end
+    end
+
+  end
+end

data/lib/company/mapping/version.rb

@@ -0,0 +1,5 @@
+module Company
+  module Mapping
+    VERSION = "0.1.0"
+  end
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: company-mapping
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- vasgat
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-03-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+description: Given a Corpus of WikiRate Company Names and a new (incoming) Company
+  Name, CompanyMapper class, finds the closest match if exists based on the calculated
+  tf-idf similarity and a defined threshold.
+email:
+- vasgat@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".idea/company-mapping.iml"
+- ".idea/misc.xml"
+- ".idea/modules.xml"
+- ".idea/workspace.xml"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- company-mapping.gemspec
+- lib/company/mapping.rb
+- lib/company/mapping/company_mapper.rb
+- lib/company/mapping/document_utils/basic_tokenizer.rb
+- lib/company/mapping/document_utils/corpus.rb
+- lib/company/mapping/document_utils/text_document.rb
+- lib/company/mapping/tfidf/idf/inverse_document_frequency.rb
+- lib/company/mapping/tfidf/tf/normalized_term_frequency.rb
+- lib/company/mapping/tfidf/tf/term_frequency.rb
+- lib/company/mapping/tfidf/tfidf.rb
+- lib/company/mapping/vector_similarity/cosine_similarity.rb
+- lib/company/mapping/version.rb
+homepage: https://github.com/vasgat/company-mapping
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: Maps new companies with those in a given corpus.
+test_files: []