categorize 0.0.1 → 0.0.2

data/lib/{constants.rb → categorize/constants.rb}

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Constants
   module Words
     # only include words > 2 chars

data/lib/{categorize.rb → categorize/model.rb}

@@ -1,9 +1,8 @@
-require File.join(File.dirname(__FILE__), 'models', 'bag_of_words')
-require File.join(File.dirname(__FILE__), 'constants')
+# encoding: utf-8
 
 module Categorize
   MIN_WORD_LENGTH = 3
-  @bag_of_words = BagOfWords.new
+  @bag_of_words = Models::BagOfWords.new
 
   class << self
     #include Bow
@@ -24,7 +23,7 @@ module Categorize
       strings.map { |s| preprocess(s) }
       #ret = model_bow(array_of_tokens);
       count = 0
-      ret.inject({}) do |hash, term|
+      ret.reduce({}) do |hash, term|
         hash[term] ||= []
         hash[term] << count += 1
         hash
@@ -39,8 +38,9 @@ module Categorize
       end
 
       def preprocess(string)
-        string.split(Constants::Words::SPLIT_REGEX).map(&:downcase).delete_if do
-          |word|
+        split_lower_strings = string.split(
+            Constants::Words::SPLIT_REGEX).map(&:downcase)
+        split_lower_strings.delete_if do |word|
           word.length < MIN_WORD_LENGTH ||
             Constants::Words::COMMON.include?(word)
         end

data/lib/categorize/models/bag_of_words.rb

@@ -0,0 +1,112 @@
+# encoding: utf-8
+
+require 'categorize/utils/grams'
+
+module Categorize
+  module Models
+    class BagOfWords
+      include Utils::Grams
+
+      # DEBUG = false
+      # TODO: some gradient descent to choose this number
+      # 0 <= MIN_SUPP <= 1, we like 0.01 <= MIN_SUPP <= 0.1
+      MIN_SUPP_L = 0.07
+      MIN_SUPP_H = 0.1
+      NUM_TOP_GRAMS = 250
+      MAX_BUCKETS = 8
+
+      # function worst case
+      # O(2 x (|frequent_grams| x |gram_collections|) +
+      #   |all_grams| + MAX_BUCKETS x |gram_collections|)
+      def model(query, records_to_tokens)
+        @gram_cover_cache = {}
+        @gram_collections, @all_grams = create_grams(query, records_to_tokens)
+
+        top_grams = determine_frequency_term_sets(@all_grams, query)
+        top_grams = top_grams.keys.sort do |gram_c1, gram_c2|
+          top_grams[gram_c1] <=> top_grams[gram_c2]
+        end.first(MAX_BUCKETS)
+
+        # below block, worst case O(MAX_BUCKETS x |gram_collections|)
+        @gram_collections.reduce({}) do |buckets, gram_collection|
+          max_fitness = 0
+          max_fit = nil
+          top_grams.each do |top_gram|
+            # the >= removes the 'none' possibility
+            if gram_collection.fitness[top_gram] &&
+                gram_collection.fitness[top_gram] >= max_fitness
+              max_fitness = gram_collection.fitness[top_gram]
+              max_fit = top_gram
+            end
+          end
+          buckets[max_fit] ||= []
+          buckets[max_fit] << gram_collection.content
+          buckets
+        end
+      end
+
+      # ==== Return
+      # Hash - fitness => [gram_collection, ...]
+      # function worst case O(2 x (|frequent_grams| x |gram_collections|) +
+      #                            |all_grams|)
+      def determine_frequency_term_sets(all_grams, query)
+        # only count a result if it has non-0 words length
+        effective_length = @gram_collections.reject do |result|
+          result.grams.nil? || result.grams.empty?
+        end.length
+
+        min_cover_l = MIN_SUPP_L * effective_length
+        # min_cover_h = MIN_SUPP_H * effective_length
+
+        # for speed only look at top N grams
+        # below block, worst case O(|all_grams|)
+        frequent_grams = all_grams.sort do |gram1, gram2|
+          gram2.frequency <=> gram1.frequency
+        end.first(NUM_TOP_GRAMS)
+
+        # below block, worst case O(|frequent_grams| x |gram_collections|)
+        frequent_grams = frequent_grams.delete_if do |gram|
+          !cover(gram, min_cover_l)
+        end
+
+        # below block, worst case O(|frequent_grams| x |gram_collections|)
+        @gram_collections.reduce(Hash.new(0)) do |top_grams, gram_collection|
+          max_fitness = 0
+          max_fit = nil
+
+          frequent_grams.each do |gram|
+            content_frequency = (
+              gram_collection.content_to_frequency[gram.content] || 0)
+            fitness = content_frequency / gram.frequency.to_f
+            gram_collection.fitness[gram.content] = fitness
+
+            if fitness > max_fitness
+              max_fitness = fitness
+              max_fit = gram.content
+            end
+          end
+
+          # puts "#{max_fit}: #{max_fitness}"# if DEBUG
+          top_grams[max_fit] += 1 if max_fit
+          top_grams
+        end
+      end
+
+      # function worstcase O(#gram_collections)
+      def cover(gram, min_length)
+        ((cached = @gram_cover_cache[gram]) != nil) and return cached
+        count = 0
+
+        @gram_collections.each do |gram_collection|
+          frequency = gram_collection.content_to_frequency[gram.content]
+          if !frequency.nil? && frequency > 0
+            count += 1
+            return @gram_cover_cache[gram] = true if count >= min_length
+          end
+        end
+
+        @gram_cover_cache[gram] = false
+      end
+    end
+  end
+end

data/lib/categorize/utils/grams.rb

@@ -0,0 +1,46 @@
+# encoding: utf-8
+
+module Categorize
+  module Utils
+    module Grams
+      def create_grams(query, records_to_words)
+        all_grams = []
+        @query = query
+        @query_terms = query.split.map(&:downcase).map(&:strip)
+        @query_alt = "#{@query_terms[1..-1]} #{@query_terms[0]}"
+
+        invalid = Proc.new do |gram, *args|
+          # remove [[gram]] if == [[query]]
+          gram == @query || gram == @query_alt || @query_terms.include?(gram)
+        end
+
+        gram_collections = records_to_words.map do |record, words|
+          gram_collection = GramCollection.new(record, words, invalid)
+          all_grams += gram_collection.grams
+          gram_collection
+        end
+        return gram_collections, make_grams_unique(all_grams)
+      end
+
+      def check_plurals(frequent_grams)
+        # if exists [[gram]] and [[gram]]s then remove [[gram]]s
+        frequent_grams_contents = frequent_grams.map(&:content)
+        frequent_grams.delete_if do |gram|
+          gram.content[-1] == 's' and
+            frequent_grams_contents.include?(gram.content[0...-1])
+        end
+      end
+
+      def make_grams_unique(grams)
+        grams.reduce({}) do |hash, gram|
+          if hash[gram.content]
+            hash[gram.content].frequency += gram.frequency
+          else
+            hash[gram.content] = gram
+          end
+          hash
+        end.values
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: categorize
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -18,10 +18,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/categorize.rb
-- lib/constants.rb
-- lib/models/bag_of_words.rb
-- lib/utils/grams.rb
+- lib/categorize/model.rb
+- lib/categorize/constants.rb
+- lib/categorize/models/bag_of_words.rb
+- lib/categorize/utils/grams.rb
 homepage: http://www.helioid.com/
 licenses: []
 post_install_message: 
@@ -42,7 +42,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Text categorization library

data/lib/models/bag_of_words.rb

@@ -1,97 +0,0 @@
-require File.join(File.dirname(__FILE__), '..', 'utils', 'grams')
-
-class BagOfWords
-  include ::Utils::Grams
-
-  # DEBUG = false
-  # TODO: some gradient descent to choose this number
-  # 0 <= MIN_SUPP <= 1, we like 0.01 <= MIN_SUPP <= 0.1
-  MIN_SUPP_L = 0.07
-  MIN_SUPP_H = 0.1
-  NUM_TOP_GRAMS = 250
-  MAX_BUCKETS = 8
-
-  # function worst case
-  # O(2 x (#frequent_grams x #gram_collections) + #all_grams + MAX_BUCKETS x #gram_collections)
-  def model(query, records_to_tokens)
-    @gram_cover_cache = {}
-    @gram_collections, @all_grams = create_grams(query, records_to_tokens)
-
-    top_grams = determine_frequency_term_sets(@all_grams, query)
-    top_grams = top_grams.keys.sort do |gram_c1, gram_c2|
-      top_grams[gram_c1] <=> top_grams[gram_c2]
-    end.first(MAX_BUCKETS)
-
-    # below block, worst case O(MAX_BUCKETS x #gram_collections)
-    @gram_collections.inject({}) do |buckets, gram_collection|
-      max_fitness = 0
-      max_fit = nil
-      top_grams.each do |top_gram|
-        # the >= removes the 'none' possibility
-        if gram_collection.fitness[top_gram] && gram_collection.fitness[top_gram] >= max_fitness
-          max_fitness = gram_collection.fitness[top_gram]
-          max_fit = top_gram
-        end
-      end
-      buckets[max_fit] ||= []
-      buckets[max_fit] << gram_collection.content
-      buckets
-    end
-  end
-
-  # ==== Return
-  # Hash - fitness => [gram_collection, ...]
-  # function worst case O(2 x (#frequent_grams x #gram_collections) + #all_grams)
-  def determine_frequency_term_sets(all_grams, query)
-    # only count a result if it has non-0 words length
-    effective_length = @gram_collections.reject do |result|
-      result.grams.nil? || result.grams.empty?
-    end.length
-    
-    min_cover_l = MIN_SUPP_L * effective_length 
-    # min_cover_h = MIN_SUPP_H * effective_length 
-
-    # for speed only look at top N grams
-    # below block, worst case O(#all_grams)
-    frequent_grams = all_grams.sort do |gram1, gram2|
-      gram2.frequency <=> gram1.frequency
-    end.first(NUM_TOP_GRAMS)
-
-    # below block, worst case O(#frequent_grams x #gram_collections)
-    frequent_grams = frequent_grams.delete_if do |gram|
-      !cover(gram, min_cover_l)
-    end
-
-    # below block, worst case O(#frequent_grams x #gram_collections)
-    @gram_collections.inject(Hash.new(0)) do |top_grams, gram_collection|
-      max_fitness = 0
-      max_fit = nil
-
-      frequent_grams.each do |gram|
-        fitness = gram_collection.fitness[gram.content] = (gram_collection.content_to_frequency[gram.content] || 0) / gram.frequency.to_f
-        if fitness > max_fitness
-          max_fitness = fitness
-          max_fit = gram.content
-        end
-      end
-
-      # puts "#{max_fit}: #{max_fitness}"# if DEBUG
-      top_grams[max_fit] += 1 if max_fit
-      top_grams
-    end
-  end
-
-  # function worstcase O(#gram_collections)
-  def cover(gram, min_length)
-    ((cached = @gram_cover_cache[gram]) != nil) and return cached
-    count = 0
-    @gram_collections.each do |gram_collection|
-      frequency = gram_collection.content_to_frequency[gram.content]
-      if !frequency.nil? && frequency > 0
-        count += 1
-        return @gram_cover_cache[gram] = true if count >= min_length
-      end
-    end
-    @gram_cover_cache[gram] = false
-  end
-end

data/lib/utils/grams.rb

@@ -1,45 +0,0 @@
-require File.join(File.dirname(__FILE__), 'gram_collection')
-require File.join(File.dirname(__FILE__), 'gram_node')
-
-module Utils
-  module Grams
-    def create_grams(query, records_to_words)
-      all_grams = []
-      @query = query
-      @query_terms = query.split.map(&:downcase).map(&:strip)
-      @query_alt = "#{@query_terms[1..-1]} #{@query_terms[0]}"
-
-      invalid = Proc.new do |gram, *args|
-        # remove [[gram]] if == [[query]]
-        gram == @query || gram == @query_alt || @query_terms.include?(gram)
-      end
-
-      gram_collections = records_to_words.map do |record, words|
-        gram_collection = GramCollection.new(record, words, invalid)
-        all_grams += gram_collection.grams
-        gram_collection
-      end
-      return gram_collections, make_grams_unique(all_grams)
-    end
-
-    def check_plurals(frequent_grams)
-      # if exists [[gram]] and [[gram]]s then remove [[gram]]s
-      frequent_grams_contents = frequent_grams.map(&:content)
-      frequent_grams.delete_if do |gram|
-        gram.content[-1] == 's' and
-          frequent_grams_contents.include?(gram.content[0...-1])
-      end
-    end
-
-    def make_grams_unique(grams)
-      grams.inject({}) do |hash, gram|
-        if hash[gram.content]
-          hash[gram.content].frequency += gram.frequency
-        else
-          hash[gram.content] = gram
-        end
-        hash
-      end.values
-    end
-  end
-end