groupie 0.2.2 → 0.5.0

data/bin/setup

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -x
+
+bundle install
+bundle clean
+
+# Do any other automated setup that you need to do here

data/groupie.gemspec

@@ -1,63 +1,38 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
-# -*- encoding: utf-8 -*-
+# frozen_string_literal: true
 
-Gem::Specification.new do |s|
-  s.name = %q{groupie}
-  s.version = "0.2.2"
+require_relative 'lib/groupie/version'
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Wes Oldenbeuving"]
-  s.date = %q{2010-07-25}
-  s.description = %q{Group and classify text based on likelyhood of being included in a text of a specific category}
-  s.email = %q{narnach@gmail.com}
-  s.extra_rdoc_files = [
-    "LICENSE"
-  ]
-  s.files = [
-    ".document",
-     "LICENSE",
-     "Rakefile",
-     "VERSION",
-     "groupie.gemspec",
-     "lib/groupie.rb",
-     "lib/groupie/core_ext/string.rb",
-     "lib/groupie/group.rb",
-     "readme.rdoc",
-     "spec/fixtures/ham/email_ham1.txt",
-     "spec/fixtures/ham/spam.la-44116217.txt",
-     "spec/fixtures/spam/email_spam1.txt",
-     "spec/fixtures/spam/email_spam2.txt",
-     "spec/fixtures/spam/spam.la-44118014.txt",
-     "spec/groupie/core_ext/string_spec.rb",
-     "spec/groupie/group_spec.rb",
-     "spec/groupie_spec.rb",
-     "spec/spec_helper.rb"
-  ]
-  s.homepage = %q{http://github.com/Narnach/groupie}
-  s.rdoc_options = ["--charset=UTF-8"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
-  s.summary = %q{Group and classify text}
-  s.test_files = [
-    "spec/groupie/core_ext/string_spec.rb",
-     "spec/groupie/group_spec.rb",
-     "spec/groupie_spec.rb",
-     "spec/spec_helper.rb"
-  ]
+Gem::Specification.new do |spec|
+  spec.name = 'groupie'
+  spec.version = Groupie::VERSION
+  spec.authors = ['Wes Oldenbeuving']
+  spec.email = ['wes@narnach.com']
 
-  if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 3
+  spec.summary = 'Library to help you group texts and classify new ones'
+  spec.description = 'Groupie is a simple way to group texts and classify new texts as being a likely member' \
+                     ' of one of the defined groups. Think of bayesian spam filters.'
+  spec.homepage = 'https://github.com/Narnach/groupie'
+  spec.license = 'MIT'
+  spec.required_ruby_version = '>= 2.6.0' # EOL for 2.6 is 2022-03-31, so support this as the minimum for now
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<testy>, [">= 0"])
-    else
-      s.add_dependency(%q<testy>, [">= 0"])
-    end
-  else
-    s.add_dependency(%q<testy>, [">= 0"])
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/Narnach/groupie'
+  spec.metadata['changelog_uri'] = 'https://github.com/Narnach/groupie/blob/stable/CHANGELOG.md'
+  # Require multi-factor authentication to publish the gem
+  spec.metadata['rubygems_mfa_required'] = 'true'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
   end
-end
+  # spec.bindir        = "exe"
+  # spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
 
+  # Uncomment to register a new dependency of your gem
+  # spec.add_dependency "example-gem", "~> 1.0"
+
+  # For more information and examples about making a new gem, checkout our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end

data/lib/groupie/group.rb

@@ -1,10 +1,17 @@
+# frozen_string_literal: true
+
 class Groupie
+  # Group represents a group or category that words can be classified into.
   class Group
-    def initialize(name)
+    attr_reader :word_counts, :total_word_count
+
+    def initialize(name, groupie)
       @name = name
+      @groupie = groupie
       @word_counts = {}
+      @total_word_count = 0
     end
-    
+
     def words
       @word_counts.keys
     end
@@ -17,16 +24,23 @@ class Groupie
       nil
     end
 
+    alias << add
+
     # Return the count for a specific +word+.
     def count(word)
       @word_counts[word] || 0
     end
 
+    private
+
     # Add a single word and count it.
     def add_word(word)
       @word_counts[word] ||= 0
-      @word_counts[word] += 1
+      current_count = @word_counts[word] += 1
+      @total_word_count += 1
+      # If this word is new for this Group, it might be new for the entire Groupie
+      @groupie.add_word(word) if current_count == 1
+      nil
     end
-    private :add_word
   end
-end
+end

data/lib/groupie/version.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# This extends Groupie and adds a version number
+class Groupie
+  VERSION = '0.5.0'
+
+  def self.version
+    VERSION
+  end
+end

data/lib/groupie.rb

@@ -1,72 +1,166 @@
-lib_dir = File.expand_path(File.dirname(__FILE__))
-$LOAD_PATH.unshift(lib_dir) unless $LOAD_PATH.include?(lib_dir)
-require 'groupie/group'
-require 'groupie/core_ext/string'
+# frozen_string_literal: true
 
+require_relative 'groupie/version'
+require_relative 'groupie/group'
+require 'set'
+
+# Groupie is a text grouper and classifier, using naive Bayesian filtering.
 class Groupie
-  def initialize
+  # Wrap all errors we raise in this so our own errors are recognizable.
+  class Error < StandardError; end
+
+  attr_accessor :smart_weight
+
+  # @param [true, false] smart_weight (false) Whether smart weight is enabled or not.
+  def initialize(smart_weight: false)
     @groups = {}
+    @smart_weight = smart_weight
+    @known_words = Set.new
   end
 
+  # Turn a String (or anything else that responds to #to_s) into an Array of String tokens.
+  # This attempts to remove most common punctuation marks and types of whitespace.
+  #
+  # @param [String, #to_s] object
+  # @return [Array<String>]
+  def self.tokenize(object)
+    object
+      .to_s
+      .downcase
+      .gsub(/\s/, ' ')
+      .gsub(/[$']/, '')
+      .gsub(/<[^>]+?>|[^\w -.,]/, '')
+      .split.map { |str| str.gsub(/\A['"]+|[!,."']+\Z/, '') }
+  end
+
+  # Access an existing Group or create a new one.
+  #
+  # @param [Object] group The name of the group to access.
+  # @return [Groupie::Group] An existing or new group identified by +group+.
   def [](group)
-    @groups[group] ||= Group.new(group)
+    @groups[group] ||= Group.new(group, self)
   end
 
-  def classify(entry, strategy=:sum)
-    results = {}
-    total_count = @groups.inject(0) do |sum, name_group|
-      group = name_group.last
-      count = group.count(entry)
-      if strategy==:sum
-        sum += count
-      elsif strategy==:sqrt
-        sum += Math::sqrt(count)
-      elsif strategy==:log
-        sum += Math::log10(count) if count > 0
-      else
-        raise "Invalid strategy: #{strategy}"
-      end
-      next sum
+  # Classify a text by taking the average of all word classifications.
+  #
+  # @param [Array<String>] words List of words to be classified
+  # @param [Symbol] strategy
+  # @return [Hash<Object, Float>] Hash with <group, score> pairings. Scores are always in 0.0..1.0
+  # @raise [Groupie::Error] Raise when an invalid strategy is provided
+  def classify_text(words, strategy = :sum)
+    words &= unique_words if strategy == :unique
+    group_score_sums, hits = calculate_group_scores(words, strategy)
+
+    group_score_sums.each.with_object({}) do |(group, sum), averages|
+      averages[group] = hits.positive? ? sum / hits : 0
     end
-    return results if 0 == total_count
-
-    @groups.each do |name, group|
-      count = group.count(entry)
-      if strategy==:sum
-        # keep count
-      elsif strategy==:sqrt
-        count = Math::sqrt(count)
-      elsif strategy==:log
-        count = Math::log10(count) if count > 0
-      else
-        raise "Invalid strategy: #{strategy}"
-      end
-      results[name] = count > 0 ? count.to_f / total_count : 0.0
+  end
+
+  # Classify a single word against all groups, returning the probability distribution.
+  #
+  # @param [String] entry A word to be classified
+  # @param [Symbol] strategy (:sum) the strategy to use on the score
+  # @return [Hash<Object, Float>] Hash with <group, probability> pairings.
+  #   Probabilities are always in 0.0..1.0, and add up to 1.0 (i.e. it's a probability distribution)
+  # @raise [Groupie::Error] Raise when an invalid strategy is provided
+  def classify(entry, strategy = :sum)
+    # Calculate default weight once outside of the loop
+    default_weight = self.default_weight
+    # Each group calculates the count, then reduces it to a score: <group name, score>
+    per_group_score = @groups.transform_values do |group|
+      apply_count_strategy(default_weight + group.count(entry), strategy)
     end
-    return results
+    # When we have no scores, we have no results, so abort early
+    # Note that when smart_weight is enabled we always have a score.
+    total_score = per_group_score.values.sum
+    return {} if total_score.zero?
+
+    # Final results must be within 0.0..1.0, so divide each score by the total score
+    per_group_score.transform_values { |group_score| group_score.to_f / total_score }
   end
 
-  # Classify a text by taking the average of all word classifications.
-  def classify_text(words, strategy=:sum)
+  # Return a word score dictionary that excludes the 4th quartile most popular words.
+  # Why do this? So the most common (and thus meaningless) words are ignored
+  # and less common words gain more predictive power.
+  #
+  # This is used by the :unique strategy of the classifier.
+  #
+  # @return [Hash<String, Integer>]
+  def unique_words
+    # Iterate over all Groups and merge their <word, count> dictionaries into one
+    total_count = @groups.inject({}) do |total, (_name, group)|
+      total.merge!(group.word_counts) { |_key, o, n| o + n }
+    end
+    # Extract the word count that's at the top 75%
+    top_quartile_index = [((total_count.size * 3) / 4) - 1, 1].max
+    top_quartile_frequency = total_count.values.sort[top_quartile_index]
+    # Throw out all words which have a count that's above this frequency
+    total_count.reject! { |_word, count| count > top_quartile_frequency }
+    total_count.keys
+  end
+
+  # Default weight is used when +smart_weight+ is enabled.
+  # Each word's count is increased by the +default_weight+ value,
+  # which is the average frequency of each unique word we know about.
+  #
+  # Example: if we have indexed 1000 total words, of which 500 were unique,
+  #   the default_weight would be 1000/500=2.0
+  #
+  # @return [Float] The default weight for all words
+  def default_weight
+    # Default weight only applies when smart weight is enabled
+    return 0.0 unless smart_weight
+
+    # If we don't know any words, the weight is also zero
+    return 0.0 unless @known_words.any?
+
+    # Gather counts and calculate
+    total_words = @groups.each_value.sum(&:total_word_count)
+    total_unique_words = @known_words.count
+    total_words / total_unique_words.to_f
+  end
+
+  # Private method used by Groups to register known words with the Group.
+  def add_word(word)
+    @known_words << word
+  end
+
+  private
+
+  # Calculate grouped scores
+  #
+  # @param [Array<String>] words
+  # @param [Symbol] strategy
+  # @return [Array<Enumerator<String>, Integer>] a Hash with <group, score> pairs and an integer with the number of hits
+  def calculate_group_scores(words, strategy)
     hits = 0
-    group_score_sums = words.inject({}) do |results, word|
+    group_score_sums = words.each.with_object({}) do |word, results|
       word_results = classify(word, strategy)
       next results if word_results.empty?
-      hits += 1
-      results.merge(word_results) do |key, old, new|
-        old + new
-      end
-    end
 
-    averages={}
-    group_score_sums.each do |group, sum|
-      averages[group] = hits > 0 ? sum / hits : 0
+      hits += 1
+      results.merge!(word_results) { |_key, old, new| old + new }
     end
 
-    averages
+    [group_score_sums, hits]
   end
 
-  def self.version
-    File.read(File.join(File.dirname(File.expand_path(__FILE__)), "..", "VERSION")).strip
+  # Helper function to reduce a raw word count to a strategy-modified weight.
+  # @param [Integer] count
+  # @param [Symbol] strategy
+  # @return [Integer, Float]
+  # @raise [Groupie::Error] Raise when an invalid strategy is provided
+  def apply_count_strategy(count, strategy)
+    case strategy
+    when :sum
+      # keep count
+    when :sqrt, :unique
+      count = Math.sqrt(count)
+    when :log
+      count = Math.log10(count) if count.positive?
+    else
+      raise Error, "Invalid strategy: #{strategy}"
+    end
+    count
   end
-end
+end

metadata

@@ -1,100 +1,71 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: groupie
-version: !ruby/object:Gem::Version 
-  hash: 19
-  prerelease: false
-  segments: 
-  - 0
-  - 2
-  - 2
-  version: 0.2.2
+version: !ruby/object:Gem::Version
+  version: 0.5.0
 platform: ruby
-authors: 
+authors:
 - Wes Oldenbeuving
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-
-date: 2010-07-25 00:00:00 +02:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: testy
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  type: :development
-  version_requirements: *id001
-description: Group and classify text based on likelyhood of being included in a text of a specific category
-email: narnach@gmail.com
+date: 2022-02-16 00:00:00.000000000 Z
+dependencies: []
+description: Groupie is a simple way to group texts and classify new texts as being
+  a likely member of one of the defined groups. Think of bayesian spam filters.
+email:
+- wes@narnach.com
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
-- LICENSE
-files: 
-- .document
-- LICENSE
+extra_rdoc_files: []
+files:
+- ".github/ISSUE_TEMPLATE/bug_report.md"
+- ".github/ISSUE_TEMPLATE/feature_request.md"
+- ".github/dependabot.yml"
+- ".github/workflows/gem.yml"
+- ".github/workflows/rspec.yml"
+- ".github/workflows/rubocop.yml"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
 - Rakefile
-- VERSION
+- SECURITY.md
+- bin/console
+- bin/rubocop
+- bin/setup
 - groupie.gemspec
 - lib/groupie.rb
-- lib/groupie/core_ext/string.rb
 - lib/groupie/group.rb
-- readme.rdoc
-- spec/fixtures/ham/email_ham1.txt
-- spec/fixtures/ham/spam.la-44116217.txt
-- spec/fixtures/spam/email_spam1.txt
-- spec/fixtures/spam/email_spam2.txt
-- spec/fixtures/spam/spam.la-44118014.txt
-- spec/groupie/core_ext/string_spec.rb
-- spec/groupie/group_spec.rb
-- spec/groupie_spec.rb
-- spec/spec_helper.rb
-has_rdoc: true
-homepage: http://github.com/Narnach/groupie
-licenses: []
-
-post_install_message: 
-rdoc_options: 
-- --charset=UTF-8
-require_paths: 
+- lib/groupie/version.rb
+homepage: https://github.com/Narnach/groupie
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/Narnach/groupie
+  source_code_uri: https://github.com/Narnach/groupie
+  changelog_uri: https://github.com/Narnach/groupie/blob/stable/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
-rubyforge_project: 
-rubygems_version: 1.3.7
-signing_key: 
-specification_version: 3
-summary: Group and classify text
-test_files: 
-- spec/groupie/core_ext/string_spec.rb
-- spec/groupie/group_spec.rb
-- spec/groupie_spec.rb
-- spec/spec_helper.rb
+rubygems_version: 3.3.3
+signing_key:
+specification_version: 4
+summary: Library to help you group texts and classify new ones
+test_files: []

data/.document

@@ -1,5 +0,0 @@
-README.rdoc
-lib/**/*.rb
-bin/*
-features/**/*.feature
-LICENSE

data/LICENSE

@@ -1,20 +0,0 @@
-Copyright (c) 2009 Wes Oldenbeuving
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/VERSION

@@ -1 +0,0 @@
-0.2.2

data/lib/groupie/core_ext/string.rb

@@ -1,17 +0,0 @@
-class Groupie
-  module CoreExt
-    module String
-      def tokenize
-        downcase.
-          gsub(/\s/," ").
-          gsub(/[$']/,'').
-          gsub(/<[^>]+?>|[^\w -.,]/,'').
-          split(" ").map {|str| str.gsub(/\A['"]+|[!,."']+\Z/,'')}
-      end
-    end
-  end
-end
-
-class String
-  include Groupie::CoreExt::String
-end

data/readme.rdoc

@@ -1,27 +0,0 @@
-= Groupie
-
-Groupie is a simple way to group texts and classify new texts as being a likely member of one of the defined groups. Think of bayesian spam filters.
-
-The eventual goal is to have Groupie work as a sort of bayesian spam filter, where you feed it spam and ham (non-spam) and ask it to classify new texts as spam or ham. Applications for this are e-mail spam filtering and blog spam filtering. Other sorts of categorizing might be interesting as well, such as finding suitable tags for a blog post or bookmark.
-
-== Goals 
-
-Groupie is a 'fun' project that has the following goals, in descending order of importance:
-* Have fun playing with code
-* Play with Bayesian-like (spam) filtering
-
-== Current functionality
-
-Current funcionality includes:
-* Tokenize an input text to prepare it for grouping.
-  * Strip XML and HTML tag.
-  * Keep certain infix characters, such as period and comma.
-* Add texts (as an Array of Strings) to any number of groups.
-* Classify a single word to check the likelihood it belongs to each group.
-* Do classification for complete (tokenized) texts.
-
-== License
-
-As always, the code is licensed under the MIT license.
-
-Wes Oldenbeuving

data/spec/fixtures/ham/email_ham1.txt

@@ -1,13 +0,0 @@
-Re: [ubuntu-art] [Breathe] Network Manager-icons
-Am Sonntag, den 31.05.2009, 17:53 +0200 schrieb Steve Dodier:
-> Hello,
->
-> I think the notify-osd icons have a completely different style, which
-> is looking great within the notification bubbles, but i doubt it'd
-> look great to have the notify-osd wifi icons in the panel. I think the
-> drawing of the notification- wifi icons should be done afterwards, and
-> if they should be based on those of the icon set, they could be made
-> smoother, and possibly desaturated for some of them, to avoid drawing
-> too much attention from the user when popping up.
->
-> Cordially, SD.