groupie 0.1.0 → 0.4.0

data/Rakefile

@@ -1,53 +1,12 @@
-require 'rubygems'
-require 'rake'
+# frozen_string_literal: true
 
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gem|
-    gem.name = "groupie"
-    gem.summary = %Q{Group and classify text}
-    gem.description = %Q{Group and classify text based on likelyhood of being included in a text of a specific category}
-    gem.email = "narnach@gmail.com"
-    gem.homepage = "http://github.com/Narnach/groupie"
-    gem.authors = ["Wes Oldenbeuving"]
-    gem.add_development_dependency "testy", ">= 0"
-    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
-  end
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
-end
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
-require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/*_test.rb'
-  test.verbose = true
-end
+RSpec::Core::RakeTask.new(:spec)
 
-begin
-  require 'rcov/rcovtask'
-  Rcov::RcovTask.new do |test|
-    test.libs << 'test'
-    test.pattern = 'test/**/*_test.rb'
-    test.verbose = true
-  end
-rescue LoadError
-  task :rcov do
-    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
-  end
-end
+require 'rubocop/rake_task'
 
-task :test => :check_dependencies
+RuboCop::RakeTask.new
 
-task :default => :test
-
-require 'rake/rdoctask'
-Rake::RDocTask.new do |rdoc|
-  version = File.exist?('VERSION') ? File.read('VERSION') : ""
-
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "groupie #{version}"
-  rdoc.rdoc_files.include('readme*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end
+task default: [:spec, :rubocop]

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'groupie'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/rubocop

@@ -0,0 +1,2 @@
+#!/bin/bash
+bundle exec rubocop --force-exclusion $*

data/bin/setup

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -x
+
+bundle install
+bundle clean
+
+# Do any other automated setup that you need to do here

data/groupie.gemspec

@@ -1,60 +1,36 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
-# -*- encoding: utf-8 -*-
+# frozen_string_literal: true
 
-Gem::Specification.new do |s|
-  s.name = %q{groupie}
-  s.version = "0.1.0"
+require_relative 'lib/groupie/version'
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Wes Oldenbeuving"]
-  s.date = %q{2010-07-25}
-  s.description = %q{Group and classify text based on likelyhood of being included in a text of a specific category}
-  s.email = %q{narnach@gmail.com}
-  s.extra_rdoc_files = [
-    "LICENSE"
-  ]
-  s.files = [
-    ".document",
-     "LICENSE",
-     "Rakefile",
-     "VERSION",
-     "groupie.gemspec",
-     "lib/groupie.rb",
-     "lib/groupie/core_ext/string.rb",
-     "lib/groupie/group.rb",
-     "readme.rdoc",
-     "test/fixtures/ham/spam.la-44116217.txt",
-     "test/fixtures/spam/spam.la-44118014.txt",
-     "test/groupie/core_ext/string_test.rb",
-     "test/groupie/group_test.rb",
-     "test/groupie_test.rb",
-     "test/test_helper.rb"
-  ]
-  s.homepage = %q{http://github.com/Narnach/groupie}
-  s.rdoc_options = ["--charset=UTF-8"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
-  s.summary = %q{Group and classify text}
-  s.test_files = [
-    "test/groupie/core_ext/string_test.rb",
-     "test/groupie/group_test.rb",
-     "test/groupie_test.rb",
-     "test/test_helper.rb"
-  ]
+Gem::Specification.new do |spec|
+  spec.name = 'groupie'
+  spec.version = Groupie::VERSION
+  spec.authors = ['Wes Oldenbeuving']
+  spec.email = ['wes@narnach.com']
 
-  if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 3
+  spec.summary = 'Library to help you group texts and classify new ones'
+  spec.description = 'Groupie is a simple way to group texts and classify new texts as being a likely member' \
+                     ' of one of the defined groups. Think of bayesian spam filters.'
+  spec.homepage = 'https://github.com/Narnach/groupie'
+  spec.license = 'MIT'
+  spec.required_ruby_version = '>= 2.6.0' # EOL for 2.6 is 2022-03-31, so support this as the minimum for now
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<testy>, [">= 0"])
-    else
-      s.add_dependency(%q<testy>, [">= 0"])
-    end
-  else
-    s.add_dependency(%q<testy>, [">= 0"])
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/Narnach/groupie'
+  spec.metadata['changelog_uri'] = 'https://github.com/Narnach/groupie/blob/stable/Changelog.md'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
   end
-end
+  # spec.bindir        = "exe"
+  # spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
 
+  # Uncomment to register a new dependency of your gem
+  # spec.add_dependency "example-gem", "~> 1.0"
+
+  # For more information and examples about making a new gem, checkout our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end

data/lib/groupie/core_ext/string.rb

@@ -1,12 +1,12 @@
+# frozen_string_literal: true
+
 class Groupie
   module CoreExt
+    # This module monkey patches String to respond to #tokenize
     module String
       def tokenize
-        downcase.
-          gsub(/\s/," ").
-          gsub(/[$']/,'').
-          gsub(/<[^>]+?>|[^\w -.,]/,'').
-          split(" ").map {|str| str.gsub(/[,.]+\Z/,'')}
+        warn "Please use Groupie.tokenize instead of String#tokenize (from #{caller(1..1).first})"
+        Groupie.tokenize(self)
       end
     end
   end
@@ -14,4 +14,4 @@ end
 
 class String
   include Groupie::CoreExt::String
-end
+end

data/lib/groupie/group.rb

@@ -1,10 +1,15 @@
+# frozen_string_literal: true
+
 class Groupie
+  # Group represents a group or category that words can be classified into.
   class Group
+    attr_reader :word_counts
+
     def initialize(name)
       @name = name
       @word_counts = {}
     end
-    
+
     def words
       @word_counts.keys
     end
@@ -17,16 +22,19 @@ class Groupie
       nil
     end
 
+    alias << add
+
     # Return the count for a specific +word+.
     def count(word)
       @word_counts[word] || 0
     end
 
+    private
+
     # Add a single word and count it.
     def add_word(word)
       @word_counts[word] ||= 0
       @word_counts[word] += 1
     end
-    private :add_word
   end
-end
+end

data/lib/groupie/version.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# This extends Groupie and adds a version number
+class Groupie
+  VERSION = '0.4.0'
+
+  def self.version
+    VERSION
+  end
+end

data/lib/groupie.rb

@@ -1,51 +1,133 @@
-lib_dir = File.expand_path(File.dirname(__FILE__))
-$LOAD_PATH.unshift(lib_dir) unless $LOAD_PATH.include?(lib_dir)
-require 'groupie/group'
-require 'groupie/core_ext/string'
+# frozen_string_literal: true
 
+require_relative 'groupie/version'
+require_relative 'groupie/group'
+require_relative 'groupie/core_ext/string'
+
+# Groupie is a text grouper and classifier, using naive Bayesian filtering.
 class Groupie
+  # Wrap all errors we raise in this so our own errors are recognizable.
+  class Error < StandardError; end
+
   def initialize
     @groups = {}
   end
 
+  # Turn a String (or anything else that responds to #to_s) into an Array of String tokens.
+  # This attempts to remove most common punctuation marks and types of whitespace.
+  #
+  # @param [String, #to_s] object
+  # @return [Array<String>]
+  def self.tokenize(object)
+    object
+      .to_s
+      .downcase
+      .gsub(/\s/, ' ')
+      .gsub(/[$']/, '')
+      .gsub(/<[^>]+?>|[^\w -.,]/, '')
+      .split.map { |str| str.gsub(/\A['"]+|[!,."']+\Z/, '') }
+  end
+
+  # Access an existing Group or create a new one.
+  #
+  # @param [Object] group The name of the group to access.
+  # @return [Groupie::Group] An existing or new group identified by +group+.
   def [](group)
     @groups[group] ||= Group.new(group)
   end
 
-  def classify(entry)
+  # Classify a text by taking the average of all word classifications.
+  #
+  # @param [Array<String>] words List of words to be classified
+  # @param [Symbol] strategy
+  # @return [Hash<Object, Float>] Hash with <group, score> pairings. Scores are always in 0.0..1.0
+  # @raise [Groupie::Error] Raise when an invalid strategy is provided
+  def classify_text(words, strategy = :sum)
+    words &= unique_words if strategy == :unique
+    group_score_sums, hits = calculate_group_scores(words, strategy)
+
+    group_score_sums.each.with_object({}) do |(group, sum), averages|
+      averages[group] = hits.positive? ? sum / hits : 0
+    end
+  end
+
+  # Classify a single word against all groups.
+  #
+  # @param [String] entry A word to be classified
+  # @param [Symbol] strategy
+  # @return [Hash<Object, Float>] Hash with <group, score> pairings. Scores are always in 0.0..1.0
+  # @raise [Groupie::Error] Raise when an invalid strategy is provided
+  def classify(entry, strategy = :sum)
     results = {}
-    total_count = @groups.inject(0) do |sum, name_group|
-      group = name_group.last
-      sum + group.count(entry)
+    total_count = @groups.values.inject(0) do |sum, group|
+      sum + apply_count_strategy(group.count(entry), strategy)
     end
-    return results if 0 == total_count
+    return results if total_count.zero?
 
     @groups.each do |name, group|
-      count = group.count(entry)
-      results[name] = count > 0 ? count.to_f / total_count : 0.0
+      count = apply_count_strategy(group.count(entry), strategy)
+      results[name] = count.positive? ? count.to_f / total_count : 0.0
     end
-    return results
+
+    results
   end
 
-  # Classify a text by taking the average of all word classifications.
-  def classify_text(words)
-    group_score_sums = words.inject({}) do |results, word|
-      word_results = classify(word)
-      results.merge(word_results) do |key, old, new|
-        old + new
-      end
+  # Return a word score dictionary that excludes the 4th quartile most popular words.
+  # Why do this? So the most common (and thus meaningless) words are ignored
+  # and less common words gain more predictive power.
+  #
+  # This is used by the :unique strategy of the classifier.
+  #
+  # @return [Hash<String, Integer>]
+  def unique_words
+    # Iterate over all Groups and merge their <word, count> dictionaries into one
+    total_count = @groups.inject({}) do |total, (_name, group)|
+      total.merge!(group.word_counts) { |_key, o, n| o + n }
     end
+    # Extract the word count that's at the top 75%
+    top_quartile_index = [total_count.size * 3 / 4 - 1, 1].max
+    top_quartile_frequency = total_count.values.sort[top_quartile_index]
+    # Throw out all words which have a count that's above this frequency
+    total_count.reject! { |_word, count| count > top_quartile_frequency }
+    total_count.keys
+  end
 
-    words_count = words.size.to_f
-    averages={}
-    group_score_sums.each do |group, sum|
-      averages[group] = sum / words_count
+  private
+
+  # Calculate grouped scores
+  #
+  # @param [Array<String>] words
+  # @param [Symbol] strategy
+  # @return [Array<Enumerator<String>, Integer>] a Hash with <group, score> pairs and an integer with the number of hits
+  def calculate_group_scores(words, strategy)
+    hits = 0
+    group_score_sums = words.each.with_object({}) do |word, results|
+      word_results = classify(word, strategy)
+      next results if word_results.empty?
+
+      hits += 1
+      results.merge!(word_results) { |_key, old, new| old + new }
     end
 
-    averages
+    [group_score_sums, hits]
   end
-  
-  def self.version
-    File.read(File.join(File.dirname(File.expand_path(__FILE__)), "..", "VERSION")).strip
+
+  # Helper function to reduce a raw word count to a strategy-modified weight.
+  # @param [Integer] count
+  # @param [Symbol] strategy
+  # @return [Integer, Float]
+  # @raise [Groupie::Error] Raise when an invalid strategy is provided
+  def apply_count_strategy(count, strategy)
+    case strategy
+    when :sum
+      # keep count
+    when :sqrt, :unique
+      count = Math.sqrt(count)
+    when :log
+      count = Math.log10(count) if count.positive?
+    else
+      raise Error, "Invalid strategy: #{strategy}"
+    end
+    count
   end
-end
+end

metadata

@@ -1,97 +1,68 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: groupie
-version: !ruby/object:Gem::Version 
-  hash: 27
-  prerelease: false
-  segments: 
-  - 0
-  - 1
-  - 0
-  version: 0.1.0
+version: !ruby/object:Gem::Version
+  version: 0.4.0
 platform: ruby
-authors: 
+authors:
 - Wes Oldenbeuving
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-
-date: 2010-07-25 00:00:00 +02:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: testy
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  type: :development
-  version_requirements: *id001
-description: Group and classify text based on likelyhood of being included in a text of a specific category
-email: narnach@gmail.com
+date: 2021-09-07 00:00:00.000000000 Z
+dependencies: []
+description: Groupie is a simple way to group texts and classify new texts as being
+  a likely member of one of the defined groups. Think of bayesian spam filters.
+email:
+- wes@narnach.com
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
-- LICENSE
-files: 
-- .document
-- LICENSE
+extra_rdoc_files: []
+files:
+- ".github/dependabot.yml"
+- ".github/workflows/gem.yml"
+- ".github/workflows/rspec.yml"
+- ".github/workflows/rubocop.yml"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
 - Rakefile
-- VERSION
+- bin/console
+- bin/rubocop
+- bin/setup
 - groupie.gemspec
 - lib/groupie.rb
 - lib/groupie/core_ext/string.rb
 - lib/groupie/group.rb
-- readme.rdoc
-- test/fixtures/ham/spam.la-44116217.txt
-- test/fixtures/spam/spam.la-44118014.txt
-- test/groupie/core_ext/string_test.rb
-- test/groupie/group_test.rb
-- test/groupie_test.rb
-- test/test_helper.rb
-has_rdoc: true
-homepage: http://github.com/Narnach/groupie
-licenses: []
-
-post_install_message: 
-rdoc_options: 
-- --charset=UTF-8
-require_paths: 
+- lib/groupie/version.rb
+homepage: https://github.com/Narnach/groupie
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/Narnach/groupie
+  source_code_uri: https://github.com/Narnach/groupie
+  changelog_uri: https://github.com/Narnach/groupie/blob/stable/Changelog.md
+post_install_message:
+rdoc_options: []
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
-rubyforge_project: 
-rubygems_version: 1.3.7
-signing_key: 
-specification_version: 3
-summary: Group and classify text
-test_files: 
-- test/groupie/core_ext/string_test.rb
-- test/groupie/group_test.rb
-- test/groupie_test.rb
-- test/test_helper.rb
+rubygems_version: 3.2.24
+signing_key:
+specification_version: 4
+summary: Library to help you group texts and classify new ones
+test_files: []