groupie 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 18730d0a05fad9b05bb1dc50302436fb7c15043fbfe665dc96b6ea927a50aa73
-  data.tar.gz: 8040f137e1f57080009b55cd08f0406f0ebb2e4112c7bae0838b08e4e70e0f82
+  metadata.gz: dd7e47aef7d4ed19c206e46d5eb716991562ee2c257c5cf8488341f646801f2f
+  data.tar.gz: 692108e3c8c2b7d4b26a3c7d702133d0a5e01324d0c42cd9c4143b3a4b601b6c
 SHA512:
-  metadata.gz: edc0af06023ed354667a3c36857d9a004785a78506a085c97591435eb67081ddd58a42165b8c00db029e2ec97c039f60682c33a82be5488f3a35264a52cd6e77
-  data.tar.gz: 0c002dfbcc1e8b4df108042be00d9588eb5ff8e1b9697de99e1bc87292f07500dc1d3f33307e853f1b2dc7c8bb2e5097e4649f1a716331afed424f3a9d4c0253
+  metadata.gz: dba999961a8c6d7ba2999770d9125dc1c4b2c6468452c115a0eeb3256d8d484ef0cba8d975053e5e4e7d907afaf402063fb0b75c6d1c24b5bd3dc7698a251f35
+  data.tar.gz: cb5d99d029d237a37354b81b97a4d0a03b6224040ddbbbd0cd96cb3536ea0cfe4ecfb55871e35749b133df0178c97937f2b98523dc3504e4620b3a2473403478

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,32 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+## Describe the bug
+
+A clear and concise description of what the bug is.
+
+## How to reproduce it
+
+Include the minimum code sample required to reproduce the bug.
+
+## Expected behavior
+
+A clear and concise description of what you expected to happen.
+
+## Context
+
+Please describe:
+
+* What version of Groupie did you use? (`bundle list|grep groupie`)
+* What version of Ruby did you use? (`ruby -v`)
+* What platform and architecture do you use? (macOS version (Intel vs Apple M1), Windows version, Linux distro/architecture)
+
+## Additional context
+
+Add any other context about the problem here.

data/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,24 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+## Is your feature request related to a problem? Please describe.
+
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+## Describe the solution you'd like
+
+A clear and concise description of what you want to happen.
+
+## Describe alternatives you've considered
+
+A clear and concise description of any alternative solutions or features you've considered.
+
+## Additional context
+
+Add any other context or screenshots about the feature request here.

data/.github/dependabot.yml

@@ -3,6 +3,7 @@
 # Check every week (Monday 5:00 CET by default) for updates.
 # Each package manager gets 5 non-security updates by default.
 # Security updates bypass most configuration here and show up when found.
+# We rely on DepFu for Ruby and Gem updates
 version: 2
 updates:
   - package-ecosystem: github-actions
@@ -10,12 +11,3 @@ updates:
     directory: '/'
     schedule:
       interval: weekly
-  - package-ecosystem: bundler
-    # Bundler handles Ruby dependencies
-    directory: '/'
-    schedule:
-      interval: weekly
-    versioning-strategy: increase-if-necessary
-    allow:
-      - dependency-type: direct
-      - dependency-type: indirect

data/.github/workflows/rspec.yml

@@ -7,9 +7,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        # Maintained versions: 2.7 and 3.0
-        # Security updates only: 2.6
-        ruby: [ 2.6, 2.7, 3.0 ]
+        # Maintained versions: 2.7, 3.0, 3.1
+        # Security updates only: 2.6 (EOL: 2022-03-31)
+        # Source: https://www.ruby-lang.org/en/downloads/branches/
+        ruby: [ 2.6, 2.7, 3.0, 3.1 ]
     steps:
       - uses: actions/checkout@v2
       - name: Set up Ruby

data/.rubocop.yml

@@ -36,3 +36,18 @@ RSpec/ExampleLength:
 # For strings I enjoy using %w[], but for symbols the %i[] syntax just does not click.
 Style/SymbolArray:
   EnforcedStyle: brackets
+
+# Indentation is something I've got strong opinions about which differ from Rubocop.
+Layout/ArgumentAlignment:
+  EnforcedStyle: with_fixed_indentation # default is with_first_argument
+Layout/ArrayAlignment:
+  EnforcedStyle: with_fixed_indentation # default is with_first_element
+Layout/FirstArgumentIndentation:
+  EnforcedStyle: consistent # default is special_for_inner_method_call_in_parentheses
+Layout/FirstArrayElementIndentation:
+  EnforcedStyle: consistent # default is special_inside_parentheses
+Layout/FirstHashElementIndentation:
+  EnforcedStyle: consistent # default is special_inside_parentheses
+# Let's enforce this to be consistent
+Layout/EndOfLine:
+  EnforcedStyle: lf # \n (unix line end) enforced everywhere, default is native

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 ## Unreleased changes
 
+## Version 0.5.0 -- 2022-02-16
+
+This release has breaking changes (deprecation cleanup and internals rework), a new feature (smart weights!) and is officially tested on Ruby 3.1.0 (it's what I use). I've enabled the setting to require MFA to publish this gem, to help protect those who use it.
+
+- Breaking: remove `String#tokenize` core extension; please use `Groupie.tokenize(string)` instead [#39](https://github.com/Narnach/groupie/pull/39)
+- Breaking: due to changed internals, YAML serialized data from 0.4.x will lack some of the new internal caches. I'd suggest loading the old data and adding the words from each group to a new (0.5.x) instance of Groupie. [#40](https://github.com/Narnach/groupie/pull/40)
+- Feat: add support for smart default weights, reducing the effect of low data on predictions [#40](https://github.com/Narnach/groupie/pull/40)
+- Deps: add Ruby 3.1 to list of tested & supported gems
+- Chore: require multi-factor authentication to publish gem updates
+- Chore: add Security.md to advertise a security policy
+- Style: addressed Lint/AmbiguousOperatorPrecedence
+- Dev: bump development dependencies multiple times
+- Dev: switch to DepFu to manage development dependencies
+
 ## Version 0.4.1 -- 2021-09-08
 
 Non-functional fixes to the CI config and Rubygems.org metadata.

data/Gemfile

@@ -5,6 +5,7 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in groupie.gemspec
 gemspec
 
+gem 'psych', '~> 4.0'
 gem 'rake', '~> 13.0'
 gem 'rspec', '~> 3.0'
 gem 'rubocop', '~> 1.7'

data/Gemfile.lock

@@ -1,61 +1,65 @@
 PATH
   remote: .
   specs:
-    groupie (0.4.1)
+    groupie (0.5.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
-    diff-lcs (1.4.4)
-    parallel (1.20.1)
-    parser (3.0.2.0)
+    diff-lcs (1.5.0)
+    parallel (1.21.0)
+    parser (3.1.0.0)
       ast (~> 2.4.1)
-    rainbow (3.0.0)
+    psych (4.0.3)
+      stringio
+    rainbow (3.1.1)
     rake (13.0.6)
-    regexp_parser (2.1.1)
+    regexp_parser (2.2.1)
     rexml (3.2.5)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.2)
-    rubocop (1.20.0)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
+    rubocop (1.25.1)
       parallel (~> 1.10)
-      parser (>= 3.0.0.0)
+      parser (>= 3.1.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.9.1, < 2.0)
+      rubocop-ast (>= 1.15.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.11.0)
+    rubocop-ast (1.15.2)
       parser (>= 3.0.1.1)
-    rubocop-performance (1.11.5)
+    rubocop-performance (1.13.2)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
     rubocop-rake (0.6.0)
       rubocop (~> 1.0)
-    rubocop-rspec (2.4.0)
-      rubocop (~> 1.0)
-      rubocop-ast (>= 1.1.0)
+    rubocop-rspec (2.8.0)
+      rubocop (~> 1.19)
     ruby-progressbar (1.11.0)
-    unicode-display_width (2.0.0)
+    stringio (3.0.1)
+    unicode-display_width (2.1.0)
 
 PLATFORMS
   x86_64-darwin-20
+  x86_64-darwin-21
   x86_64-linux
 
 DEPENDENCIES
   groupie!
+  psych (~> 4.0)
   rake (~> 13.0)
   rspec (~> 3.0)
   rubocop (~> 1.7)
@@ -64,4 +68,4 @@ DEPENDENCIES
   rubocop-rspec (~> 2.4)
 
 BUNDLED WITH
-   2.2.24
+   2.3.4

data/README.md

@@ -1,5 +1,7 @@
 # Groupie
 
+[![Depfu](https://badges.depfu.com/badges/367956233b3b31a6fc19db4515263b9e/overview.svg)](https://depfu.com/github/Narnach/groupie?project_id=34004)
+
 Groupie is a simple way to group texts and classify new texts as being a likely member of one of the defined groups. Think of bayesian spam filters.
 
 The eventual goal is to have Groupie work as a sort of bayesian spam filter, where you feed it spam and ham (non-spam) and ask it to classify new texts as spam or ham. Applications for this are e-mail spam filtering and blog spam filtering. Other sorts of categorizing might be interesting as well, such as finding suitable tags for a blog post or bookmark.
@@ -90,6 +92,21 @@ groupie.classify_text(test_tokens, :unique)
 test_tokens - (test_tokens & groupie.unique_words)
 # => ["please", "to", "reset", "awesome"]
 # If you'd be classifying email, you can assume that common email headers will get ignored this way.
+
+# If you're just starting out, your incomplete data could lead to dramatic misrepresentations of the data.
+# To balance against this, you can enable smart weight:
+groupie.smart_weight = true
+# You could also set it during initialization via Groupie.new(smart_weight: true)
+# What's so useful about it? It adds a default weight to _all_ words, even the ones you haven't
+# seen yet, which counter-acts the data you have. This shines in low data situations,
+# reducing the impact of the few words you have seen before.
+groupie.default_weight
+# => 1.2285714285714286
+# Classifying the same text as before should consider all words, and add this default weight to all words
+# It basically gives all groups the likelihood of "claiming" a word,
+# unless there is strong data to suggest otherwise.
+groupie.classify_text(test_tokens)
+# => {:spam=>0.5241046831955923, :ham=>0.4758953168044077}
 ```
 
 Persistence can be naively done by using YAML:
@@ -110,7 +127,9 @@ For I'm still experimenting with Groupie in [Infinity Feed](https://www.infinity
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. Rubocop is available via `bin/rubocop` with some friendly default settings.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`.
+
+To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org). For obvious reasons, only the project maintainer can do this.
 
 ## Contributing
 

data/SECURITY.md

@@ -0,0 +1,18 @@
+# Security Policy
+
+Groupie is inherently not a user-facing library, so possible vectors for exploitation seem small to me.
+That said, in the event of a security vulnerability being found, this document describes how to report it.
+
+## Supported Versions
+
+As a small library with infrequent updates, I will accept bug and security reports for the current minor version.
+Severe issues might be backported to previous minor versions. I'll handle this on a case-by-case basis.
+
+## Reporting a Vulnerability
+
+For low-risk things you can create an issue or PR.
+In case of a high risk thing, you can email me at security@narnach.com.
+
+## Thanks
+
+Once we have successfully handled a security vulnerability, we'll add an attribution to the list below.

data/groupie.gemspec

@@ -18,6 +18,8 @@ Gem::Specification.new do |spec|
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = 'https://github.com/Narnach/groupie'
   spec.metadata['changelog_uri'] = 'https://github.com/Narnach/groupie/blob/stable/CHANGELOG.md'
+  # Require multi-factor authentication to publish the gem
+  spec.metadata['rubygems_mfa_required'] = 'true'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.

data/lib/groupie/group.rb

@@ -3,11 +3,13 @@
 class Groupie
   # Group represents a group or category that words can be classified into.
   class Group
-    attr_reader :word_counts
+    attr_reader :word_counts, :total_word_count
 
-    def initialize(name)
+    def initialize(name, groupie)
       @name = name
+      @groupie = groupie
       @word_counts = {}
+      @total_word_count = 0
     end
 
     def words
@@ -34,7 +36,11 @@ class Groupie
     # Add a single word and count it.
     def add_word(word)
       @word_counts[word] ||= 0
-      @word_counts[word] += 1
+      current_count = @word_counts[word] += 1
+      @total_word_count += 1
+      # If this word is new for this Group, it might be new for the entire Groupie
+      @groupie.add_word(word) if current_count == 1
+      nil
     end
   end
 end

data/lib/groupie/version.rb

@@ -2,7 +2,7 @@
 
 # This extends Groupie and adds a version number
 class Groupie
-  VERSION = '0.4.1'
+  VERSION = '0.5.0'
 
   def self.version
     VERSION

data/lib/groupie.rb

@@ -2,15 +2,20 @@
 
 require_relative 'groupie/version'
 require_relative 'groupie/group'
-require_relative 'groupie/core_ext/string'
+require 'set'
 
 # Groupie is a text grouper and classifier, using naive Bayesian filtering.
 class Groupie
   # Wrap all errors we raise in this so our own errors are recognizable.
   class Error < StandardError; end
 
-  def initialize
+  attr_accessor :smart_weight
+
+  # @param [true, false] smart_weight (false) Whether smart weight is enabled or not.
+  def initialize(smart_weight: false)
     @groups = {}
+    @smart_weight = smart_weight
+    @known_words = Set.new
   end
 
   # Turn a String (or anything else that responds to #to_s) into an Array of String tokens.
@@ -33,7 +38,7 @@ class Groupie
   # @param [Object] group The name of the group to access.
   # @return [Groupie::Group] An existing or new group identified by +group+.
   def [](group)
-    @groups[group] ||= Group.new(group)
+    @groups[group] ||= Group.new(group, self)
   end
 
   # Classify a text by taking the average of all word classifications.
@@ -51,25 +56,27 @@ class Groupie
     end
   end
 
-  # Classify a single word against all groups.
+  # Classify a single word against all groups, returning the probability distribution.
   #
   # @param [String] entry A word to be classified
-  # @param [Symbol] strategy
-  # @return [Hash<Object, Float>] Hash with <group, score> pairings. Scores are always in 0.0..1.0
+  # @param [Symbol] strategy (:sum) the strategy to use on the score
+  # @return [Hash<Object, Float>] Hash with <group, probability> pairings.
+  #   Probabilities are always in 0.0..1.0, and add up to 1.0 (i.e. it's a probability distribution)
   # @raise [Groupie::Error] Raise when an invalid strategy is provided
   def classify(entry, strategy = :sum)
-    results = {}
-    total_count = @groups.values.inject(0) do |sum, group|
-      sum + apply_count_strategy(group.count(entry), strategy)
+    # Calculate default weight once outside of the loop
+    default_weight = self.default_weight
+    # Each group calculates the count, then reduces it to a score: <group name, score>
+    per_group_score = @groups.transform_values do |group|
+      apply_count_strategy(default_weight + group.count(entry), strategy)
     end
-    return results if total_count.zero?
+    # When we have no scores, we have no results, so abort early
+    # Note that when smart_weight is enabled we always have a score.
+    total_score = per_group_score.values.sum
+    return {} if total_score.zero?
 
-    @groups.each do |name, group|
-      count = apply_count_strategy(group.count(entry), strategy)
-      results[name] = count.positive? ? count.to_f / total_count : 0.0
-    end
-
-    results
+    # Final results must be within 0.0..1.0, so divide each score by the total score
+    per_group_score.transform_values { |group_score| group_score.to_f / total_score }
   end
 
   # Return a word score dictionary that excludes the 4th quartile most popular words.
@@ -85,13 +92,39 @@ class Groupie
       total.merge!(group.word_counts) { |_key, o, n| o + n }
     end
     # Extract the word count that's at the top 75%
-    top_quartile_index = [total_count.size * 3 / 4 - 1, 1].max
+    top_quartile_index = [((total_count.size * 3) / 4) - 1, 1].max
     top_quartile_frequency = total_count.values.sort[top_quartile_index]
     # Throw out all words which have a count that's above this frequency
     total_count.reject! { |_word, count| count > top_quartile_frequency }
     total_count.keys
   end
 
+  # Default weight is used when +smart_weight+ is enabled.
+  # Each word's count is increased by the +default_weight+ value,
+  # which is the average frequency of each unique word we know about.
+  #
+  # Example: if we have indexed 1000 total words, of which 500 were unique,
+  #   the default_weight would be 1000/500=2.0
+  #
+  # @return [Float] The default weight for all words
+  def default_weight
+    # Default weight only applies when smart weight is enabled
+    return 0.0 unless smart_weight
+
+    # If we don't know any words, the weight is also zero
+    return 0.0 unless @known_words.any?
+
+    # Gather counts and calculate
+    total_words = @groups.each_value.sum(&:total_word_count)
+    total_unique_words = @known_words.count
+    total_words / total_unique_words.to_f
+  end
+
+  # Private method used by Groups to register known words with the Group.
+  def add_word(word)
+    @known_words << word
+  end
+
   private
 
   # Calculate grouped scores

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: groupie
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Wes Oldenbeuving
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-07 00:00:00.000000000 Z
+date: 2022-02-16 00:00:00.000000000 Z
 dependencies: []
 description: Groupie is a simple way to group texts and classify new texts as being
   a likely member of one of the defined groups. Think of bayesian spam filters.
@@ -18,6 +18,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/ISSUE_TEMPLATE/bug_report.md"
+- ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/dependabot.yml"
 - ".github/workflows/gem.yml"
 - ".github/workflows/rspec.yml"
@@ -31,12 +33,12 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- SECURITY.md
 - bin/console
 - bin/rubocop
 - bin/setup
 - groupie.gemspec
 - lib/groupie.rb
-- lib/groupie/core_ext/string.rb
 - lib/groupie/group.rb
 - lib/groupie/version.rb
 homepage: https://github.com/Narnach/groupie
@@ -46,6 +48,7 @@ metadata:
   homepage_uri: https://github.com/Narnach/groupie
   source_code_uri: https://github.com/Narnach/groupie
   changelog_uri: https://github.com/Narnach/groupie/blob/stable/CHANGELOG.md
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -61,7 +64,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.24
+rubygems_version: 3.3.3
 signing_key:
 specification_version: 4
 summary: Library to help you group texts and classify new ones

data/lib/groupie/core_ext/string.rb

@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-class Groupie
-  module CoreExt
-    # This module monkey patches String to respond to #tokenize
-    module String
-      def tokenize
-        warn "Please use Groupie.tokenize instead of String#tokenize (from #{caller(1..1).first})"
-        Groupie.tokenize(self)
-      end
-    end
-  end
-end
-
-class String
-  include Groupie::CoreExt::String
-end