groupie 0.1.0 → 0.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2bbc9cfb580ee027b67d88ac1d22d58b864edbf4c8bed8544748a0d5a35c9526
+  data.tar.gz: ce283b2269613471da0ac4cc14734b1092afe798c9e81337bc676eb6b2964da9
+SHA512:
+  metadata.gz: c566f577e7fe83e0e403f4e271f5176d05a00204b0b1bf90a6c40379f5da2924910f4674ea89b2986c36bec17f989ea4f21f381bf8fd75cd679e6f78d95a0472
+  data.tar.gz: bab39365fa7ac4f4a665a803454b223749b917ba3e3c55c821e7714d66d7a6a06c35d9982c8bddeee27e2c8c232922e0f64c3ced84401e350cdf9540e566ac49

data/.github/dependabot.yml

@@ -0,0 +1,21 @@
+# https://docs.github.com/en/github/administering-a-repository/keeping-your-dependencies-updated-automatically
+# There's a lot of options, but for now let's keep it simple.
+# Check every week (Monday 5:00 CET by default) for updates.
+# Each package manager gets 5 non-security updates by default.
+# Security updates bypass most configuration here and show up when found.
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    # Github Actions are checked for updates
+    directory: '/'
+    schedule:
+      interval: weekly
+  - package-ecosystem: bundler
+    # Bundler handles Ruby dependencies
+    directory: '/'
+    schedule:
+      interval: weekly
+    versioning-strategy: increase-if-necessary
+    allow:
+      - dependency-type: direct
+      - dependency-type: indirect

data/.github/workflows/gem.yml

@@ -0,0 +1,16 @@
+name: Gem building
+
+on: [ pull_request ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.0
+          bundler-cache: true
+      - name: Build the gem
+        run: bundle exec rake build

data/.github/workflows/rspec.yml

@@ -0,0 +1,21 @@
+name: RSpec
+
+on: [ push, pull_request ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # Maintained versions: 2.7 and 3.0
+        # Security updates only: 2.6
+        ruby: [ 2.6, 2.7, 3.0 ]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: Run the tests
+        run: bundle exec rspec

data/.github/workflows/rubocop.yml

@@ -0,0 +1,26 @@
+name: Code Quality
+on: [ pull_request ]
+jobs:
+  rubocop:
+    name: Rubocop
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+      - name: Install Ruby & Gems
+        uses: ruby/setup-ruby@v1 # Uses .ruby-version as version input
+        with:
+          ruby-version: 3.0
+          bundler-cache: true
+      - name: Rubocop
+        # https://github.com/reviewdog/action-rubocop
+        uses: reviewdog/action-rubocop@v2
+        with:
+          fail_on_error: true
+          filter_mode: nofilter
+          github_token: ${{ secrets.github_token }}
+          reporter: github-pr-review
+          rubocop_version: gemfile
+          # Rely on Bundler-installed gems so don't install them again
+          use_bundler: true
+          skip_install: true

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,38 @@
+require:
+  - rubocop-rspec
+  - rubocop-rake
+  - rubocop-performance
+
+AllCops:
+  # We bump the version to get new cops, so enable them by default
+  NewCops: enable
+
+# Two lines should fit next to each other in split view on a widescreen
+Layout/LineLength:
+  Max: 120
+
+# We still have old-style rspec checks, so this triggers on functional comparisons there
+Lint/Void:
+  Exclude:
+    - 'spec/**/*_spec.rb'
+
+# RSpec has a lot of blocks, so ignore this rule there
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/**/*_spec.rb'
+
+# I prefer to see the class name over "described_class"
+RSpec/DescribedClass:
+  EnforcedStyle: explicit
+
+# I prefer groups for structure, so the defaults are a little too strict for me
+RSpec/NestedGroups:
+  Max: 4
+
+# I prefer more verbose examples, so tend to use more lines than the defaults
+RSpec/ExampleLength:
+  Max: 20
+
+# For strings I enjoy using %w[], but for symbols the %i[] syntax just does not click.
+Style/SymbolArray:
+  EnforcedStyle: brackets

data/CHANGELOG.md

@@ -0,0 +1,86 @@
+## Unreleased changes
+
+## Version 0.4.0 -- 2021-09-07
+
+Welcome to 2021, where Ruby version 2.6 is the lowest with official support, Bundler is the default for managing packages and RSpec version 3 is used to test things. This version updates Groupie into this decade.
+
+- Refactor: update Groupie to 2021 standards
+- Feat: raise Groupie::Error instead of RuntimeError
+- Feat: deprecate String#tokenize in favor of Groupie.tokenize
+- Doc: document API of Groupie
+- Doc: update readme with examples
+- Refactor: reorder Groupie methods by importance
+- Refactor: simplify Groupie#classify
+- Refactor: reduce complexity of Groupie#unique_words
+- Refactor: simplify Groupie#classify\_text
+
+## Version 0.3.0 -- 2010-07-29
+
+Multiple changes to the 'unique words' strategy, hopefully improving the behavior.
+
+- Cache unique words in an instance var to reduce time required to do subsequent lookups
+- Sanity spec
+- Unique strategy now includes all words except for the global 4th quartile
+- Unique strategy changed yet again: only ignore words that occur more than their group's median
+- Unique strategy now behaves like sqrt that only checks unique words
+- Unique word finder uses less elegant but (hopefully) faster code
+- Removed gemspec
+
+## Version 0.2.3 -- 2010-07-29
+
+Add a new 'unique words' strategy, which ignores words that occur in all categories.
+
+- Added 'unique' classification strategy
+- Added Group#<< as alias for Group#add
+- Updated readme
+
+## Version 0.2.2 -- 2010-07-25
+
+Bugfix for log10 strategy.
+
+- Fixed log10 strategy counting for Groupie.classify
+
+## Version 0.2.1 -- 2010-07-25
+
+Offer multiple ways to weigh word counts in calculating final scores.
+
+- Added sqrt and log word counting strategies
+
+## Version 0.2.0 -- 2010-07-25
+
+Classification can't raise division by zero errors anymore.
+
+- Groupie.classify_text ignores unclassified tokens
+
+## Version 0.1.1 -- 2010-07-25
+
+Swap test framework and tokenization improvements.
+
+- Regenerated gemspec
+- Strip quotes from tokens
+- Replaced testy tests with rspec
+
+## Version 0.1.0 -- 2010-07-25
+
+The initial release as a gem, after working on this on/off over a year.
+
+- Added gemspec
+- Fixed text classification to properly average group scores
+- Added test for classifying tokenized html email spam
+- Classification of texts is now possible
+- Added readme and MIT license
+- Test the full html and headers of tokenized emails
+- Support infix commas for tokenized strings
+- Allow infix dots in tokenized strings
+- Strip HTML tags when sanitizing a string
+- Classify common words based on tokenized text from spam.la e-mails
+- Added String#tokenize
+- Ensure a Group will still work when loaded from YAML
+- Added test helper file
+- Refactored Group to maintain a Hash of words and counts instead of a list of words
+- Removed obsolete method
+- Added testcase for three groups
+- Support multiple examples to add more weight to their grouping
+- Renamed tests to reflect intent of content
+- Classification now allows for a degree of certainty
+- Implemented simple spam check

data/Gemfile

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in groupie.gemspec
+gemspec
+
+gem 'rake', '~> 13.0'
+gem 'rspec', '~> 3.0'
+gem 'rubocop', '~> 1.7'
+gem 'rubocop-performance', '~> 1.11'
+gem 'rubocop-rake', '~> 0.6.0'
+gem 'rubocop-rspec', '~> 2.4'

data/Gemfile.lock

@@ -0,0 +1,67 @@
+PATH
+  remote: .
+  specs:
+    groupie (0.4.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.2)
+    diff-lcs (1.4.4)
+    parallel (1.20.1)
+    parser (3.0.2.0)
+      ast (~> 2.4.1)
+    rainbow (3.0.0)
+    rake (13.0.6)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+    rubocop (1.20.0)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.9.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.11.0)
+      parser (>= 3.0.1.1)
+    rubocop-performance (1.11.5)
+      rubocop (>= 1.7.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
+    rubocop-rake (0.6.0)
+      rubocop (~> 1.0)
+    rubocop-rspec (2.4.0)
+      rubocop (~> 1.0)
+      rubocop-ast (>= 1.1.0)
+    ruby-progressbar (1.11.0)
+    unicode-display_width (2.0.0)
+
+PLATFORMS
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  groupie!
+  rake (~> 13.0)
+  rspec (~> 3.0)
+  rubocop (~> 1.7)
+  rubocop-performance (~> 1.11)
+  rubocop-rake (~> 0.6.0)
+  rubocop-rspec (~> 2.4)
+
+BUNDLED WITH
+   2.2.24

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2009-2021 Wes Oldenbeuving
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,121 @@
+# Groupie
+
+Groupie is a simple way to group texts and classify new texts as being a likely member of one of the defined groups. Think of bayesian spam filters.
+
+The eventual goal is to have Groupie work as a sort of bayesian spam filter, where you feed it spam and ham (non-spam) and ask it to classify new texts as spam or ham. Applications for this are e-mail spam filtering and blog spam filtering. Other sorts of categorizing might be interesting as well, such as finding suitable tags for a blog post or bookmark.
+
+Started and forgotten in 2009 as a short-lived experiment, in 2010 Groupie got new features when I started using it on a RSS reader project that classified news items into "Interesting" and "Not interesting" categories.
+
+## Current functionality
+
+Current funcionality includes:
+
+* Tokenize an input text to prepare it for grouping.
+    * Strip XML and HTML tag.
+    * Keep certain infix characters, such as period and comma.
+* Add texts (as an Array of Strings) to any number of groups.
+* Classify a single word to check the likelihood it belongs to each group.
+* Do classification for complete (tokenized) texts.
+* Pick classification strategy to weigh repeat words differently (weigh by sum, square root or log10 of words in group)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'groupie'
+```
+
+You can also perform this to do this for you:
+
+    bundle add groupie
+
+And then execute:
+
+    bundle install
+
+Or install it system-wide via:
+
+    gem install groupie
+
+## Usage
+
+Here is an annotated console session that shows off the features available in Groupie.
+
+```ruby
+# Instantiate a new Groupie instance
+groupie = Groupie.new
+
+# Groups are defined as you use them, so let's get started by adding some pre-tokenized words
+groupie[:spam].add(%w[this is obvious spam please buy our product])
+groupie[:spam].add(%w[hello friend this is rich prince i have awesome bitcoin for you])
+groupie[:ham].add(%w[you are invited to my awesome party just click the link to rsvp])
+
+# Is your data less than clean? We've got a tokenizer for that!
+tokens = Groupie.tokenize('Please give me your password so I can haxx0r you!')
+# => ["please", "give", "me", "your", "password", "so", "i", "can", "haxx0r", "you"]
+groupie[:spam].add(tokens)
+
+# So, now let's attempt to classify a text and see if it's spam or ham:
+test_tokens = %w[please click the link to reset your password for our awesome product]
+groupie.classify_text(test_tokens)
+# => {:spam=>0.5909090909090909, :ham=>0.4090909090909091}
+# As you can see, this password reset email looks a little dodgy...
+# We have multiple strategies for drawing conclusions about what group it belongs to.
+# The default you saw above is :sum, it weighs each word by the total sum of occurrences.
+# Let's see if it looks less bad by using a different classification strategies.
+
+# Log reduces the weight of each word to the log10 of its occurrence count:
+# - Count 1 is weight 0
+# - Count 10 is weight 1
+# - Count 100 is weight 2
+groupie.classify_text(test_tokens, :log)
+# => {:spam=>0.5, :ham=>0.5}
+# This is even more even, most likely because it ignores all single-count words...
+
+# Square root algorithm is less harsh, it reduces the weight of each word to the square root of the count:
+# - Count 1 is weight 1
+# - Count 4 is weight 2
+# - Count 9 is weight 3
+groupie.classify_text(test_tokens, :sqrt)
+# => {:spam=>0.5909090909090909, :ham=>0.4090909090909091}
+# This seems to result in the same value as :sum
+
+# Unique uses the same weighting algorithm as the square root, but it modifies the word dictionary:
+# it discards the 25% most common words, so less common words gain higher predictive power.
+groupie.classify_text(test_tokens, :unique)
+# => {:spam=>0.625, :ham=>0.375}
+# This looks even worse for our poor password reset email.
+# In case you're curious, the ignored words in this case are:
+test_tokens - (test_tokens & groupie.unique_words)
+# => ["please", "to", "reset", "awesome"]
+# If you'd be classifying email, you can assume that common email headers will get ignored this way.
+```
+
+Persistence can be naively done by using YAML:
+
+```ruby
+# Instantiate a new Groupie instance
+groupie = Groupie.new
+groupie[:spam].add(%w[assume you have a lot of data you care about])
+
+require 'yaml'
+yaml = YAML.dump(groupie)
+loaded = YAML.safe_load(yaml, permitted_classes: [Groupie, Groupie::Group, Symbol])
+```
+
+For I'm still experimenting with Groupie in [Infinity Feed](https://www.infinity-feed.com), so persistence is a Future Problem for me there. In development, I'm building (low data count) classifiers in memory and discarding them after use.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. Rubocop is available via `bin/rubocop` with some friendly default settings.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/Narnach/groupie.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).