groupie 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd7e47aef7d4ed19c206e46d5eb716991562ee2c257c5cf8488341f646801f2f
-  data.tar.gz: 692108e3c8c2b7d4b26a3c7d702133d0a5e01324d0c42cd9c4143b3a4b601b6c
+  metadata.gz: 0afe90008ac3d4c4defc95c8317349195f6e13a9190f0a41ec47aed1bdd31f2b
+  data.tar.gz: ad64128eaf725f8a8f22c86456fc3aec97a331e975578da8fa58082f983ed839
 SHA512:
-  metadata.gz: dba999961a8c6d7ba2999770d9125dc1c4b2c6468452c115a0eeb3256d8d484ef0cba8d975053e5e4e7d907afaf402063fb0b75c6d1c24b5bd3dc7698a251f35
-  data.tar.gz: cb5d99d029d237a37354b81b97a4d0a03b6224040ddbbbd0cd96cb3536ea0cfe4ecfb55871e35749b133df0178c97937f2b98523dc3504e4620b3a2473403478
+  metadata.gz: 12ef28e989b25de4d85e6eb4f5684907b1934b9ca83aa4efdafecdeffe2e27a9c8f33ff3ebd5a4a3c2efc6a0eeaa4d5d33480f481d5608d3fa586bba415377a5
+  data.tar.gz: bf563abf04440751e5c81a7b738313a19f1f7e8feb32fe9263f86635e5fa57c277e29ffbfa0a8bd195f1a3dfc74b1d4aab0460fa04746643162f9451b33fa275

data/.github/workflows/gem.yml

@@ -6,11 +6,11 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 3.0
+          ruby-version: 2.7
           bundler-cache: true
       - name: Build the gem
         run: bundle exec rake build

data/.github/workflows/rspec.yml

@@ -3,16 +3,16 @@ name: RSpec
 on: [ push, pull_request ]
 
 jobs:
-  build:
+  rspec:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        # Maintained versions: 2.7, 3.0, 3.1
-        # Security updates only: 2.6 (EOL: 2022-03-31)
+        # Maintained versions: 3.0, 3.1
+        # Security updates only: 2.7 (expected EOL: 2023-03-31)
         # Source: https://www.ruby-lang.org/en/downloads/branches/
-        ruby: [ 2.6, 2.7, 3.0, 3.1 ]
+        ruby: [ 2.7, 3.0, 3.1 ]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:

data/.github/workflows/rubocop.yml

@@ -6,11 +6,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Install Ruby & Gems
-        uses: ruby/setup-ruby@v1 # Uses .ruby-version as version input
+        uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 3.0
+          ruby-version: 2.7
           bundler-cache: true
       - name: Rubocop
         # https://github.com/reviewdog/action-rubocop
@@ -21,6 +21,7 @@ jobs:
           github_token: ${{ secrets.github_token }}
           reporter: github-pr-review
           rubocop_version: gemfile
+          rubocop_flags: --display-only-fail-level-offenses --safe
           # Rely on Bundler-installed gems so don't install them again
           use_bundler: true
           skip_install: true

data/.rubocop.yml

@@ -18,6 +18,7 @@ Lint/Void:
 
 # RSpec has a lot of blocks, so ignore this rule there
 Metrics/BlockLength:
+  Severity: info
   Exclude:
     - 'spec/**/*_spec.rb'
 
@@ -27,10 +28,12 @@ RSpec/DescribedClass:
 
 # I prefer groups for structure, so the defaults are a little too strict for me
 RSpec/NestedGroups:
+  Severity: info
   Max: 4
 
 # I prefer more verbose examples, so tend to use more lines than the defaults
 RSpec/ExampleLength:
+  Severity: info
   Max: 20
 
 # For strings I enjoy using %w[], but for symbols the %i[] syntax just does not click.
@@ -50,4 +53,4 @@ Layout/FirstHashElementIndentation:
   EnforcedStyle: consistent # default is special_inside_parentheses
 # Let's enforce this to be consistent
 Layout/EndOfLine:
-  EnforcedStyle: lf # \n (unix line end) enforced everywhere, default is native
+  EnforcedStyle: lf # \n (unix line end) enforced everywhere, default is native

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 ## Unreleased changes
 
+## Version 0.6.0 -- 2022-05-19
+
+This release bumps the minimum Ruby version, so it's technically breaking if you're using a currently unsupported Ruby version. It also improves the tokenization of URLs by using their structure and common token boundaries to easily extract tokens.
+
+- Breaking: drop support for Ruby 2.6, minimum is 2.7 ([#58](https://github.com/Narnach/groupie/pull/58))
+- Feat: add better tokenization support for URIs ([#42](https://github.com/Narnach/groupie/pull/42), [#44](https://github.com/Narnach/groupie/pull/44))
+- Dev: Rubocop ignores unsafe cops, hides info severity ([#59](https://github.com/Narnach/groupie/pull/59))
+- Dev: enforce 100% test coverage ([#60](https://github.com/Narnach/groupie/pull/60))
+
 ## Version 0.5.0 -- 2022-02-16
 
 This release has breaking changes (deprecation cleanup and internals rework), a new feature (smart weights!) and is officially tested on Ruby 3.1.0 (it's what I use). I've enabled the setting to require MFA to publish this gem, to help protect those who use it.

data/Gemfile

@@ -12,3 +12,4 @@ gem 'rubocop', '~> 1.7'
 gem 'rubocop-performance', '~> 1.11'
 gem 'rubocop-rake', '~> 0.6.0'
 gem 'rubocop-rspec', '~> 2.4'
+gem 'simplecov', '~> 0.21.2'

data/Gemfile.lock

@@ -1,21 +1,22 @@
 PATH
   remote: .
   specs:
-    groupie (0.5.0)
+    groupie (0.6.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
     diff-lcs (1.5.0)
-    parallel (1.21.0)
-    parser (3.1.0.0)
+    docile (1.4.0)
+    parallel (1.22.1)
+    parser (3.1.2.0)
       ast (~> 2.4.1)
-    psych (4.0.3)
+    psych (4.0.4)
       stringio
     rainbow (3.1.1)
     rake (13.0.6)
-    regexp_parser (2.2.1)
+    regexp_parser (2.4.0)
     rexml (3.2.5)
     rspec (3.11.0)
       rspec-core (~> 3.11.0)
@@ -26,30 +27,36 @@ GEM
     rspec-expectations (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.11.0)
-    rspec-mocks (3.11.0)
+    rspec-mocks (3.11.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.11.0)
     rspec-support (3.11.0)
-    rubocop (1.25.1)
+    rubocop (1.29.1)
       parallel (~> 1.10)
       parser (>= 3.1.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
-      rexml
-      rubocop-ast (>= 1.15.1, < 2.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.17.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.15.2)
-      parser (>= 3.0.1.1)
-    rubocop-performance (1.13.2)
+    rubocop-ast (1.18.0)
+      parser (>= 3.1.1.0)
+    rubocop-performance (1.13.3)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
     rubocop-rake (0.6.0)
       rubocop (~> 1.0)
-    rubocop-rspec (2.8.0)
+    rubocop-rspec (2.10.0)
       rubocop (~> 1.19)
     ruby-progressbar (1.11.0)
-    stringio (3.0.1)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+    stringio (3.0.2)
     unicode-display_width (2.1.0)
 
 PLATFORMS
@@ -66,6 +73,7 @@ DEPENDENCIES
   rubocop-performance (~> 1.11)
   rubocop-rake (~> 0.6.0)
   rubocop-rspec (~> 2.4)
+  simplecov (~> 0.21.2)
 
 BUNDLED WITH
-   2.3.4
+   2.3.14

data/bin/rubocop

@@ -1,2 +1,15 @@
 #!/bin/bash
-bundle exec rubocop --force-exclusion $*
+# Run Rubocop with sane defaults
+# - only recognized file types: allows you to pipe in anything and have Ruboop only check files it thinks it can handle
+# - force exclusion: even when piping in files, it still ignores what I should not touch
+# - display style guide: help figure out what a rule is supposed to do
+# - safe: don't run experimental cops
+# - display only fail level offenses: hides INFO severity failing cops
+
+bundle exec rubocop \
+  --only-recognized-file-types \
+  --force-exclusion \
+  --display-style-guide \
+  --safe \
+  --display-only-fail-level-offenses \
+  $*

data/groupie.gemspec

@@ -13,7 +13,9 @@ Gem::Specification.new do |spec|
                      ' of one of the defined groups. Think of bayesian spam filters.'
   spec.homepage = 'https://github.com/Narnach/groupie'
   spec.license = 'MIT'
-  spec.required_ruby_version = '>= 2.6.0' # EOL for 2.6 is 2022-03-31, so support this as the minimum for now
+  # Ruby maintains support for the last 3-4 minor versions, so that's what we do as well.
+  # See: https://www.ruby-lang.org/en/downloads/branches/
+  spec.required_ruby_version = '>= 2.7.0'
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = 'https://github.com/Narnach/groupie'

data/lib/groupie/tokenizer.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+class Groupie
+  # Tokenizer helps turn a String into an Array of Strings that are the
+  # individual tokens (mostly words) from the input.
+  #
+  # Please consider this entire class to be a private API,
+  # and use Groupie.tokenize to tokenize things.
+  class Tokenizer
+    def initialize(input)
+      # Ensure our input is converted to a String and duplicated so we can modify it in-place
+      @raw = input.to_s.dup
+    end
+
+    def to_tokens
+      return @tokens if @tokens
+
+      # In-place modifications to our @raw String
+      downcase!
+      normalize_whitespace!
+      strip_html_tags!
+      tokenize_urls!
+      strip_non_word_characters!
+
+      # Split the resulting string on whitespace and clean up the token candidates
+      @tokens = @raw.split.map { |str| remove_interpunction!(str) }
+
+      @tokens
+    end
+
+    private
+
+    # Ignore case by downcasing everything
+    def downcase!
+      @raw.downcase!
+    end
+
+    # Convert all types of whitespace (space, tab, newline) into regular spaces
+    def normalize_whitespace!
+      @raw.gsub!(/\s+/, ' ')
+    end
+
+    # Strip HTML tags entirely
+    def strip_html_tags!
+      @raw.gsub!(/<[^>]+?>/, ' ')
+    end
+
+    # Intelligently split URLs into their component parts
+    def tokenize_urls!
+      @raw.gsub!(%r{http[\w\-\#:/_.?&=]+}) do |url|
+        maybe_parse_url(url) do |uri|
+          path = uri.path.tap { |str| str.tr!('/_\-', ' ') }
+          query = uri.query.tap { |str| str&.tr!('?=&#_\-', ' ') }
+          fragment = uri.fragment.tap { |str| str&.tr!('#_/\-', ' ') }
+
+          "#{uri.scheme} #{uri.host} #{path} #{query} #{fragment}"
+        end
+      end
+    end
+
+    # Strip characters not likely to be part of a word or number
+    def strip_non_word_characters!
+      @raw.gsub!(/[^\w\ \-.,]/, ' ')
+    end
+
+    # Remove wrapping quotes and interpunction from individual token candidates
+    def remove_interpunction!(str)
+      str.gsub!(/\A['"]+|[!,."']+\Z/, '')
+      str
+    end
+
+    # Sometimes a String looks like a URL, but it's not.
+    # This method attempts to parse the input string into a URI.
+    # If it's successful, yield it to the block and return its response.
+    # In case of failure, return the original string.
+    def maybe_parse_url(input)
+      uri = URI.parse(input)
+      yield uri
+    rescue URI::InvalidURIError
+      input
+    end
+  end
+end

data/lib/groupie/version.rb

@@ -2,7 +2,7 @@
 
 # This extends Groupie and adds a version number
 class Groupie
-  VERSION = '0.5.0'
+  VERSION = '0.6.0'
 
   def self.version
     VERSION

data/lib/groupie.rb

@@ -2,6 +2,7 @@
 
 require_relative 'groupie/version'
 require_relative 'groupie/group'
+require_relative 'groupie/tokenizer'
 require 'set'
 
 # Groupie is a text grouper and classifier, using naive Bayesian filtering.
@@ -24,13 +25,7 @@ class Groupie
   # @param [String, #to_s] object
   # @return [Array<String>]
   def self.tokenize(object)
-    object
-      .to_s
-      .downcase
-      .gsub(/\s/, ' ')
-      .gsub(/[$']/, '')
-      .gsub(/<[^>]+?>|[^\w -.,]/, '')
-      .split.map { |str| str.gsub(/\A['"]+|[!,."']+\Z/, '') }
+    Tokenizer.new(object).to_tokens
   end
 
   # Access an existing Group or create a new one.
@@ -52,7 +47,7 @@ class Groupie
     group_score_sums, hits = calculate_group_scores(words, strategy)
 
     group_score_sums.each.with_object({}) do |(group, sum), averages|
-      averages[group] = hits.positive? ? sum / hits : 0
+      averages[group] = sum / hits
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: groupie
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Wes Oldenbeuving
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-16 00:00:00.000000000 Z
+date: 2022-05-19 00:00:00.000000000 Z
 dependencies: []
 description: Groupie is a simple way to group texts and classify new texts as being
   a likely member of one of the defined groups. Think of bayesian spam filters.
@@ -40,6 +40,7 @@ files:
 - groupie.gemspec
 - lib/groupie.rb
 - lib/groupie/group.rb
+- lib/groupie/tokenizer.rb
 - lib/groupie/version.rb
 homepage: https://github.com/Narnach/groupie
 licenses:
@@ -57,14 +58,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.6.0
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.3.14
 signing_key:
 specification_version: 4
 summary: Library to help you group texts and classify new ones