dejunk 0.5.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1ab7cd51156a993d232efbc68bfeb4ea1657f0e112878a4d03ae8963122d359
-  data.tar.gz: 456178beb94060ea3589c5917ee978aa9e96f6534eeb2db97d97fab05175e823
+  metadata.gz: 06c7eece281db7bb6218b0f3a56da9541a5830f8b7d0e739daf08b6d9f6b004d
+  data.tar.gz: b2dc0dd0f457164085eb19e418f75db4bca6d7cb0a315fa1e5bc0f9fad2ac3aa
 SHA512:
-  metadata.gz: 122869629a43aa3885a942a63b76536eac52fbe0c83a311921feee498987b99e591d9c42c496c2129ec718c37e0c2b753eff562a4a707375ea49375c5038ec73
-  data.tar.gz: cfd3126d2d94102ca27c28f36d5543c39c0629a7717b2c13a35d4588e487bb8103471661b6dbf418839ed8c19392a235aa259f2dd3a382d72c603d08aa6ce384
+  metadata.gz: 367bf74302776bf03dbef89b2f5aad58c5f446511b30ae2cfc765372b86a9600ef9e2faf7d4b639629938ebfc59dd9c7d2a1720564dcfd94aeedd05235b023cb
+  data.tar.gz: a47f13d1b05d25641329ac4ed963288096e84a30186a16789facef75ecb5709137987d65b7cf727b935c71a16f01481b39b50e46bae8154cdd5ea28843ac1b3c

data/.github/workflows/build.yml

@@ -0,0 +1,15 @@
+name: build
+
+on:
+  push:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: .tool-versions
+          bundler-cache: true
+      - run: bundle exec rspec

data/.tool-versions

@@ -0,0 +1 @@
+ruby 4.0.5

data/dejunk.gemspec

@@ -6,7 +6,7 @@ require 'dejunk/version'
 Gem::Specification.new do |spec|
   spec.name          = "dejunk"
   spec.version       = Dejunk::VERSION
-  spec.required_ruby_version = '>= 2.3'
+  spec.required_ruby_version = '>= 3.3'
   spec.authors       = ["David Judd"]
   spec.email         = ["david@academia.edu"]
 
@@ -20,7 +20,6 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'activesupport'
 
-  spec.add_development_dependency "bundler", "~> 2.0"
-  spec.add_development_dependency "rake", "~> 12.3.3"
+  spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
 end

data/lib/dejunk/version.rb

@@ -1,3 +1,3 @@
 module Dejunk
-  VERSION = "0.5.0"
+  VERSION = "0.6.1"
 end

data/lib/dejunk.rb

@@ -1,6 +1,8 @@
-require "dejunk/version"
-require "yaml"
+require "active_support/core_ext/object/blank"
 require "active_support/core_ext/string"
+require "yaml"
+
+require "dejunk/version"
 
 module Dejunk
   extend self
@@ -113,7 +115,7 @@ module Dejunk
     string.
       chars.
       zip(string.chars[1..-1]).
-      map { |c1,c2| "#{c1.mb_chars.downcase}#{c2.mb_chars.downcase}" if c1 && c2 }.
+      map { |c1,c2| "#{c1.downcase}#{c2.downcase}" if c1 && c2 }.
       compact.
       map { |bigram| bigram.gsub(/[0-9]/, '0'.freeze) }.
       map { |bigram| bigram.gsub(/[[:space:]]/, ' '.freeze) }
@@ -130,22 +132,25 @@ module Dejunk
 
     return 0 unless bigrams.present?
 
-    prob_bigrams_given_mashing = bigrams.
-      map { |bigram| BigDecimal(mashing_probability(bigram).to_s) }.
-      inject(&:*)
-
-    prob_bigrams_given_corpus = bigrams.
-      map { |bigram| BigDecimal(corpus_probability(bigram).to_s) }.
-      inject(&:*)
+    # Work in log space because the raw products of per-bigram probabilities
+    # underflow Float.
+    log_prob_given_mashing = bigrams.sum { |bigram| Math.log(mashing_probability(bigram)) }
+    log_prob_given_corpus = bigrams.sum { |bigram| Math.log(corpus_probability(bigram)) }
 
-    numerator = prob_bigrams_given_mashing * apriori_probability_of_mashing
+    # Equivalent to mashing / (mashing + corpus), with the priors applied.
+    log_odds_ratio = log_prob_given_corpus + Math.log(1 - apriori_probability_of_mashing) -
+      log_prob_given_mashing - Math.log(apriori_probability_of_mashing)
 
-    numerator / (numerator + prob_bigrams_given_corpus * (1 - apriori_probability_of_mashing))
+    1.0 / (1.0 + Math.exp(log_odds_ratio))
   end
 
   def normalize_for_comparison(string)
+    # This mirrors what mb_chars did, assuming that non-UTF-8 encoded strings
+    # are actually UTF-8 in disguise. It's unclear whether this is necessary,
+    # but we left it in to avoid having to figure this out.
+    string = string.dup.force_encoding(Encoding::UTF_8) if string.encoding != Encoding::UTF_8
+
     string.
-      mb_chars.
       unicode_normalize(:nfkd).
       gsub(/\p{Mn}+/, ''.freeze).
       gsub(/[^[:alnum:]]+/, ''.freeze).

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: dejunk
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.1
 platform: ruby
 authors:
 - David Judd
-autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-06-01 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -24,34 +23,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 12.3.3
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 12.3.3
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -66,16 +51,16 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
 email:
 - david@academia.edu
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".circleci/config.yml"
+- ".github/workflows/build.yml"
 - ".gitignore"
 - ".rspec"
+- ".tool-versions"
 - ".travis.yml"
 - Gemfile
 - LICENSE
@@ -90,7 +75,6 @@ files:
 homepage: https://github.com/academia-edu/dejunk
 licenses: []
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -98,15 +82,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '3.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.33
-signing_key: 
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Detect keyboard mashing and other junk in your data.
 test_files: []

data/.circleci/config.yml

@@ -1,19 +0,0 @@
-version: 2.1
-
-orbs:
-  ruby: circleci/ruby@1.4.0
-
-jobs:
-  build:
-    docker:
-      - image: cimg/ruby:3.0.1
-    steps:
-      - checkout
-      - ruby/install-deps
-      - ruby/rspec-test
-
-workflows:
-  version: 2
-  build_and_test:
-    jobs:
-      - build