dejunk 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 78bada0220a075c407f09e81eeb3602b669f9c64a6b0107bf26764c2721c1f0c
-  data.tar.gz: 189aa6798ca085cee985df90a28c7efa42c57c0574ef6652382992e0ef0ff41c
+  metadata.gz: 06c7eece281db7bb6218b0f3a56da9541a5830f8b7d0e739daf08b6d9f6b004d
+  data.tar.gz: b2dc0dd0f457164085eb19e418f75db4bca6d7cb0a315fa1e5bc0f9fad2ac3aa
 SHA512:
-  metadata.gz: 719a2c5bcc2f9a9c201f8856fbd5a219fe7b1b497c1c585d3c49c6104d4b37099fef3aef7635dc2fd5b054c1ae25fbd91c98b9bf569e1293fb9ee2c68049a10e
-  data.tar.gz: 0a0a75d55d940cd8e757a458385ab603541150a1a94e42532c9628a26b46be1b198220d781fddcb9d4d3da59ced552d7a8b056c7934c7192437420288298110a
+  metadata.gz: 367bf74302776bf03dbef89b2f5aad58c5f446511b30ae2cfc765372b86a9600ef9e2faf7d4b639629938ebfc59dd9c7d2a1720564dcfd94aeedd05235b023cb
+  data.tar.gz: a47f13d1b05d25641329ac4ed963288096e84a30186a16789facef75ecb5709137987d65b7cf727b935c71a16f01481b39b50e46bae8154cdd5ea28843ac1b3c

data/.github/workflows/build.yml

@@ -0,0 +1,15 @@
+name: build
+
+on:
+  push:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: .tool-versions
+          bundler-cache: true
+      - run: bundle exec rspec

data/.tool-versions

@@ -0,0 +1 @@
+ruby 4.0.5

data/lib/dejunk/version.rb

@@ -1,3 +1,3 @@
 module Dejunk
-  VERSION = "0.6.0"
+  VERSION = "0.6.1"
 end

data/lib/dejunk.rb

@@ -1,6 +1,5 @@
 require "active_support/core_ext/object/blank"
 require "active_support/core_ext/string"
-require "bigdecimal"
 require "yaml"
 
 require "dejunk/version"
@@ -133,17 +132,16 @@ module Dejunk
 
     return 0 unless bigrams.present?
 
-    prob_bigrams_given_mashing = bigrams.
-      map { |bigram| BigDecimal(mashing_probability(bigram).to_s) }.
-      inject(&:*)
+    # Work in log space because the raw products of per-bigram probabilities
+    # underflow Float.
+    log_prob_given_mashing = bigrams.sum { |bigram| Math.log(mashing_probability(bigram)) }
+    log_prob_given_corpus = bigrams.sum { |bigram| Math.log(corpus_probability(bigram)) }
 
-    prob_bigrams_given_corpus = bigrams.
-      map { |bigram| BigDecimal(corpus_probability(bigram).to_s) }.
-      inject(&:*)
+    # Equivalent to mashing / (mashing + corpus), with the priors applied.
+    log_odds_ratio = log_prob_given_corpus + Math.log(1 - apriori_probability_of_mashing) -
+      log_prob_given_mashing - Math.log(apriori_probability_of_mashing)
 
-    numerator = prob_bigrams_given_mashing * apriori_probability_of_mashing
-
-    numerator / (numerator + prob_bigrams_given_corpus * (1 - apriori_probability_of_mashing))
+    1.0 / (1.0 + Math.exp(log_odds_ratio))
   end
 
   def normalize_for_comparison(string)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dejunk
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - David Judd
@@ -57,9 +57,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".circleci/config.yml"
+- ".github/workflows/build.yml"
 - ".gitignore"
 - ".rspec"
+- ".tool-versions"
 - ".travis.yml"
 - Gemfile
 - LICENSE
@@ -88,7 +89,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Detect keyboard mashing and other junk in your data.
 test_files: []

data/.circleci/config.yml

@@ -1,15 +0,0 @@
-version: 2.1
-
-jobs:
-  build:
-    docker:
-      - image: cimg/ruby:3.4.8
-    steps:
-      - checkout
-      - run: bundle install
-      - run: bundle exec rspec
-
-workflows:
-  build_and_test:
-    jobs:
-      - build