text-metrics 1.0.0.beta2 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e71911a9dd9a27cc6dcaa9eb9525cf48d73d48c5fc49526d475cb2a1f6de98c6
-  data.tar.gz: c686dfd9cf3cdb730b27c0329b1a0ba13748dea1de1f5730cd4283cea9885de2
+  metadata.gz: a7420551fa74c4621b2ce5d2d8d1e83a5bb46a1790c4bb340ed2556cf08b7bc1
+  data.tar.gz: 599106caafbdb730d6ea72d532afdc5f2e7b5bf69402d73f8c782523ed49c6c4
 SHA512:
-  metadata.gz: 57de157f47ac0ab6cf79956509087371d9c9e12fbdbe6d80b7314110c1d8211e831561adb1c9476199e086bb3f6d33c55c1f3e7c39dcf144e04f0766c7e1773d
-  data.tar.gz: f7b5f77a6fba329093f3f55c38844ac8696f28c9d5bbfaf08750d464732bdf5a5484ebea57444a928ebe8a65e6fdec76a8ade187987df17cd419edc3c4f8555b
+  metadata.gz: e35abe082c3a30179a7a1c171c80ae997690b609bd81ec21ea5f12d4b506e5cdad87802bd89d07867d193bd499f7a91e6f3fa1f05d9e11d9c8ab59c763413f1b
+  data.tar.gz: b9b9a018a2dc761540358386586923826ee5defd1fc5426a39d01acc2c5757de53c69629ec88daee674c0d6f14e9e2ba0c8b112e5a40348d52aea013f9815298

data/CHANGELOG.md

@@ -1,6 +1,17 @@
 # Change log
 
 ## main
+- ... TBD ...
+
+## 1.1.0
+- Accept `:en` as an alias for `:en_us` in `TextMetrics.new`.
+- Add five metrics to the analyzer and `#to_h`: `automated_readability_index`,
+  `type_token_ratio` (lexical diversity), `reading_time` (minutes, with a configurable
+  `wpm:`), and the previously-internal `polysyllabic_words_count` and `long_words_count`.
+
+## 1.0.0
+- First public release.
+- see `UPGRADING.md` for migration details from the pre-release `0.x` API.
 
 ## ## 1.0.0.beta2
 - Add `gunning_fog_index` to the analyzer metrics and `#to_h` output.

data/README.md

@@ -8,6 +8,8 @@ Text Metrics is a Ruby library for analysing text. It was inspired by the Python
 
 It gives you the everyday counts you need — words, characters, sentences and syllables — plus readability scores such as Flesch Reading Ease, Flesch-Kincaid Grade Level, SMOG, Gunning Fog, Coleman-Liau Index and LIX. English and French are supported.
 
+It is battle-tested in production on millions of student writings at [Plume](https://plume-app.co).
+
 ## Accurate, dictionary-based syllable counting
 
 Readability scores such as Flesch, Flesch-Kincaid and SMOG depend heavily on syllable counts. Many libraries estimate those counts with hyphenation rules, which are fast but often wrong. Text Metrics starts from real pronunciation data instead:
@@ -24,6 +26,8 @@ That makes syllable-sensitive scores more reliable, especially for longer or les
 - **Flesch-Kincaid Grade** maps to a US school grade. It uses the same formula for every language because there is no validated French adaptation.
 - **Gunning Fog** uses words with three or more syllables as complex words, matching the same syllable counts used by SMOG.
 - **Coleman-Liau** counts alphabetic letters only (not digits or punctuation), per its definition.
+- **Automated Readability Index** counts characters (letters, digits and punctuation, spaces excluded), per its original "strokes" definition — so it uses `characters_count`, unlike Coleman-Liau which uses letters only.
+- **Type-token ratio** is distinct-words over total-words and is sensitive to text length: longer texts trend lower because words repeat. Compare it only across texts of similar length.
 
 ## Features
 
@@ -32,10 +36,14 @@ _Basic metrics:_
 - [x] word count
 - [x] character count
 - [x] sentence count
+- [x] polysyllabic word count (three or more syllables)
+- [x] long word count (more than six characters)
 - [x] average syllables per word
 - [x] average letters per word
 - [x] average words per sentence
 - [x] average characters per sentence
+- [x] type-token ratio (lexical diversity)
+- [x] reading time (minutes, configurable words-per-minute)
 
 _Readability tests:_
 
@@ -45,6 +53,7 @@ _Readability tests:_
 - [x] Gunning Fog Index
 - [x] Coleman-Liau Index
 - [x] Lix Index
+- [x] Automated Readability Index
 
 ## Installation
 
@@ -52,7 +61,7 @@ Text Metrics is not published to RubyGems yet. For now, install it from GitHub:
 
 ```ruby
 # Gemfile
-gem "text-metrics", github: "plume-app/text-metrics", branch: "main"
+gem "text-metrics", "~> 1.1.0"
 ```
 
 ### Supported Ruby versions
@@ -68,11 +77,13 @@ metrics = TextMetrics.new("This gem analyses all kinds of text.")
 metrics.to_h
 # {
 #   words_count: 7, characters_count: 30, sentences_count: 1, syllables_count: 10,
-#   punctuation_count: 1, syllables_per_word_average: 1.4, letters_per_word_average: 4.14,
+#   punctuation_count: 1, polysyllabic_words_count: 1, long_words_count: 1,
+#   syllables_per_word_average: 1.4, letters_per_word_average: 4.14,
 #   words_per_sentence_average: 7.0, characters_per_sentence_average: 30.0,
 #   words_per_punctuation_average: 7.0, punctuation_per_sentence_average: 1.0,
-#   flesch_reading_ease: 78.87, flesch_kincaid_grade: 4.0, lix: 21.29,
-#   smog_index: 0.0, gunning_fog_index: 8.5, coleman_liau_index: 4.33
+#   type_token_ratio: 1.0, flesch_reading_ease: 78.87, flesch_kincaid_grade: 4.0,
+#   lix: 21.29, smog_index: 0.0, gunning_fog_index: 8.5, coleman_liau_index: 4.33,
+#   automated_readability_index: 2.3, reading_time: 0.04
 # }
 
 # Or ask for a single metric:
@@ -81,18 +92,48 @@ metrics.characters_count                # => 30
 metrics.flesch_reading_ease             # => 78.87
 metrics.flesch_kincaid_grade            # => 4.0
 metrics.gunning_fog_index                # => 8.5
+
+# Reading time is in minutes; pass a custom reading pace if you like:
+metrics.reading_time                    # => 0.04   (at the default 200 wpm)
+metrics.reading_time(wpm: 130)          # => 0.05
 ```
 
 ### Languages
 
-American English (`:en_us`) is the default. To analyse French text, pass `language: :fr`:
+American English (`:en_us`) is the default. `:en` is accepted as an alias for `:en_us`. To analyse French text, pass `language: :fr`:
 
 ```ruby
 TextMetrics.new("Bonjour le monde.", language: :fr)
+TextMetrics.new("Hello.", language: :en)    # same as :en_us
 ```
 
 Unsupported languages raise `TextMetrics::Error`.
 
+### Configuration
+
+Global defaults can be set via `TextMetrics.configure`. Call this once at boot time — in a Rails initializer, for example:
+
+```ruby
+TextMetrics.configure do |config|
+  # Words per minute used by #reading_time (default: 200).
+  config.wpm = 250
+end
+```
+
+In a **Rails app**, generate the initializer with:
+
+```sh
+rails generate text_metrics:install
+```
+
+This creates `config/initializers/text_metrics.rb` with the options commented out at their defaults.
+
+Per-call overrides are still supported and always take precedence over the global config:
+
+```ruby
+metrics.reading_time(wpm: 130)
+```
+
 ### Comparing two texts
 
 Levenshtein distance compares two strings, so it is exposed on the `TextMetrics` module:

data/lib/generators/text_metrics/install_generator.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module TextMetrics
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      desc "Creates a TextMetrics initializer in config/initializers"
+
+      def create_initializer
+        create_file "config/initializers/text_metrics.rb", <<~RUBY
+          # frozen_string_literal: true
+
+          TextMetrics.configure do |config|
+            # Words per minute used to compute reading_time (default: 200).
+            # config.wpm = 200
+          end
+        RUBY
+      end
+    end
+  end
+end

data/lib/text_metrics/configuration.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module TextMetrics
+  class Configuration
+    attr_accessor :wpm
+
+    def initialize
+      @wpm = 200
+    end
+  end
+end

data/lib/text_metrics/processors/base.rb

@@ -7,26 +7,30 @@ module TextMetrics
     class Base
       GEM_PATH = File.dirname(__FILE__, 2).freeze
 
-      # The public metric surface. #to_h and the individual readers are both derived
-      # from this list, so they can never drift apart.
+      # Single source of truth: #to_h and reader methods are both derived from this list.
       METRICS = %i[
         words_count
         characters_count
         sentences_count
         syllables_count
         punctuation_count
+        polysyllabic_words_count
+        long_words_count
         syllables_per_word_average
         letters_per_word_average
         words_per_sentence_average
         characters_per_sentence_average
         words_per_punctuation_average
         punctuation_per_sentence_average
+        type_token_ratio
         flesch_reading_ease
         flesch_kincaid_grade
         lix
         smog_index
         gunning_fog_index
         coleman_liau_index
+        automated_readability_index
+        reading_time
       ].freeze
 
       attr_reader :text, :language
@@ -36,8 +40,6 @@ module TextMetrics
         @language = language
       end
 
-      # Every metric in one hash. Single source of truth for the public surface.
-      # Memoized — the analyzer is immutable once built.
       def to_h
         @to_h ||= METRICS.to_h { |metric| [metric, public_send(metric)] }
       end
@@ -65,8 +67,14 @@ module TextMetrics
         punctuation_marks.size
       end
 
-      # averages — rounded for display only. The readability scores below are computed
-      # from the full-precision ratios (#average_*), not from these rounded values.
+      def polysyllabic_words_count
+        words.count { |word| count_syllables_in_word(word) >= 3 }
+      end
+
+      def long_words_count
+        words.count { |word| word.length > 6 }
+      end
+
       def syllables_per_word_average
         average_syllables_per_word.round(1)
       end
@@ -97,16 +105,16 @@ module TextMetrics
         (punctuation_count.to_f / sentences_count).round(2)
       end
 
-      # readability scores — computed from full-precision ratios, rounded only at the end,
-      # and returned unclamped (a Flesch score can legitimately exceed 100 or go negative).
+      def type_token_ratio
+        return 0.0 if words_count.zero?
+
+        (words.uniq.size.to_f / words_count).round(2)
+      end
 
-      # Language-specific; subclasses supply the constants.
       def flesch_reading_ease
         raise NotImplementedError
       end
 
-      # Flesch-Kincaid Grade Level (US school grade). The same formula is used for every
-      # language — there is no validated non-English adaptation.
       def flesch_kincaid_grade
         return 0.0 if words_count.zero?
 
@@ -116,7 +124,7 @@ module TextMetrics
       def smog_index
         return 0.0 if sentences_count < 3
 
-        (1.043 * Math.sqrt(30.0 * count_polysyllabic_words / sentences_count) + 3.1291).round(1)
+        (1.043 * Math.sqrt(30.0 * polysyllabic_words_count / sentences_count) + 3.1291).round(1)
       rescue ZeroDivisionError
         0.0
       end
@@ -124,7 +132,7 @@ module TextMetrics
       def gunning_fog_index
         return 0.0 if words_count.zero?
 
-        (0.4 * (average_words_per_sentence + 100.0 * count_polysyllabic_words / words_count)).round(1)
+        (0.4 * (average_words_per_sentence + 100.0 * polysyllabic_words_count / words_count)).round(1)
       end
 
       def coleman_liau_index
@@ -139,14 +147,24 @@ module TextMetrics
       def lix
         return 0.0 if words_count.zero?
 
-        long_words = words.count { |word| word.length > 6 }
+        (average_words_per_sentence + 100.0 * long_words_count / words_count).round(2)
+      end
+
+      def automated_readability_index
+        return 0.0 if words_count.zero?
+
+        (4.71 * characters_count / words_count.to_f + 0.5 * average_words_per_sentence - 21.43).round(1)
+      end
+
+      def reading_time(wpm: nil)
+        wpm ||= TextMetrics.configuration.wpm
+        return 0.0 if words_count.zero? || wpm.zero?
 
-        (average_words_per_sentence + 100.0 * long_words / words_count).round(2)
+        (words_count / wpm.to_f).round(2)
       end
 
       private
 
-      # full-precision ratios feeding the readability formulas
       def average_syllables_per_word
         return 0.0 if words_count.zero?
 
@@ -165,34 +183,20 @@ module TextMetrics
         words_count.to_f / sentences_count
       end
 
-      # Count of alphabetic characters only (letters), as required by Coleman-Liau and the
-      # letters-per-word metric — distinct from #characters_count, which includes digits
-      # and punctuation.
       def letters_count
         @letters_count ||= text.scan(/[[:alpha:]]/).size
       end
 
-      # Subclasses provide the language-specific syllable counting.
       def count_syllables_in_word(word)
         raise NotImplementedError
       end
 
-      def count_polysyllabic_words
-        words.count { |word| count_syllables_in_word(word) >= 3 }
-      end
-
-      # tokenizers
       def punctuation_marks
         @punctuation_marks ||= text.scan(/[.,!?;:]/)
       end
 
       def words
-        @words ||= begin
-          normalized_text = text.downcase.strip
-
-          # Split into words, including hyphenated words, and excluding numbers
-          normalized_text.scan(/\b[A-Za-zÀ-ÖØ-öø-ÿ'-]+\b/)
-        end
+        @words ||= text.downcase.strip.scan(/\b[A-Za-zÀ-ÖØ-öø-ÿ'-]+\b/)
       end
 
       def sentences

data/lib/text_metrics/railtie.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module TextMetrics
+  class Railtie < Rails::Railtie
+    generators do
+      require "generators/text_metrics/install_generator"
+    end
+  end
+end

data/lib/text_metrics/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module TextMetrics # :nodoc:
-  VERSION = "1.0.0.beta2"
+  VERSION = "1.1.0"
 end

data/lib/text_metrics.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require "text_metrics/version"
+require "text_metrics/configuration"
+require "text_metrics/railtie" if defined?(Rails::Railtie)
 require "text_metrics/levenshtein"
 require "text_metrics/processors/american_english"
 require "text_metrics/processors/french"
@@ -10,6 +12,22 @@ module TextMetrics
 
   DEFAULT_LANGUAGE = :en_us
 
+  class << self
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure
+      yield configuration
+    end
+
+    private
+
+    def reset_configuration!
+      @configuration = Configuration.new
+    end
+  end
+
   PROCESSORS = {
     en_us: Processors::AmericanEnglish,
     fr: Processors::French
@@ -34,8 +52,11 @@ module TextMetrics
 
   # Coerce to a known language symbol, or raise a helpful error.
   # Handles nil, strings and symbols without leaking a NoMethodError.
+  LANGUAGE_ALIASES = {en: :en_us}.freeze
+
   def self.resolve_language(language)
     resolved = language.to_s.to_sym
+    resolved = LANGUAGE_ALIASES.fetch(resolved, resolved)
     return resolved if PROCESSORS.key?(resolved)
 
     raise Error, "Unknown language #{language.inspect}. Available languages: #{PROCESSORS.keys.join(", ")}"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: text-metrics
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta2
+  version: 1.1.0
 platform: ruby
 authors:
 - Adrien POLY
@@ -120,14 +120,17 @@ files:
 - LICENSE.txt
 - README.md
 - UPGRADING.md
+- lib/generators/text_metrics/install_generator.rb
 - lib/text-metrics.rb
 - lib/text_metrics.rb
+- lib/text_metrics/configuration.rb
 - lib/text_metrics/dictionaries/english_word_syllable_database.txt
 - lib/text_metrics/dictionaries/french_word_syllable_exceptions.yml
 - lib/text_metrics/levenshtein.rb
 - lib/text_metrics/processors/american_english.rb
 - lib/text_metrics/processors/base.rb
 - lib/text_metrics/processors/french.rb
+- lib/text_metrics/railtie.rb
 - lib/text_metrics/version.rb
 homepage: https://github.com/plume-app/text-metrics
 licenses: