text-metrics 0.0.1

data/lib/text_metrics/processors/american_english.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "text_metrics/processors/base"
+
+module TextMetrics
+  module Processors
+    class AmericanEnglish < TextMetrics::Processors::Base
+      def flesch_reading_ease
+        sentence_length = words_per_sentence_average
+        syllables_per_word = syllables_per_word_average
+        flesch = 206.835 - 1.015 * sentence_length - 84.6 * syllables_per_word
+
+        flesch.round(2).clamp(0.0, 100.0)
+      end
+
+      def flesch_kincaid_grade
+        sentence_length = words_per_sentence_average
+        syllables_per_word = syllables_per_word_average
+        flesch = 0.39 * sentence_length + 11.8 * syllables_per_word - 15.59
+
+        flesch.round(1).clamp(0.0, 18.0)
+      end
+
+      private
+
+      def count_syllables_in_word(word)
+        hyphen_dictionary.visualise(word).count("-") + 1
+      end
+
+      def hyphen_dictionary
+        @hyphen_dictionary ||= Text::Hyphen.new(language: "en_us", left: 0, right: 0)
+      end
+    end
+  end
+end

data/lib/text_metrics/processors/base.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+
+require "text/hyphen"
+
+module TextMetrics
+  module Processors
+    class Base
+      attr_reader :text, :with_syllable_exceptions
+
+      def initialize(text:, with_syllable_exceptions: true)
+        @text = text&.squeeze(" ") || ""
+        @with_syllable_exceptions = with_syllable_exceptions
+      end
+
+      def all
+        @all ||= {
+          words_count: words_count,
+          characters_count: characters_count,
+          sentences_count: sentences_count,
+          syllables_count: syllables_count,
+          punctuations_count: punctuations_count,
+          syllables_per_word_average: syllables_per_word_average,
+          letters_per_word_average: letters_per_word_average,
+          words_per_sentence_average: words_per_sentence_average,
+          words_per_punctuations_average: words_per_punctuations_average,
+          characters_per_sentence_average: characters_per_sentence_average,
+          punctuations_per_sentence_average: punctuations_per_sentence_average,
+          flesch_reading_ease: flesch_reading_ease,
+          flesch_kincaid_grade: flesch_kincaid_grade,
+          lix: lix,
+          smog_index: smog_index,
+          coleman_liau_index: coleman_liau_index
+        }
+      end
+
+      # _count methods
+      def characters_count(ignore_spaces: true)
+        ignore_spaces ? text.delete(" ").length : text.length
+      end
+
+      def words_count
+        words.size
+      end
+
+      def sentences_count
+        return 0 if words_count.zero?
+
+        [1, sentences.size].max
+      end
+
+      def syllables_count
+        words.sum { |word| count_syllables_in_word(word) }
+      end
+
+      def poly_syllabes_count
+        words.count { |word| count_syllables_in_word(word) >= 3 }
+      end
+
+      def punctuations_count
+        punctuations.size
+      end
+
+      # _average methods
+
+      def syllables_per_word_average
+        return 0.0 if words_count.zero? || syllables_count.zero?
+
+        (syllables_count.to_f / words_count).round(1)
+      end
+
+      def letters_per_word_average
+        return 0.0 if words_count.zero? || characters_count.zero?
+
+        (characters_count.to_f / words_count).round(2)
+      end
+
+      def words_per_sentence_average
+        return 0.0 if words_count.zero? || sentences_count.zero?
+
+        (words_count.to_f / sentences_count).round(2)
+      end
+
+      def characters_per_sentence_average
+        return 0.0 if characters_count.zero? || sentences_count.zero?
+
+        (characters_count.to_f / sentences_count).round(2)
+      end
+
+      def punctuations_per_sentence_average
+        return 0.0 if punctuations_count.zero? || sentences_count.zero?
+
+        (punctuations_count.to_f / sentences_count).round(2)
+      end
+
+      def words_per_punctuations_average
+        return 0.0 if words_count.zero? || punctuations_count.zero?
+
+        (words_count.to_f / punctuations_count).round(2)
+      end
+
+      # readability scores
+      def flesch_reading_ease
+        raise NotImplementedError
+      end
+
+      def flesch_kincaid_grade
+        raise NotImplementedError
+      end
+
+      def smog_index
+        if sentences_count >= 3
+          begin
+            smog = 1.043 * Math.sqrt(30.0 * poly_syllabes_count / sentences_count) + 3.1291
+            smog.round(1)
+          rescue ZeroDivisionError
+            0.0
+          end
+        else
+          0.0
+        end
+      end
+
+      def coleman_liau_index
+        return 0.0 if words_per_sentence_average.zero? || letters_per_word_average.zero?
+
+        letters = (letters_per_word_average * 100).round(2)
+        sentences = (1.to_f / words_per_sentence_average * 100).round(2)
+        coleman = 0.0588 * letters - 0.296 * sentences - 15.8
+        coleman.round(2).clamp(0.0, 20.0)
+      end
+
+      def lix
+        return 0.0 if words_count.zero?
+
+        long_words = words.count { |word| word.length > 6 }
+
+        per_long_words = 100.0 * long_words / words_count
+        lix = words_per_sentence_average + per_long_words
+
+        lix.round(2).clamp(0.0, 100.0)
+      end
+
+      # similarity
+      def levenshtein_distance_from(other_text, normalize: true)
+        distance = levenshtein_distance(@text, other_text)
+        return distance unless normalize
+
+        # Normalize to a score out of 100
+        max_length = [@text.length, other_text.length].max
+        normalized_score = if max_length.zero?
+          100
+        else
+          ((max_length - distance).to_f / max_length) * 100
+        end
+
+        normalized_score.round(2)
+      end
+
+      # tokenizers
+      #
+      def punctuations
+        @punctuations ||= text.scan(/[.,!?;:]/)
+      end
+
+      def words
+        @words ||= begin
+          normalized_text = text.downcase.strip
+
+          # Split the sentence into words, including hyphenated words, and excluding numbers
+          normalized_text.scan(/\b[A-Za-zÀ-ÖØ-öø-ÿ'-]+\b/)
+        end
+      end
+
+      def sentences
+        @sentences ||= text.scan(/(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\.|\?|!)(?=\s|$)/)
+      end
+
+      private
+
+      def levenshtein_distance(s1, s2)
+        m = s1.length
+        n = s2.length
+
+        # Return if one of the strings is empty
+        return n if m == 0
+        return m if n == 0
+
+        # Create a matrix
+        matrix = Array.new(m + 1) { Array.new(n + 1) }
+
+        # Initialize the first row and column
+        (0..m).each { |i| matrix[i][0] = i }
+        (0..n).each { |j| matrix[0][j] = j }
+
+        # Fill in the matrix
+        (1..m).each do |i|
+          (1..n).each do |j|
+            cost = (s1[i - 1] == s2[j - 1]) ? 0 : 1
+            matrix[i][j] = [
+              matrix[i - 1][j] + 1,      # Deletion
+              matrix[i][j - 1] + 1,      # Insertion
+              matrix[i - 1][j - 1] + cost  # Substitution
+            ].min
+          end
+        end
+
+        # Return the Levenshtein distance
+        matrix[m][n]
+      end
+    end
+  end
+end

data/lib/text_metrics/processors/french.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "text_metrics/processors/base"
+require "yaml"
+
+module TextMetrics
+  module Processors
+    class French < TextMetrics::Processors::Base
+      GEM_PATH = File.dirname(__FILE__, 2).freeze
+      SYLLABLE_EXCEPTIONS_PATH = File.join(GEM_PATH, "dictionnaries/french_word_syllable_exceptions.yml").freeze
+      SYLLABLE_EXCEPTIONS = YAML.load_file(SYLLABLE_EXCEPTIONS_PATH).freeze
+
+      def flesch_reading_ease
+        sentence_length = words_per_sentence_average
+        syllables_per_word = syllables_per_word_average
+        flesch = 206.835 - 1.015 * sentence_length - 73.6 * syllables_per_word
+
+        flesch.round(2).clamp(0.0, 100.0)
+      end
+
+      def flesch_kincaid_grade
+        sentence_length = words_per_sentence_average
+        syllables_per_word = syllables_per_word_average
+        flesch = (0.55 * sentence_length) + (11.76 * syllables_per_word) - 15.79
+
+        flesch.round(1).clamp(0.0, 18.0)
+      end
+
+      private
+
+      def count_syllables_in_word(word)
+        return SYLLABLE_EXCEPTIONS[word].to_i if with_syllable_exceptions && SYLLABLE_EXCEPTIONS.key?(word)
+
+        word = word.downcase.gsub(/[^a-zàâäéèêëîïôöùûüç]/, "")
+
+        # Define vowel patterns including accents
+        vowels = "aàâeéèêëiîïoôöuùûüy"
+
+        # Remove final silent 'e' or 'es' unless the word is a single letter
+        word.gsub!(/(e|es|ent)$/, "") unless word.size == 1
+
+        # Handle special case for words ending with a consonant followed by 'r' (e.g., 'arbre')
+        if /[^aeiouy]r$/.match?(word)
+          word += "e" # Temporarily treat the final 'r' as part of a vowel sound for syllable counting
+        end
+
+        # Split the word into parts based on vowel-consonant transitions
+        parts = word.scan(/[^#{vowels}]*[#{vowels}]+/)
+
+        # Count the syllables based on vowel groups in each part
+        parts.sum { |part| part.scan(/[#{vowels}]+/).size }
+      end
+    end
+  end
+end

data/lib/text_metrics/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module TextMetrics # :nodoc:
+  VERSION = "0.0.1"
+end

data/lib/text_metrics.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "text_metrics/version"
+require "text_metrics/processors/french"
+require "text_metrics/processors/american_english"
+require "forwardable"
+
+module TextMetrics
+  class Error < StandardError; end
+
+  class TextMetrics
+    extend Forwardable
+    def_delegators :text_metrics_processor, :words_count, :characters_count, :syllables_count,
+      :sentences_count, :words_per_sentence_average, :syllables_per_word_average,
+      :letters_per_word_average, :words_per_sentence_average, :characters_per_sentence_average,
+      :flesch_reading_ease, :flesch_kincaid_grade, :all, :levenshtein_distance_from
+
+    PROCESSORS = {
+      "fr" => Processors::French,
+      "en_us" => Processors::AmericanEnglish
+    }
+
+    attr_reader :text, :language, :text_metrics_processor
+
+    def initialize(text:, language: "en_us")
+      @text = text
+      @language = language
+      @text_metrics_processor = PROCESSORS[language].new(text: text)
+    end
+
+    private
+
+    def processor_for(language)
+      PROCESSORS[language] || raise("Unknown language: #{language}, available languages: #{PROCESSORS.keys}")
+    end
+  end
+
+  def self.new(text:, language: "en_us")
+    TextMetrics.new(text: text, language: language)
+  end
+end

metadata

@@ -0,0 +1,160 @@
+--- !ruby/object:Gem::Specification
+name: text-metrics
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Adrien POLY
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2026-06-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: text-hyphen
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A Ruby gem to compute various metrics for text, Currently focusing on
+  English and French
+email:
+- adrienpoly@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- lib/text_metrics.rb
+- lib/text_metrics/dictionnaries/en_us.txt
+- lib/text_metrics/dictionnaries/fr.txt
+- lib/text_metrics/dictionnaries/french_word_syllable_database.yml
+- lib/text_metrics/dictionnaries/french_word_syllable_exceptions.yml
+- lib/text_metrics/dictionnaries/lexique-383.csv
+- lib/text_metrics/processors/american_english.rb
+- lib/text_metrics/processors/base.rb
+- lib/text_metrics/processors/french.rb
+- lib/text_metrics/version.rb
+homepage: https://github.com/plume-app/text-metrics
+licenses:
+- MIT
+metadata:
+  bug_tracker_uri: https://github.com/plume-app/text-metrics/issues
+  changelog_uri: https://github.com/plume-app/text-metrics/blob/main/CHANGELOG.md
+  documentation_uri: https://github.com/plume-app/text-metrics
+  homepage_uri: https://github.com/plume-app/text-metrics
+  source_code_uri: https://github.com/plume-app/text-metrics
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.19
+signing_key:
+specification_version: 4
+summary: A Ruby gem to compute various metrics for text
+test_files: []