textstat 0.1.9 → 1.0.1

data/lib/textstat/basic_stats.rb

@@ -0,0 +1,197 @@
+module TextStat
+  # Basic text statistics calculations
+  #
+  # This module provides fundamental text analysis methods such as counting
+  # characters, words, syllables, and sentences. These statistics form the
+  # foundation for more advanced readability calculations.
+  #
+  # @author Jakub Polak
+  # @since 1.0.0
+  # @example Basic usage
+  #   text = "Hello world! This is a test."
+  #   TextStat.char_count(text)          # => 23
+  #   TextStat.lexicon_count(text)       # => 6
+  #   TextStat.syllable_count(text)      # => 6
+  #   TextStat.sentence_count(text)      # => 2
+  module BasicStats
+    # Frozen regex constants to avoid recompilation overhead
+    NON_ALPHA_REGEX = /[^a-zA-Z\s]/.freeze
+    SENTENCE_BOUNDARY_REGEX = /[.?!]['\\)\]]*[ |\n][A-Z]/.freeze
+
+    # Cache for Text::Hyphen instances to avoid recreating them for each call
+    @hyphenator_cache = {}
+
+    class << self
+      attr_accessor :hyphenator_cache
+
+      # Get or create a cached Text::Hyphen instance for the specified language
+      #
+      # @param language [String] language code
+      # @return [Text::Hyphen] cached hyphenator instance
+      # @private
+      def get_hyphenator(language)
+        @hyphenator_cache[language] ||= Text::Hyphen.new(language: language, left: 0, right: 0)
+      end
+
+      # Clear all cached hyphenators
+      #
+      # @return [Hash] empty cache
+      # @private
+      def clear_hyphenator_cache
+        @hyphenator_cache.clear
+      end
+    end
+    # Count characters in text
+    #
+    # @param text [String] the text to analyze
+    # @param ignore_spaces [Boolean] whether to ignore spaces in counting
+    # @return [Integer] number of characters
+    # @example
+    #   TextStat.char_count("Hello world!")        # => 11
+    #   TextStat.char_count("Hello world!", false) # => 12
+    def char_count(text, ignore_spaces = true)
+      text = text.delete(' ') if ignore_spaces
+      text.length
+    end
+
+    # Count words (lexicons) in text
+    #
+    # @param text [String] the text to analyze
+    # @param remove_punctuation [Boolean] whether to remove punctuation before counting
+    # @return [Integer] number of words
+    # @example
+    #   TextStat.lexicon_count("Hello, world!")       # => 2
+    #   TextStat.lexicon_count("Hello, world!", false) # => 2
+    def lexicon_count(text, remove_punctuation = true)
+      text = text.gsub(NON_ALPHA_REGEX, '').squeeze(' ') if remove_punctuation
+      text.split.count
+    end
+
+    # Count syllables in text using hyphenation
+    #
+    # Uses the text-hyphen library for accurate syllable counting across
+    # different languages. Supports 22 languages including English, Spanish,
+    # French, German, and more. Hyphenator instances are cached for performance.
+    #
+    # @param text [String] the text to analyze
+    # @param language [String] language code for hyphenation dictionary
+    # @return [Integer] number of syllables
+    # @example
+    #   TextStat.syllable_count("beautiful")          # => 3
+    #   TextStat.syllable_count("hello", "en_us")      # => 2
+    #   TextStat.syllable_count("bonjour", "fr")       # => 2
+    # @see TextStat::DictionaryManager.supported_languages
+    def syllable_count(text, language = 'en_us')
+      return 0 if text.empty?
+
+      text = text.downcase
+      text.gsub(NON_ALPHA_REGEX, '').squeeze(' ') # NOTE: not assigned back (matches original behavior)
+      hyphenator = BasicStats.get_hyphenator(language)
+      count = 0
+      text.split.each do |word|
+        word_hyphenated = hyphenator.visualise(word)
+        count += word_hyphenated.count('-') + 1
+      end
+      count
+    end
+
+    # Count sentences in text
+    #
+    # Identifies sentence boundaries using punctuation marks (.!?) followed
+    # by whitespace and capital letters.
+    #
+    # @param text [String] the text to analyze
+    # @return [Integer] number of sentences
+    # @example
+    #   TextStat.sentence_count("Hello world! How are you?")  # => 2
+    #   TextStat.sentence_count("Dr. Smith went to the U.S.A.") # => 1
+    def sentence_count(text)
+      text.scan(SENTENCE_BOUNDARY_REGEX).map(&:strip).count + 1
+    end
+
+    # Calculate average sentence length
+    #
+    # @param text [String] the text to analyze
+    # @return [Float] average number of words per sentence
+    # @example
+    #   TextStat.avg_sentence_length("Hello world! How are you?")  # => 3.0
+    def avg_sentence_length(text)
+      asl = lexicon_count(text).to_f / sentence_count(text)
+      asl.round(1)
+    rescue ZeroDivisionError
+      0.0
+    end
+
+    # Calculate average syllables per word
+    #
+    # @param text [String] the text to analyze
+    # @param language [String] language code for hyphenation dictionary
+    # @return [Float] average number of syllables per word
+    # @example
+    #   TextStat.avg_syllables_per_word("beautiful morning")  # => 2.5
+    def avg_syllables_per_word(text, language = 'en_us')
+      syllable = syllable_count(text, language)
+      words = lexicon_count(text)
+      syllables_per_word = syllable.to_f / words
+      syllables_per_word.round(1)
+    rescue ZeroDivisionError
+      0.0
+    end
+
+    # Calculate average letters per word
+    #
+    # @param text [String] the text to analyze
+    # @return [Float] average number of letters per word
+    # @example
+    #   TextStat.avg_letter_per_word("hello world")  # => 5.0
+    def avg_letter_per_word(text)
+      letters_per_word = char_count(text).to_f / lexicon_count(text)
+      letters_per_word.round(2)
+    rescue ZeroDivisionError
+      0.0
+    end
+
+    # Calculate average sentences per word
+    #
+    # @param text [String] the text to analyze
+    # @return [Float] average number of sentences per word
+    # @example
+    #   TextStat.avg_sentence_per_word("Hello world! How are you?")  # => 0.4
+    def avg_sentence_per_word(text)
+      sentence_per_word = sentence_count(text).to_f / lexicon_count(text)
+      sentence_per_word.round(2)
+    rescue ZeroDivisionError
+      0.0
+    end
+
+    # Count polysyllabic words (3+ syllables)
+    #
+    # Optimized to count syllables for all words in one pass using a cached hyphenator.
+    #
+    # @param text [String] the text to analyze
+    # @param language [String] language code for hyphenation dictionary
+    # @return [Integer] number of polysyllabic words
+    # @example
+    #   TextStat.polysyllab_count("beautiful complicated")  # => 2
+    def polysyllab_count(text, language = 'en_us')
+      return 0 if text.empty?
+
+      # Clean and split text once
+      cleaned_text = text.downcase.gsub(NON_ALPHA_REGEX, '').squeeze(' ')
+      words = cleaned_text.split
+      return 0 if words.empty?
+
+      # Use cached hyphenator for better performance
+      hyphenator = BasicStats.get_hyphenator(language)
+      count = 0
+      words.each do |word|
+        next if word.empty?
+
+        word_hyphenated = hyphenator.visualise(word)
+        syllables = word_hyphenated.count('-') + 1
+        count += 1 if syllables >= 3
+      end
+      count
+    end
+  end
+end

data/lib/textstat/dictionary_manager.rb

@@ -0,0 +1,168 @@
+require 'set'
+
+module TextStat
+  # Dictionary management with high-performance caching
+  #
+  # This module handles loading and caching of language-specific dictionaries
+  # used for identifying difficult words. The caching system provides a 36x
+  # performance improvement over reading dictionaries from disk on every call.
+  #
+  # @author Jakub Polak
+  # @since 1.0.0
+  # @example Performance optimization
+  #   # First call loads dictionary from disk
+  #   TextStat.difficult_words(text, 'en_us')  # ~0.047s
+  #
+  #   # Subsequent calls use cached dictionary
+  #   TextStat.difficult_words(text, 'en_us')  # ~0.0013s (36x faster!)
+  #
+  #   # Check cache status
+  #   TextStat::DictionaryManager.cache_size        # => 1
+  #   TextStat::DictionaryManager.cached_languages  # => ['en_us']
+  #
+  # @example Multi-language support
+  #   TextStat.difficult_words(english_text, 'en_us')
+  #   TextStat.difficult_words(spanish_text, 'es')
+  #   TextStat.difficult_words(french_text, 'fr')
+  #   TextStat::DictionaryManager.cache_size  # => 3
+  module DictionaryManager
+    # Cache for loaded dictionaries
+    @dictionary_cache = {}
+    @dictionary_path = nil
+
+    class << self
+      attr_accessor :dictionary_cache
+
+      # Set dictionary path
+      #
+      # @param path [String] path to dictionary directory
+      # @return [String] the set path
+      def dictionary_path=(path)
+        @dictionary_path = path
+      end
+
+      # Load dictionary with automatic caching
+      #
+      # Loads a language-specific dictionary from disk and caches it in memory
+      # for subsequent calls. This provides significant performance improvements
+      # for repeated operations. Uses optimized file reading with streaming for
+      # better performance and memory efficiency.
+      #
+      # @param language [String] language code (e.g., 'en_us', 'es', 'fr')
+      # @return [Set] set of easy words for the specified language
+      # @example
+      #   dict = TextStat::DictionaryManager.load_dictionary('en_us')
+      #   dict.include?('hello')  # => true
+      #   dict.include?('comprehensive')  # => false
+      # @see #supported_languages
+      def load_dictionary(language)
+        # Return cached dictionary if available
+        return @dictionary_cache[language] if @dictionary_cache[language]
+
+        # Load dictionary from file
+        dictionary_file = File.join(dictionary_path, "#{language}.txt")
+        easy_words = Set.new
+
+        if File.exist?(dictionary_file)
+          # Use foreach for streaming - efficient and memory-friendly for large files
+          File.foreach(dictionary_file, chomp: true) do |line|
+            easy_words << line
+          end
+        end
+
+        # Cache the loaded dictionary
+        @dictionary_cache[language] = easy_words
+        easy_words
+      end
+
+      # Clear all cached dictionaries
+      #
+      # Removes all dictionaries from memory cache. Useful for memory management
+      # in long-running applications or when switching between different sets
+      # of languages.
+      #
+      # @return [Hash] empty cache hash
+      # @example
+      #   TextStat::DictionaryManager.cache_size  # => 3
+      #   TextStat::DictionaryManager.clear_cache
+      #   TextStat::DictionaryManager.cache_size  # => 0
+      def clear_cache
+        @dictionary_cache.clear
+      end
+
+      # Get list of cached languages
+      #
+      # @return [Array<String>] array of language codes currently in cache
+      # @example
+      #   TextStat::DictionaryManager.cached_languages  # => ['en_us', 'es', 'fr']
+      def cached_languages
+        @dictionary_cache.keys
+      end
+
+      # Get number of cached dictionaries
+      #
+      # @return [Integer] number of dictionaries currently in cache
+      # @example
+      #   TextStat::DictionaryManager.cache_size  # => 3
+      def cache_size
+        @dictionary_cache.size
+      end
+
+      # Get path to dictionary files
+      #
+      # @return [String] absolute path to dictionary directory
+      # @example
+      #   TextStat::DictionaryManager.dictionary_path
+      #   # => \"/path/to/gem/lib/dictionaries\"
+      def dictionary_path
+        @dictionary_path ||= File.join(TextStat::GEM_PATH, 'lib', 'dictionaries')
+      end
+    end
+
+    # Count difficult words in text
+    #
+    # Identifies words that are considered difficult based on:
+    # 1. Not being in the language's easy words dictionary
+    # 2. Having more than one syllable
+    #
+    # This method uses the cached dictionary and hyphenator systems for optimal performance.
+    #
+    # @param text [String] the text to analyze
+    # @param language [String] language code for dictionary selection
+    # @param return_words [Boolean] whether to return words array or count
+    # @return [Integer, Set] number of difficult words or set of difficult words
+    # @example Count difficult words
+    #   TextStat.difficult_words(\"This is a comprehensive analysis\")  # => 2
+    #
+    # @example Get list of difficult words
+    #   words = TextStat.difficult_words(\"comprehensive analysis\", 'en_us', true)
+    #   words.to_a  # => [\"comprehensive\", \"analysis\"]
+    #
+    # @example Multi-language support
+    #   TextStat.difficult_words(spanish_text, 'es')  # Spanish dictionary
+    #   TextStat.difficult_words(french_text, 'fr')   # French dictionary
+    def difficult_words(text, language = 'en_us', return_words = false)
+      easy_words = DictionaryManager.load_dictionary(language)
+
+      # Clean and split text once
+      text_list = text.downcase.gsub(/[^0-9a-z ]/i, '').split
+      return return_words ? Set.new : 0 if text_list.empty?
+
+      # Get cached hyphenator for syllable counting
+      hyphenator = BasicStats.get_hyphenator(language)
+      diff_words_set = Set.new
+
+      # Process each word once
+      text_list.each do |word|
+        next if easy_words.include?(word)
+
+        # Count syllables inline using cached hyphenator
+        word_hyphenated = hyphenator.visualise(word)
+        syllables = word_hyphenated.count('-') + 1
+        diff_words_set.add(word) if syllables > 1
+      end
+
+      return_words ? diff_words_set : diff_words_set.length
+    end
+  end
+end

data/lib/textstat/main.rb

@@ -0,0 +1,137 @@
+require 'text-hyphen'
+require_relative 'basic_stats'
+require_relative 'dictionary_manager'
+require_relative 'readability_formulas'
+
+module TextStat
+  # Path to the TextStat gem installation directory
+  #
+  # This constant is used internally to locate dictionary files and other
+  # gem resources. It points to the root directory of the installed gem.
+  #
+  # @return [String] absolute path to gem root directory
+  # @example
+  #   TextStat::GEM_PATH  # => \"/path/to/gems/textstat-1.0.0\"
+  GEM_PATH = File.dirname(File.dirname(File.dirname(__FILE__)))
+
+  # Main class providing text readability analysis
+  #
+  # This class combines all TextStat modules to provide a unified interface
+  # for text analysis. It includes basic statistics, dictionary management,
+  # and readability formulas in a single class.
+  #
+  # The class maintains backward compatibility through method delegation,
+  # ensuring that existing code continues to work seamlessly.
+  #
+  # @author Jakub Polak
+  # @since 1.0.0
+  # @example Creating an instance
+  #   analyzer = TextStat::Main.new
+  #   analyzer.flesch_reading_ease(\"Sample text\")  # => 83.32
+  #
+  # @example Using class methods (backward compatibility)
+  #   TextStat::Main.flesch_reading_ease(\"Sample text\")  # => 83.32
+  #   TextStat.flesch_reading_ease(\"Sample text\")        # => 83.32
+  class Main
+    include BasicStats
+    include DictionaryManager
+    include ReadabilityFormulas
+
+    # Legacy class methods for backward compatibility
+    class << self
+      # Handle method delegation for backward compatibility
+      #
+      # This method ensures that all instance methods can be called as class methods,
+      # maintaining compatibility with the pre-1.0 API.
+      #
+      # @param method_name [Symbol] the method name being called
+      # @param args [Array] method arguments
+      # @param kwargs [Hash] keyword arguments
+      # @param block [Proc] block if provided
+      # @return [Object] result of the method call
+      # @private
+      def method_missing(method_name, *args, **kwargs, &block)
+        instance = new
+        if instance.respond_to?(method_name)
+          instance.send(method_name, *args, **kwargs, &block)
+        else
+          super
+        end
+      end
+
+      # Check if method exists for delegation
+      #
+      # @param method_name [Symbol] the method name to check
+      # @param include_private [Boolean] whether to include private methods
+      # @return [Boolean] true if method exists
+      # @private
+      def respond_to_missing?(method_name, include_private = false)
+        new.respond_to?(method_name, include_private) || super
+      end
+
+      # Set dictionary path for all instances
+      #
+      # @param path [String] path to dictionary directory
+      # @return [String] the set path
+      # @example
+      #   TextStat::Main.dictionary_path = \"/custom/dictionaries\"
+      def dictionary_path=(path)
+        DictionaryManager.dictionary_path = path
+      end
+
+      # Get current dictionary path
+      #
+      # @return [String] current dictionary path
+      # @example
+      #   TextStat::Main.dictionary_path  # => \"/path/to/dictionaries\"
+      def dictionary_path
+        DictionaryManager.dictionary_path
+      end
+
+      # Clear all cached dictionaries
+      #
+      # @return [Hash] empty cache
+      # @example
+      #   TextStat::Main.clear_dictionary_cache
+      def clear_dictionary_cache
+        DictionaryManager.clear_cache
+      end
+
+      # Load dictionary for specified language
+      #
+      # @param language [String] language code
+      # @return [Set] set of easy words for the language
+      # @example
+      #   TextStat::Main.load_dictionary('en_us')
+      def load_dictionary(language)
+        DictionaryManager.load_dictionary(language)
+      end
+    end
+  end
+end
+
+# For backward compatibility, expose TextStat module class methods
+# This ensures that TextStat.method_name works exactly like TextStat::Main.method_name
+TextStat.extend(Module.new do
+  # Handle method delegation at module level
+  #
+  # @param method_name [Symbol] the method name being called
+  # @param args [Array] method arguments
+  # @param kwargs [Hash] keyword arguments
+  # @param block [Proc] block if provided
+  # @return [Object] result of the method call
+  # @private
+  def method_missing(method_name, *args, **kwargs, &block)
+    TextStat::Main.send(method_name, *args, **kwargs, &block)
+  end
+
+  # Check if method exists for delegation
+  #
+  # @param method_name [Symbol] the method name to check
+  # @param include_private [Boolean] whether to include private methods
+  # @return [Boolean] true if method exists
+  # @private
+  def respond_to_missing?(method_name, include_private = false)
+    TextStat::Main.respond_to?(method_name, include_private) || super
+  end
+end)