psychowl 0.1.0

data/lib/psychowl/postgres.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Psychowl
+  # PostgreSQL full-text search configurations for supported languages, for
+  # to_tsvector / to_tsquery. Languages without a built-in stemmer map to
+  # "simple".
+  #
+  #   config = Psychowl.detect(body)&.lang&.pg_regconfig || "simple"
+  #   Post.where("to_tsvector(?::regconfig, body) @@ plainto_tsquery(?::regconfig, ?)", config, config, query)
+  module Postgres
+    REGCONFIGS = {
+      'ara' => 'arabic',
+      'deu' => 'german',
+      'eng' => 'english',
+      'fra' => 'french',
+      'ind' => 'indonesian',
+      'ita' => 'italian',
+      'nld' => 'dutch',
+      'por' => 'portuguese',
+      'rus' => 'russian',
+      'spa' => 'spanish',
+      'tur' => 'turkish'
+    }.freeze
+
+    # @param lang [Lang, String, Symbol]
+    # @return [String] a built-in PostgreSQL text search configuration
+    def self.regconfig(lang) = REGCONFIGS.fetch(Lang.fetch(lang).code, 'simple')
+  end
+
+  # PostgreSQL text search configuration per language.
+  class Lang
+    # @return [String] PostgreSQL text search configuration, see {Postgres}
+    def pg_regconfig = Postgres.regconfig(self)
+  end
+end

data/lib/psychowl/railtie.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Psychowl
+  # Makes the `language:` validator available in Rails apps.
+  class Railtie < Rails::Railtie
+    initializer 'psychowl.active_model' do
+      require 'psychowl/active_model'
+    end
+  end
+end

data/lib/psychowl/ruby_engine/segmentation.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Psychowl
+  # Per-sentence detection, mirrored by segment.rs.
+  module RubyEngine
+    # Sentence-ending marks that need a following space ("3.14" is no boundary).
+    SENTENCE_MARKS = %W[. ! ? \u2026].freeze
+    # CJK sentence-ending marks, no space needed.
+    CLOSING_MARKS = %W[\u3002 \uFF01 \uFF1F].freeze
+
+    class << self
+      # Runs of same-language sentences. Sentences without a language join the
+      # previous run (or the first). Each run is re-detected whole and keeps its
+      # first sentence's result if that changes the language.
+      #
+      # @return [Array<Array(Integer, Integer, Integer, Integer, Float)>]
+      #   [[start, end, language, script, confidence], ...] with character
+      #   offsets, end exclusive
+      def segments(text, filter_mode, filter_langs)
+        runs = [] # [start, end, first sentence's detection]
+        leading_start = nil
+
+        sentences(text).each do |start, finish|
+          result = detect(text[start...finish], filter_mode, filter_langs)
+
+          if result.nil?
+            if runs.empty?
+              leading_start ||= start
+            else
+              runs.last[1] = finish
+            end
+          elsif runs.any? && runs.last[2][0] == result[0]
+            runs.last[1] = finish
+          else
+            runs << [leading_start || start, finish, result]
+            leading_start = nil
+          end
+        end
+
+        runs.map do |start, finish, first|
+          whole = detect(text[start...finish], filter_mode, filter_langs)
+          whole && whole[0] == first[0] ? [start, finish, *whole] : [start, finish, *first]
+        end
+      end
+
+      # [start, end] character offsets. A sentence ends after a newline, 。！？,
+      # or . ! ? … followed by a space; trailing spaces stay with it.
+      def sentences(text)
+        chars = text.chars
+        sentences = []
+        start = 0
+        index = 0
+
+        while index < chars.size
+          char = chars[index]
+          boundary = false
+
+          if char == "\n" || CLOSING_MARKS.include?(char)
+            index += 1
+            boundary = true
+          elsif SENTENCE_MARKS.include?(char)
+            index += 1 while index < chars.size && SENTENCE_MARKS.include?(chars[index])
+            boundary = index == chars.size || space?(chars[index])
+          else
+            index += 1
+          end
+          next unless boundary
+
+          index += 1 while index < chars.size && space?(chars[index])
+          sentences << [start, index]
+          start = index
+        end
+
+        sentences << [start, chars.size] if start < chars.size
+        sentences
+      end
+
+      private
+
+      # ASCII whitespace and controls, no-break space, ideographic space.
+      def space?(char) = char <= ' ' || char == "\u00A0" || char == "\u3000"
+    end
+  end
+end

data/lib/psychowl/ruby_engine/trigrams.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Psychowl
+  # Trigram model, mirrored by trigram.rs.
+  module RubyEngine
+    # A profile trigram missing from the text costs this much distance.
+    MAX_TRIGRAM_DISTANCE = 300
+    MAX_TOTAL_DISTANCE = MAX_TRIGRAM_DISTANCE * MAX_TRIGRAM_DISTANCE
+    # Only the most frequent trigrams of the text are compared.
+    TEXT_TRIGRAMS_SIZE = 600
+
+    class << self
+      private
+
+      # Similarity between the text's trigram ranking and each profile.
+      def trigram_scores(lowercase, languages)
+        positions = trigram_positions(lowercase)
+        max_distance = positions.size * MAX_TRIGRAM_DISTANCE
+
+        scores = languages.to_h do |language|
+          profile = Tables::TRIGRAMS[language]
+          if profile
+            distance = trigram_distance(profile, positions)
+            [language, (max_distance - distance).to_f / max_distance]
+          else
+            [language, 0.0]
+          end
+        end
+        [scores, positions.size]
+      end
+
+      # {trigram => rank}: the text's most frequent trigrams, most frequent
+      # first (ties broken by the trigram itself, descending).
+      def trigram_positions(lowercase)
+        ranked = count_trigrams(lowercase).sort do |(trigram_a, count_a), (trigram_b, count_b)|
+          [count_b, trigram_b] <=> [count_a, trigram_a]
+        end
+        ranked.first(TEXT_TRIGRAMS_SIZE).each_with_index.to_h { |(trigram, _count), rank| [trigram, rank] }
+      end
+
+      # {trigram => occurrences}. Punctuation and digits act as spaces;
+      # trigrams that are only word boundary are skipped.
+      def count_trigrams(lowercase)
+        counts = Hash.new(0)
+        first = ' '
+        second = nil
+
+        "#{lowercase.tr(Tables::STOP_CHARS, ' ')} ".each_char do |third|
+          if second.nil?
+            second = third
+            next
+          end
+
+          counts["#{first}#{second}#{third}"] += 1 unless second == ' ' && (first == ' ' || third == ' ')
+          first = second
+          second = third
+        end
+        counts
+      end
+
+      def trigram_distance(profile, positions)
+        total = 0
+        profile.each_with_index do |trigram, index|
+          position = positions[trigram]
+          total += position ? (position - index).abs : MAX_TRIGRAM_DISTANCE
+        end
+
+        unique = positions.size
+        total -= (MAX_TRIGRAM_DISTANCE - unique) * MAX_TRIGRAM_DISTANCE if unique < MAX_TRIGRAM_DISTANCE
+        total.clamp(0, MAX_TOTAL_DISTANCE)
+      end
+    end
+  end
+end

data/lib/psychowl/ruby_engine.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+module Psychowl
+  # Pure Ruby engine, mirrored by the psychowl crate; parity_test.rb keeps them
+  # identical. Primitives in and out: languages and scripts are Tables indices,
+  # text is valid UTF-8, a filter is an Engine::FILTER_* mode plus indices.
+  # Internal calls never go through Engine.
+  module RubyEngine
+    class << self
+      # @return [Array(Integer, Integer, Float), nil] [language, script, confidence]
+      def detect(text, filter_mode, filter_langs)
+        case evaluate(text, filter_mode, filter_langs)
+        in [:decided, result] then result
+        in [:scored, script, [[best, best_score], [_, runner_up_score], *], trigram_count]
+          [best, script, confidence(best_score, runner_up_score, trigram_count)]
+        in [:nothing] then nil
+        end
+      end
+
+      # Every candidate language with its score, best first.
+      #
+      # @return [Array<Array(Integer, Float)>] [[language, score], ...]
+      def candidates(text, filter_mode, filter_langs)
+        case evaluate(text, filter_mode, filter_langs)
+        in [:decided, [language, _script, confidence]] then [[language, confidence]]
+        in [:scored, _script, scores, _trigram_count] then scores
+        in [:nothing] then []
+        end
+      end
+
+      # @return [Integer, nil] the script with the most characters
+      def detect_script(text) = main_script(script_counts(text))
+
+      # @return [Array<Integer>] number of characters per script, by script index
+      def script_counts(text)
+        Tables::SCRIPTS.map { text.count(it.charset) }
+      end
+
+      # @return [Array<Array(Integer, Integer, Float), nil>] one #detect result per text
+      def detect_many(texts, filter_mode, filter_langs)
+        texts.map { detect(it, filter_mode, filter_langs) }
+      end
+
+      private
+
+      # Mirrors the Rust `Outcome`:
+      #   [:nothing]                                 no letters / no allowed language
+      #   [:decided, [language, script, confidence]] one allowed language, or Han
+      #   [:scored, script, scores, trigram_count]   several languages, best first
+      def evaluate(text, filter_mode, filter_langs)
+        counts = script_counts(text)
+        script = main_script(counts)
+        return [:nothing] if script.nil?
+
+        if script == Tables::HAN
+          result = detect_han(counts, filter_mode, filter_langs)
+          return result ? [:decided, result] : [:nothing]
+        end
+
+        case allowed(Tables::SCRIPTS[script].languages, filter_mode, filter_langs)
+        in [] then [:nothing]
+        in [only] then [:decided, [only, script, 1.0]]
+        in languages then [:scored, script, *language_scores(text, languages)]
+        end
+      end
+
+      # Ties go to the script listed first in scripts.tsv.
+      def main_script(counts)
+        best = nil
+        counts.each_with_index do |count, index|
+          best = index if count.positive? && (best.nil? || count > counts[best])
+        end
+        best
+      end
+
+      def allowed(languages, filter_mode, filter_langs)
+        case filter_mode
+        when Engine::FILTER_ALLOW then languages.select { filter_langs.include?(it) }
+        when Engine::FILTER_DENY then languages.reject { filter_langs.include?(it) }
+        else languages
+        end
+      end
+
+      # Han characters are shared by Chinese and Japanese; kana tips it to
+      # Japanese.
+      def detect_han(counts, filter_mode, filter_langs)
+        languages = allowed(Tables::SCRIPTS[Tables::HAN].languages, filter_mode, filter_langs)
+        mandarin = languages.include?(Tables::CMN)
+        japanese = languages.include?(Tables::JPN)
+
+        case [mandarin, japanese]
+        in [false, false] then return nil
+        in [true, false] then return [Tables::CMN, Tables::HAN, 1.0]
+        in [false, true] then return [Tables::JPN, Tables::HAN, 1.0]
+        in [true, true] then nil
+        end
+
+        kana = counts[Tables::HIRAGANA] + counts[Tables::KATAKANA]
+        kana_ratio = kana.to_f / (counts[Tables::HAN] + kana)
+
+        if kana_ratio > 0.2 then [Tables::JPN, Tables::HAN, 1.0]
+        elsif kana_ratio > 0.05 then [Tables::JPN, Tables::HAN, 0.5]
+        elsif kana_ratio > 0.02 then [Tables::CMN, Tables::HAN, 0.5]
+        else [Tables::CMN, Tables::HAN, 1.0]
+        end
+      end
+
+      # @return [Array(Array<Array(Integer, Float)>, Integer)] scores best first, trigram count
+      def language_scores(text, languages)
+        lowercase = text.downcase
+        alphabet_scores, char_count = alphabet_scores(lowercase, languages)
+        trigram_scores, trigram_count = trigram_scores(lowercase, languages)
+
+        alphabet_weight = alphabet_weight(char_count)
+        trigram_weight = 1.0 - alphabet_weight
+
+        scores = languages.map do |language|
+          score = (alphabet_scores[language] * alphabet_weight) + (trigram_scores[language] * trigram_weight)
+          [language, score]
+        end
+        [sort_by_score(scores), trigram_count]
+      end
+
+      # 2/3 for an empty text, falling to 1/3 at 100 letters and beyond.
+      def alphabet_weight(char_count) = (-(char_count / 300.0) + (2.0 / 3.0)).clamp(1.0 / 3.0, 2.0 / 3.0)
+
+      # Highest score first; ties go to the lower language index.
+      def sort_by_score(scores)
+        scores.sort do |(language_a, score_a), (language_b, score_b)|
+          order = score_b <=> score_a
+          order.zero? ? language_a <=> language_b : order
+        end
+      end
+
+      # Share of letters that belong to each language's alphabet. A letter
+      # outside the alphabet counts against the language.
+      def alphabet_scores(lowercase, languages)
+        char_count = lowercase.length - lowercase.count(Tables::STOP_CHARS)
+
+        unless languages.any? { Tables::ALPHABETS.key?(it) }
+          return [languages.to_h { [it, 1.0] }, 1]
+        end
+
+        scores = languages.to_h do |language|
+          letters = Tables::ALPHABETS[language]
+          hits = letters ? lowercase.count(letters) : 0
+          raw = (2 * hits) - char_count
+          raw = 0 if raw.negative?
+          [language, raw.to_f / char_count]
+        end
+        [scores, char_count]
+      end
+
+      # How far the best score is ahead of the runner-up, scaled by how much
+      # evidence (trigrams) there was.
+      def confidence(best, runner_up, count)
+        return 0.0 if best.zero?
+        return best if runner_up.zero?
+
+        confident_rate = (3.0 / count) + 0.015
+        rate = (best - runner_up) / runner_up
+        rate > confident_rate ? 1.0 : rate / confident_rate
+      end
+    end
+  end
+end

data/lib/psychowl/script.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Psychowl
+  # A writing system, e.g. Latin or Cyrillic. Instances are canonical.
+  #
+  # @!attribute [r] name
+  #   @return [String] e.g. "Cyrillic"
+  # @!attribute [r] langs
+  #   @return [Array<Lang>] supported languages written in this script
+  Script = ::Data.define(:name, :langs) do
+    # @return [String] the script name
+    def to_s = name
+
+    def inspect = "#<#{self.class} #{name}>"
+  end
+
+  # The canonical instances and lookup by name.
+  class Script
+    singleton_class.prepend(Lookup) # ahead of Data.define's own `[]` constructor
+
+    ALL = Tables::SCRIPTS.map do |script|
+      new(name: script.name, langs: script.languages.map { Lang.at(it) }.freeze)
+    end.freeze
+    BY_NAME = ALL.to_h { |script| [script.name.downcase, script] }.freeze
+    private_constant :ALL, :BY_NAME
+
+    class << self
+      # @return [Array<Script>] every known script (frozen)
+      def all = ALL
+
+      private :new
+
+      private
+
+      def by_key = BY_NAME
+      def unknown_key_message = 'unknown script'
+    end
+  end
+end

data/lib/psychowl/segment.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Psychowl
+  # A stretch of text in one language, from {Psychowl.segments}.
+  #
+  # @!attribute [r] text
+  #   @return [String] the segment's text
+  # @!attribute [r] range
+  #   @return [Range] character offsets in the original text
+  # @!attribute [r] info
+  #   @return [Info] the detection for this segment
+  Segment = ::Data.define(:text, :range, :info) do
+    # @return [Lang]
+    def lang = info.lang
+
+    # @return [Script]
+    def script = info.script
+
+    # @return [Float]
+    def confidence = info.confidence
+
+    def inspect = "#<#{self.class} #{range} #{info.lang.code} #{text[0, 30].inspect}>"
+  end
+end

data/lib/psychowl/tables.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Psychowl
+  # The data tables shared with the Rust crate, read once at load time.
+  # Languages and scripts are referred to by line index, as in the crate.
+  module Tables
+    DIR = File.expand_path('../../ext/psychowl_native/core/data', __dir__)
+
+    # ASCII controls, spaces, digits and punctuation, as a String#count set.
+    STOP_CHARS = "\u0000-@[-`{-~"
+
+    class << self
+      def rows(file) = DataFile.lines(File.join(DIR, file)).map { it.split("\t") }
+
+      # Escapes the characters String#count treats specially.
+      def charset(chars) = chars.gsub(/[\\^-]/) { |char| "\\#{char}" }
+
+      def range_charset(ranges)
+        ranges.split(',').map do |range|
+          low, high = range.split('-').map { it.to_i(16).chr(Encoding::UTF_8) }
+          high ? "#{charset(low)}-#{charset(high)}" : charset(low)
+        end.join
+      end
+    end
+
+    # [[code, iso639_1, eng_name, name], ...]
+    LANGUAGES = rows('languages.tsv').map(&:freeze).freeze
+    LANGUAGE_INDEX = LANGUAGES.each_with_index.to_h { |(code, *), index| [code, index] }.freeze
+
+    Script = ::Data.define(:name, :languages, :charset)
+    # [Script(name, [language index, ...], String#count set), ...]
+    SCRIPTS = rows('scripts.tsv').map do |name, codes, ranges|
+      languages = codes == '-' ? [] : codes.split(',').map { LANGUAGE_INDEX.fetch(it) }
+      Script.new(name: name.freeze, languages: languages.freeze, charset: range_charset(ranges).freeze)
+    end.freeze
+    SCRIPT_INDEX = SCRIPTS.each_with_index.to_h { |script, index| [script.name, index] }.freeze
+
+    # {language index => String#count set of its letters}
+    ALPHABETS = rows('alphabets.tsv').to_h do |code, letters|
+      [LANGUAGE_INDEX.fetch(code), charset(letters).freeze]
+    end.freeze
+
+    # {language index => [trigram, ...]} most frequent first, " " for a word boundary.
+    TRIGRAMS = Dir[File.join(DIR, 'trigrams', '*.txt')].to_h do |path|
+      code = File.basename(path, '.txt')
+      [LANGUAGE_INDEX.fetch(code), DataFile.lines(path).map { it.tr('_', ' ').freeze }.freeze]
+    end.freeze
+
+    HAN = SCRIPT_INDEX.fetch('Han')
+    HIRAGANA = SCRIPT_INDEX.fetch('Hiragana')
+    KATAKANA = SCRIPT_INDEX.fetch('Katakana')
+    CMN = LANGUAGE_INDEX['cmn']
+    JPN = LANGUAGE_INDEX['jpn']
+  end
+end

data/lib/psychowl/text.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Psychowl
+  # Input validation shared by both engines: they only ever see valid UTF-8.
+  module Text
+    class << self
+      # @param text [String, #to_str]
+      # @return [String] valid UTF-8
+      # @raise [TypeError] not a String
+      # @raise [EncodingError] invalid bytes, or not convertible to UTF-8
+      def prepare(text)
+        string = String.try_convert(text)
+        raise TypeError, "no implicit conversion of #{text.nil? ? 'nil' : text.class} into String" unless string
+
+        utf8 =
+          case string.encoding
+          when Encoding::UTF_8, Encoding::US_ASCII, Encoding::BINARY then string.dup.force_encoding(Encoding::UTF_8)
+          else string.encode(Encoding::UTF_8)
+          end
+        raise EncodingError, 'text is not valid UTF-8' unless utf8.valid_encoding?
+
+        utf8
+      end
+    end
+  end
+end

data/lib/psychowl/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Psychowl
+  VERSION = '0.1.0'
+end

data/lib/psychowl.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require_relative 'psychowl/version'
+require_relative 'psychowl/data_file'
+require_relative 'psychowl/tables'
+require_relative 'psychowl/text'
+require_relative 'psychowl/engine'
+require_relative 'psychowl/ruby_engine'
+require_relative 'psychowl/ruby_engine/segmentation'
+require_relative 'psychowl/ruby_engine/trigrams'
+require_relative 'psychowl/lookup'
+require_relative 'psychowl/lang'
+require_relative 'psychowl/script'
+require_relative 'psychowl/info'
+require_relative 'psychowl/segment'
+require_relative 'psychowl/detector'
+
+# Natural language and script detection. It knows what language you speak.
+# It knows you skipped your lesson.
+#
+#   info = Psychowl.detect("Die Deutsche Bahn ist heute pünktlich. Wir ermitteln.")
+#   info.lang.code   # => "deu"
+#   info.script.name # => "Latin"
+module Psychowl
+  DEFAULT_DETECTOR = Detector.new
+  private_constant :DEFAULT_DETECTOR
+
+  class << self
+    # @param text [String]
+    # @param allowlist [Array<Lang, String, Symbol>, nil] only consider these
+    # @param denylist [Array<Lang, String, Symbol>, nil] never consider these
+    # @return [Info, nil] nil when no supported language was found
+    # @raise [TypeError] text is not a String
+    # @raise [EncodingError] text is not valid UTF-8 (or convertible to it)
+    # @raise [ArgumentError] invalid allowlist/denylist, see {Detector#initialize}
+    def detect(text, allowlist: nil, denylist: nil) = detector(allowlist, denylist).detect(text)
+
+    # @param (see .detect)
+    # @return [Lang, nil]
+    def detect_lang(text, allowlist: nil, denylist: nil) = detector(allowlist, denylist).detect_lang(text)
+
+    # @param text [String]
+    # @return [Script, nil]
+    def detect_script(text) = DEFAULT_DETECTOR.detect_script(text)
+
+    # @param (see .detect)
+    # @param limit [Integer, nil]
+    # @return [Array<Array(Lang, Float)>] see {Detector#candidates}
+    def candidates(text, limit: nil, allowlist: nil, denylist: nil)
+      detector(allowlist, denylist).candidates(text, limit:)
+    end
+
+    # @param texts [Array<String>]
+    # @param allowlist [Array<Lang, String, Symbol>, nil]
+    # @param denylist [Array<Lang, String, Symbol>, nil]
+    # @return [Array<Info, nil>] see {Detector#detect_many}
+    def detect_many(texts, allowlist: nil, denylist: nil) = detector(allowlist, denylist).detect_many(texts)
+
+    # @param (see .detect)
+    # @return [Array<Segment>] see {Detector#segments}
+    def segments(text, allowlist: nil, denylist: nil) = detector(allowlist, denylist).segments(text)
+
+    # @param text [String]
+    # @return [Hash{Script => Float}] see {Detector#scripts}
+    def scripts(text) = DEFAULT_DETECTOR.scripts(text)
+
+    # @return [Symbol] :native when the Rust extension is loaded, :ruby otherwise
+    def backend = Engine.native? ? :native : :ruby
+
+    private
+
+    def detector(allowlist, denylist)
+      return DEFAULT_DETECTOR if allowlist.nil? && denylist.nil?
+
+      Detector.new(allowlist:, denylist:)
+    end
+  end
+end
+
+require_relative 'psychowl/postgres'
+require_relative 'psychowl/native_speedup'
+require_relative 'psychowl/railtie' if defined?(Rails::Railtie)