psychowl 0.1.0

data/ext/psychowl_native/core/data/trigrams/vie.txt

@@ -0,0 +1,302 @@
+# vie: 300 most frequent trigrams, most frequent first. "_" marks a word boundary.
+# Ported from whatlang 0.18.0 (MIT), see ../LICENSE-whatlang.
+ng_
+̣c_
+́c_
+_qu
+_th
+à_
+nh_
+_ng
+̣i_
+_nh
+và
+_va
+̀n_
+uyê
+_ph
+_ca
+quy
+ền
+yề
+̀i_
+_ch
+̀nh
+_tr
+_cu
+ngư
+i_n
+gươ
+ườ
+́t_
+ời
+_gi
+ác
+_co
+̣t_
+ó_
+c_t
+ự_
+n_t
+cá
+ông
+_kh
+ượ
+ợc
+_tư
+_đư
+iệ
+đươ
+ìn
+́i_
+_ha
+có
+i_đ
+gia
+_đê
+pha
+_mo
+ọi
+mọ
+như
+n_n
+củ
+_ba
+̣n_
+̉a_
+ủa
+n_c
+̀u_
+̃ng
+ân_
+ều
+ất
+_bi
+tự
+hôn
+_vi
+g_t
+_la
+n_đ
+đề
+nhâ
+_ti
+t_c
+_đô
+ên_
+bả
+hiê
+u_c
+_tô
+do_
+hân
+_do
+ch_
+́_q
+̀_t
+_na
+́n_
+ay_
+_hi
+àn
+̣_d
+ới
+há
+_đi
+hay
+g_n
+_mô
+ốc
+uố
+n_v
+ội
+hữ
+thư
+́p_
+quô
+_ho
+̣p_
+nà
+ào
+̀ng
+̉n_
+ị_
+́ch
+ôn_
+̀o_
+khô
+c_h
+i_c
+c_đ
+_hô
+i_v
+tro
+_đa
+́ng
+mộ
+i_t
+ột
+g_v
+ia_
+̣ng
+ản
+ướ
+ữn
+̉ng
+h_t
+hư_
+ện
+n_b
+ộc
+ả_
+là
+c_c
+g_c
+_đo
+̉_c
+n_h
+hà
+hộ
+_bâ
+ã_
+̀y_
+_vơ
+̣_t
+̉i_
+iế
+_cô
+t_t
+g_đ
+ức
+iên
+_vê
+viê
+vớ
+h_v
+ớc
+ực
+ật
+tha
+̉m_
+ron
+ong
+áp
+g_b
+hươ
+_sư
+a_c
+sự
+̉o_
+ảo
+h_c
+ể_
+o_v
+uậ
+a_m
+ế_
+iá
+̀_c
+cho
+qua
+hạ
+ục
+_mi
+̀_n
+phâ
+c_q
+côn
+o_c
+á_
+i_h
+ại
+_hơ
+̃_h
+_cư
+n_l
+bị
+_lu
+bấ
+cả
+ín
+h_đ
+_xa
+độ
+g_h
+c_n
+c_p
+thu
+ải
+ệ_
+_hư
+́_c
+o_n
+_nư
+ốn
+́o_
+áo
+xã
+oà
+y_t
+hả
+tộ
+̣_c
+_tâ
+thô
+_du
+m_v
+mì
+ho_
+hứ
+ệc
+́_t
+hợ
+án
+n_p
+cũ
+ũn
+iể
+ối
+tiê
+ề_
+hấ
+ợp
+hoa
+y_đ
+chi
+o_h
+ở_
+ày
+̉_t
+đó
+c_l
+về
+̀_đ
+i_b
+kha
+c_b
+_đâ
+luâ
+ai_
+̉_n
+đố
+ết
+hự
+tri
+p_q
+nươ
+dụ
+hí
+g_q
+yên
+họ
+́nh
+_ta
+_bă
+c_g
+n_g
+thê
+o_t
+c_v
+am_
+c_m
+an_

data/lib/psychowl/active_model.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require 'active_model'
+require 'psychowl'
+
+I18n.load_path << File.expand_path('locale/en.yml', __dir__)
+
+module ActiveModel
+  module Validations
+    # Validates the language an attribute is written in.
+    #
+    #   validates :body, language: { in: %i[en fr] }
+    #   validates :title, language: { not_in: :ru, reliable: true }, allow_blank: true
+    #   validates :bio, language: { in: :en, minimum_confidence: 0.5 }
+    #
+    # Options:
+    # * +in+ / +not_in+ - languages (codes or Psychowl::Lang) to accept / reject
+    # * +reliable+ - require a reliable detection
+    # * +minimum_confidence+ - require at least this confidence (0.0 to 1.0)
+    #
+    # Errors: +:language_undetected+, +:language_not_allowed+,
+    # +:language_uncertain+, each with a +%{language}+ interpolation.
+    class LanguageValidator < EachValidator
+      def check_validity!
+        raise ArgumentError, 'pass either :in or :not_in, not both' if options[:in] && options[:not_in]
+
+        @allowed = resolve(options[:in])
+        @denied = resolve(options[:not_in])
+        minimum = options[:minimum_confidence]
+        return if minimum.nil? || (minimum.is_a?(Numeric) && minimum.between?(0, 1))
+
+        raise ArgumentError, ':minimum_confidence must be between 0 and 1'
+      end
+
+      def validate_each(record, attribute, value)
+        info = Psychowl.detect(value.to_s)
+        return add_error(record, attribute, :language_undetected, nil) if info.nil?
+
+        lang = info.lang
+        return add_error(record, attribute, :language_not_allowed, lang) if @allowed && !@allowed.include?(lang)
+        return add_error(record, attribute, :language_not_allowed, lang) if @denied&.include?(lang)
+
+        add_error(record, attribute, :language_uncertain, lang) if uncertain?(info)
+      end
+
+      private
+
+      def uncertain?(info)
+        return true if options[:reliable] && !info.reliable?
+
+        minimum = options[:minimum_confidence]
+        !minimum.nil? && info.confidence < minimum
+      end
+
+      def resolve(list) = list && Array(list).map { Psychowl::Lang.fetch(it) }
+
+      def add_error(record, attribute, type, lang)
+        details = { language: lang&.eng_name }
+        details[:message] = options[:message] if options[:message]
+        record.errors.add(attribute, type, **details)
+      end
+    end
+  end
+end

data/lib/psychowl/data_file.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Psychowl
+  # Reads the plain-text data files. No load-time side effects, so the data
+  # tools can use it before the tables exist.
+  module DataFile
+    # Lines of a file, without blank lines and # comments.
+    def self.lines(path)
+      File.readlines(path, chomp: true, encoding: Encoding::UTF_8).reject do |line|
+        line.empty? || line.start_with?('#')
+      end
+    end
+  end
+end

data/lib/psychowl/detector.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Psychowl
+  # A reusable, frozen detector, optionally limited by an allowlist or denylist.
+  #
+  #   detector = Psychowl::Detector.new(allowlist: %i[en fr de])
+  #   detector.detect_lang("Traversez la rue, je vous trouverai un travail.")
+  #   # => #<Psychowl::Lang fra (French)>
+  class Detector
+    # @return [Array<Lang>, nil]
+    attr_reader :allowlist
+
+    # @return [Array<Lang>, nil]
+    attr_reader :denylist
+
+    # @param allowlist [Array<Lang, String, Symbol>, nil] only consider these
+    # @param denylist [Array<Lang, String, Symbol>, nil] never consider these
+    # @raise [ArgumentError] both lists given, an empty allowlist, or an
+    #   unsupported language code
+    def initialize(allowlist: nil, denylist: nil)
+      raise ArgumentError, 'pass either allowlist: or denylist:, not both' unless allowlist.nil? || denylist.nil?
+
+      @allowlist = allowlist && resolve(allowlist)
+      @denylist = denylist && resolve(denylist)
+      raise ArgumentError, 'allowlist must not be empty' if @allowlist && @allowlist.empty?
+
+      @filter_mode, @filter_langs = engine_filter
+      freeze
+    end
+
+    # @param text [String]
+    # @return [Info, nil] nil when no supported language was found
+    # @raise [TypeError] text is not a String
+    # @raise [EncodingError] text is not valid UTF-8 (or convertible to it)
+    def detect(text)
+      result = Engine.detect(Text.prepare(text), @filter_mode, @filter_langs)
+      result && Info.from_engine(result)
+    end
+
+    # @param (see #detect)
+    # @return [Lang, nil]
+    def detect_lang(text) = detect(text)&.lang
+
+    # Scripts ignore the language filter.
+    #
+    # @param (see #detect)
+    # @return [Script, nil]
+    def detect_script(text)
+      index = Engine.detect_script(Text.prepare(text))
+      index && Script.at(index)
+    end
+
+    # Every candidate language of the text's script with its score, best
+    # first. Scores are relative to each other, not probabilities.
+    #
+    # @param (see #detect)
+    # @param limit [Integer, nil] keep only the first +limit+ candidates
+    # @return [Array<Array(Lang, Float)>]
+    def candidates(text, limit: nil)
+      ranked = Engine.candidates(Text.prepare(text), @filter_mode, @filter_langs)
+      ranked = ranked.first(limit) if limit
+      ranked.map { |lang, score| [Lang.at(lang), score] }
+    end
+
+    # On all CPU cores with the native engine.
+    #
+    # @param texts [Array<String>]
+    # @return [Array<Info, nil>] one result per text, in order
+    def detect_many(texts)
+      prepared = texts.map { Text.prepare(it) }
+      Engine.detect_many(prepared, @filter_mode, @filter_langs).map { |result| result && Info.from_engine(result) }
+    end
+
+    # Runs of same-language sentences, for mixed-language text.
+    #
+    # @param (see #detect)
+    # @return [Array<Segment>]
+    def segments(text)
+      text = Text.prepare(text)
+      Engine.segments(text, @filter_mode, @filter_langs).map do |start, finish, *result|
+        Segment.new(text: text[start...finish], range: start...finish, info: Info.from_engine(result))
+      end
+    end
+
+    # Share of each writing system among the text's letters, largest first.
+    # Ignores the language filter.
+    #
+    # @param (see #detect)
+    # @return [Hash{Script => Float}] shares summing to 1.0; empty without letters
+    def scripts(text)
+      counts = Engine.script_counts(Text.prepare(text))
+      total = counts.sum
+      return {} if total.zero?
+
+      shares = counts.each_with_index.filter_map do |count, index|
+        [Script.at(index), count.fdiv(total)] if count.positive?
+      end
+      shares.sort_by { |_script, share| -share }.to_h
+    end
+
+    def inspect
+      filter =
+        if allowlist then " allowlist=#{allowlist.map(&:code)}"
+        elsif denylist then " denylist=#{denylist.map(&:code)}"
+        end
+      "#<#{self.class}#{filter}>"
+    end
+
+    private
+
+    # [mode, language indices] as the engines take them
+    def engine_filter
+      if allowlist then [Engine::FILTER_ALLOW, indices(allowlist)]
+      elsif denylist then [Engine::FILTER_DENY, indices(denylist)]
+      else [Engine::FILTER_ALL, [].freeze]
+      end
+    end
+
+    def indices(langs) = langs.map { Lang.index(it) }.freeze
+
+    def resolve(list) = Array(list).map { Lang.fetch(it) }.uniq.freeze
+  end
+end

data/lib/psychowl/engine.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Psychowl
+  # Delegates to RubyEngine until native_speedup.rb prepends the Rust
+  # extension. Same primitives either way.
+  module Engine
+    FILTER_ALL = 0
+    FILTER_ALLOW = 1
+    FILTER_DENY = 2
+
+    class << self
+      def native? = false
+
+      def detect(text, filter_mode, filter_langs) = RubyEngine.detect(text, filter_mode, filter_langs)
+      def candidates(text, filter_mode, filter_langs) = RubyEngine.candidates(text, filter_mode, filter_langs)
+      def detect_many(texts, filter_mode, filter_langs) = RubyEngine.detect_many(texts, filter_mode, filter_langs)
+      def segments(text, filter_mode, filter_langs) = RubyEngine.segments(text, filter_mode, filter_langs)
+      def sentences(text) = RubyEngine.sentences(text)
+      def detect_script(text) = RubyEngine.detect_script(text)
+      def script_counts(text) = RubyEngine.script_counts(text)
+    end
+  end
+end

data/lib/psychowl/info.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Psychowl
+  # Outcome of a detection. Supports pattern matching:
+  #
+  #   case Psychowl.detect(speech)
+  #   in {lang: {code: "eng"}, reliable: true} then :tremendous
+  #   in {reliable: false} then :fake_news
+  #   end
+  #
+  # @!attribute [r] lang
+  #   @return [Lang]
+  # @!attribute [r] script
+  #   @return [Script]
+  # @!attribute [r] confidence
+  #   @return [Float] 0.0 to 1.0
+  # @!attribute [r] reliable
+  #   @return [Boolean] confidence above {RELIABLE_CONFIDENCE}
+  Info = ::Data.define(:lang, :script, :confidence, :reliable) do
+    # @return [Boolean]
+    def reliable? = reliable
+
+    def inspect
+      "#<#{self.class} lang=#{lang.code} script=#{script.name} " \
+        "confidence=#{confidence.round(4)} reliable=#{reliable}>"
+    end
+  end
+
+  # Reliability threshold and construction from engine results.
+  class Info
+    RELIABLE_CONFIDENCE = 0.9
+
+    # @api private
+    def self.from_engine((lang, script, confidence))
+      new(lang: Lang.at(lang), script: Script.at(script), confidence:,
+          reliable: confidence > RELIABLE_CONFIDENCE)
+    end
+  end
+end

data/lib/psychowl/lang.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Psychowl
+  # A supported language. Instances are canonical: every API hands out the
+  # same frozen object for a given language.
+  #
+  #   Psychowl::Lang[:de]      # ISO 639-1
+  #   Psychowl::Lang["deu"]    # ISO 639-3
+  #   Psychowl::Lang.fetch(:x) # raises ArgumentError
+  #
+  # @!attribute [r] code
+  #   @return [String] ISO 639-3 code, e.g. "deu"
+  # @!attribute [r] iso639_1
+  #   @return [String] ISO 639-1 code, e.g. "de"
+  # @!attribute [r] name
+  #   @return [String] name in the language itself, e.g. "Deutsch"
+  # @!attribute [r] eng_name
+  #   @return [String] English name, e.g. "German"
+  Lang = ::Data.define(:code, :iso639_1, :name, :eng_name) do
+    # @return [Symbol] the ISO 639-1 code as an I18n-style locale, e.g. :de
+    def locale = iso639_1.to_sym
+
+    # @return [String] the ISO 639-3 code
+    def to_s = code
+
+    def inspect = "#<#{self.class} #{code} (#{eng_name})>"
+  end
+
+  # The canonical instances and lookup by code.
+  class Lang
+    singleton_class.prepend(Lookup) # ahead of Data.define's own `[]` constructor
+
+    ALL = Tables::LANGUAGES.map do |code, iso639_1, eng_name, name|
+      new(code:, iso639_1:, name:, eng_name:)
+    end.freeze
+    BY_CODE = ALL.flat_map { |lang| [[lang.code, lang], [lang.iso639_1, lang]] }.to_h.freeze
+    INDEX = ALL.each_with_index.to_h.compare_by_identity.freeze
+    private_constant :ALL, :BY_CODE, :INDEX
+
+    class << self
+      # @return [Array<Lang>] every supported language (frozen)
+      def all = ALL
+
+      # @api private
+      def index(lang) = INDEX.fetch(lang)
+
+      private :new
+
+      private
+
+      def by_key = BY_CODE
+      def unknown_key_message = 'unsupported language'
+    end
+  end
+end

data/lib/psychowl/locale/en.yml

@@ -0,0 +1,6 @@
+en:
+  errors:
+    messages:
+      language_undetected: "is not written in a recognizable language"
+      language_not_allowed: "is written in %{language}, which is not allowed"
+      language_uncertain: "could not be identified reliably (looks like %{language})"

data/lib/psychowl/lookup.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Psychowl
+  # Case-insensitive lookup for Lang and Script. Prepended, because Data
+  # defines its own `[]`; expects `all`, `by_key` and `unknown_key_message`.
+  module Lookup
+    # @param key [self, String, Symbol]
+    # @return [self, nil]
+    def [](key)
+      return key if key.is_a?(self)
+
+      by_key[key.to_s.downcase]
+    end
+
+    # @param (see #[])
+    # @return [self]
+    # @raise [ArgumentError] unknown key
+    def fetch(key)
+      self[key] or raise ArgumentError, "#{unknown_key_message}: #{key.inspect}"
+    end
+
+    # @api private
+    def at(index) = all.fetch(index)
+  end
+end

data/lib/psychowl/native_speedup.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# A non-empty DISABLE_PSYCHOWL_NATIVE or DISABLE_MATRYOSHKA_NATIVE forces pure
+# Ruby; an empty one (as CI matrices set) does not.
+return if %w[DISABLE_PSYCHOWL_NATIVE DISABLE_MATRYOSHKA_NATIVE].any? { !ENV.fetch(it, '').empty? }
+
+# Platform gems ship the binary under the Ruby ABI version; a dev build (rake
+# compile) leaves it next to this file. The pure Ruby gem has neither.
+shipped = File.join(__dir__, RUBY_VERSION[/\A\d+\.\d+/], "psychowl_native.#{RbConfig::CONFIG['DLEXT']}")
+begin
+  if File.exist?(shipped)
+    require shipped
+  else
+    require 'psychowl/psychowl_native'
+  end
+rescue LoadError => e
+  # A shipped binary that will not load (glibc older than 2.34, Alpine without
+  # libgcc) is a real fault, not the pure Ruby gem: say so.
+  warn "psychowl: native extension failed to load (#{e.message}); using the pure Ruby engine" if File.exist?(shipped)
+  return
+end
+
+# An extension built from other data files would map indices to the wrong
+# languages: keep the Ruby engine instead.
+expected = [Psychowl::Tables::LANGUAGES.map(&:first), Psychowl::Tables::SCRIPTS.map(&:name)]
+unless PsychowlNative.tables == expected
+  warn "psychowl: native extension was built from different language data (rebuild it); using the pure Ruby engine"
+  return
+end
+
+# Swaps the Rust extension in behind Engine.
+module Psychowl
+  # Routes every Engine method to the Rust extension (PsychowlNative).
+  module NativeSpeedup
+    def native? = true
+    def detect(text, filter_mode, filter_langs) = PsychowlNative.detect(text, filter_mode, filter_langs)
+    def candidates(text, filter_mode, filter_langs) = PsychowlNative.candidates(text, filter_mode, filter_langs)
+    def detect_script(text) = PsychowlNative.detect_script(text)
+    def script_counts(text) = PsychowlNative.script_counts(text)
+    def detect_many(texts, filter_mode, filter_langs) = PsychowlNative.detect_many(texts, filter_mode, filter_langs)
+    def segments(text, filter_mode, filter_langs) = PsychowlNative.segments(text, filter_mode, filter_langs)
+    def sentences(text) = PsychowlNative.sentences(text)
+  end
+
+  Engine.singleton_class.prepend(NativeSpeedup)
+end