usernames 0.1.3

data/lib/usernames/configuration.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+module Usernames
+  # `Usernames.configure { |config| … }`. The generated initializer shows ten
+  # of these options; the README documents the rest.
+  #
+  # Word lists START FROM the bundled data, so `+=` adds to it and `=`
+  # replaces it:
+  #
+  #   config.reserved += %w[recipe recipes]   # the bundled words, and these
+  #   config.reserved  = %w[recipe recipes]   # only these
+  #
+  # Lists are handed out frozen. The rules compiled from them are cached, and
+  # a list changed behind the cache's back (`config.reserved << "x"`) would
+  # be ignored without a word — so that raises FrozenError instead, and the
+  # two spellings above are the ones that work.
+  #
+  # Nothing is checked as you assign: validate! (run at the end of configure,
+  # and by the Rails engine after boot) checks every value at once and raises
+  # Usernames::ConfigurationError naming the bad one.
+  class Configuration
+    # Accessors whose writers invalidate the compiled rules.
+    module Settings
+      def option(*names)
+        names.each do |name|
+          attr_reader name
+
+          define_method(:"#{name}=") do |value|
+            changed!
+            instance_variable_set(:"@#{name}", value.is_a?(Array) ? value.dup.freeze : value)
+          end
+        end
+      end
+    end
+
+    extend Settings
+
+    # config.offensive.severity / .languages / .allowed / .adapters
+    class Offensive
+      extend Settings
+
+      option :severity, :languages, :allowed, :adapters
+
+      def initialize(configuration)
+        @configuration = configuration
+        @severity = :strong
+        @languages = %i[en es].freeze
+        # Your own exemptions, so — unlike reserved and forbidden — this list
+        # starts empty: the bundled allowlists are per language and apply only
+        # inside their own language, so they are not merged in here.
+        @allowed = [].freeze
+        # A no-op unless the moderate gem is loaded.
+        @adapters = [:moderate].freeze
+      end
+
+      def to_h
+        { severity: severity, languages: languages, allowed: allowed, adapters: adapters }
+      end
+
+      private
+
+      def changed!
+        @configuration.changed!
+      end
+    end
+
+    # config.generator.adjectives / .nouns, or config.generator = ->(attempt) { … }
+    class Generator
+      extend Settings
+
+      option :adjectives, :nouns, :callable
+
+      def initialize(configuration)
+        @configuration = configuration
+        @adjectives = Data.adjectives
+        @nouns = Data.nouns
+        @callable = nil
+      end
+
+      def to_h
+        { adjectives: adjectives, nouns: nouns, callable: callable }
+      end
+
+      private
+
+      def changed!
+        @configuration.changed!
+      end
+    end
+
+    # Per-call word lists ADD to the configured ones: a model that passes
+    # `reserved: %w[recipe]` keeps every bundled reservation. Every other
+    # per-call option replaces the configured value. Adding is the choice
+    # that can never silently un-reserve a name: a per-model list fails closed.
+    LISTS = %i[reserved forbidden allowed].freeze
+
+    # Distinct per-call option sets kept compiled. They come from model and
+    # validator declarations, so a handful in practice; the limit only stops
+    # a caller passing fresh options on every request from growing it forever.
+    CACHE_LIMIT = 64
+
+    option :length, :separators, :edge_separators, :case, :first_character, :reserve_routes, :app_name,
+           :reserved, :forbidden, :allowed, :on_adapter_failure, :unique_index
+
+    attr_reader :offensive, :generator
+
+    def initialize
+      @length = 3..30
+      @separators = %w[_].freeze
+      @edge_separators = false
+      @case = :lowercase
+      @first_character = :alphanumeric
+      @reserve_routes = true
+      @app_name = nil # detected from the Rails application; false opts out
+      @reserved = Data.reserved
+      @forbidden = Data.impersonation
+      @allowed = [].freeze
+      @on_adapter_failure = :allow
+      @unique_index = nil
+      @offensive = Offensive.new(self)
+      @generator = Generator.new(self)
+    end
+
+    # `config.generator = ->(attempt) { … }` replaces the default shape;
+    # the word lists stay at config.generator.adjectives / .nouns.
+    def generator=(callable)
+      generator.callable = callable
+    end
+
+    # The compiled, frozen Usernames::Rules for these per-call options merged
+    # over this configuration, raising ConfigurationError for a bad value.
+    def rules_for(**overrides)
+      cache = (@rules_cache ||= {})
+      cache.clear if cache.size >= CACHE_LIMIT
+      cache[overrides] ||= begin
+        options = merge(overrides)
+        Rules.new(**options)
+      end
+    end
+
+    def validate!
+      rules_for
+      self
+    end
+
+    def to_h
+      {
+        length: length, separators: separators, edge_separators: edge_separators, case: @case,
+        first_character: first_character, reserve_routes: reserve_routes, app_name: app_name,
+        reserved: reserved, forbidden: forbidden, allowed: allowed, offensive: offensive.to_h,
+        on_adapter_failure: on_adapter_failure, generator: generator.to_h, unique_index: unique_index
+      }
+    end
+
+    # Called by every writer, including those of config.offensive and
+    # config.generator. :nodoc:
+    def changed!
+      @rules_cache = nil
+    end
+
+    private
+
+    def merge(overrides)
+      options = to_h
+      overrides.each do |name, value|
+        options[name] =
+          case name
+          when *LISTS then added(options[name], value)
+          when :offensive then merged_offensive(options[:offensive], value)
+          when :generator then merged_generator(options[:generator], value)
+          else value
+          end
+      end
+      options
+    end
+
+    # Anything that is not a list is passed on as given, for Rules to refuse by name.
+    def added(list, value)
+      value.is_a?(Array) ? list + value : value
+    end
+
+    def merged_offensive(settings, value)
+      return value unless value.is_a?(Hash)
+
+      settings.merge(value) do |name, configured, given|
+        name == :allowed ? added(configured, given) : given
+      end
+    end
+
+    def merged_generator(settings, value)
+      return settings.merge(callable: value) if value.nil? || value.respond_to?(:call)
+      return value unless value.is_a?(Hash)
+
+      settings.merge(value)
+    end
+  end
+end

data/lib/usernames/data.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+require "json"
+require "yaml"
+
+module Usernames
+  # The word lists that ship in data/, read once per process and frozen.
+  #
+  # Every list is a plain text file, one entry per line; blank lines and
+  # `#` comments (which carry each block's source and reason, see
+  # data/NOTICE) are skipped. Every file is mandatory: one that is missing,
+  # empty or malformed raises Usernames::DataError when the gem loads, since
+  # the alternative — rules that silently check nothing — fails open.
+  #
+  # Note the name: inside `module Usernames`, `Data` is this module. Ruby's
+  # own value-object class is `::Data`, and the gem always spells it so.
+  module Data
+    ROOT = File.expand_path("../../data", __dir__)
+
+    # The lists every offensive entry carries, besides its term and severity.
+    TERM_LISTS = %w[variants tags exceptions].freeze
+
+    # Matched exactly: system words, mailboxes, web paths.
+    EXACT_FILES = %w[reserved/system.txt reserved/mailboxes.txt reserved/web.txt].freeze
+
+    # Matched the way impersonators type them.
+    IMPERSONATION_FILES = %w[reserved/impersonation.txt].freeze
+
+    # `# source: gitlab (MIT) — GitLab top-level and project wildcard routes`
+    SOURCE_LINE = /\A#\s*source:\s*(?<name>\S+)\s+\((?<licence>[^)]+)\)\s*(?:—\s*(?<reason>.*))?\z/
+
+    # Where one bundled reserved word comes from: its file (:impersonation,
+    # :system, :mailboxes, :web) and the `# source:` comment of its block.
+    Provenance = ::Data.define(:category, :source, :licence, :reason)
+
+    class << self
+      # What `config.reserved` starts from.
+      def reserved
+        @reserved ||= EXACT_FILES.flat_map { |file| list(file) }.uniq.freeze
+      end
+
+      # What `config.forbidden` starts from.
+      def impersonation
+        @impersonation ||= IMPERSONATION_FILES.flat_map { |file| list(file) }.uniq.freeze
+      end
+
+      # { word => Provenance } over every bundled reserved file. The
+      # impersonation file is read first: a word in it and in a mailbox list
+      # (`postmaster`) is matched the stricter way, so that is its source.
+      def reserved_provenance
+        @reserved_provenance ||= (IMPERSONATION_FILES + EXACT_FILES).each_with_object({}) do |file, words|
+          category = File.basename(file, ".txt").to_sym
+          sourced(file).each { |word, source| words[word] ||= Provenance.new(category: category, **source) }
+        end.freeze
+      end
+
+      # The generator's vocabulary.
+      def adjectives
+        @adjectives ||= list("words/adjectives.txt")
+      end
+
+      # data/words/animals.txt, published as Usernames::Words::ANIMALS.
+      def nouns
+        @nouns ||= list("words/animals.txt")
+      end
+
+      # The languages whose offensive words ship (data/offensive/<locale>.json),
+      # as symbols: what config.offensive.languages may name.
+      def available_locales
+        @available_locales ||= begin
+          locales = Dir[File.join(ROOT, "offensive", "*.json")].map { |path| File.basename(path, ".json").to_sym }
+          broken!(File.join(ROOT, "offensive"), "holds no <locale>.json word list") if locales.empty?
+          locales.sort.freeze
+        end
+      end
+
+      # The parsed entries of data/offensive/<locale>.json: each a Hash with
+      # a "term", a "severity" from 1 to 4, and lists of "variants", "tags"
+      # and "exceptions". An empty list is as broken as a missing file: it
+      # would screen nothing in that language.
+      def offensive(locale)
+        (@offensive ||= {})[locale.to_sym] ||= read("offensive/#{locale}.json") do |path|
+          entries = JSON.parse(File.read(path, encoding: "UTF-8"))
+          broken!(path, "has no entries") if entries == []
+          unless entries.is_a?(Array) && entries.all? { |entry| term?(entry) }
+            broken!(path, "is not a list of {term, severity 1–4, variants, tags, exceptions} entries")
+          end
+
+          deep_freeze(entries)
+        end
+      end
+
+      # data/offensive/allowlist/<locale>.txt (each line `word # source`).
+      def allowlist(locale)
+        (@allowlist ||= {})[locale.to_sym] ||= list("offensive/allowlist/#{locale}.txt")
+      end
+
+      # data/offensive/hate.yml: the hate tokens and number rules.
+      def hate
+        @hate ||= yaml("offensive/hate.yml", "always" => Array, "digit_runs" => Array, "adjacent" => Hash,
+                                             "whole_token" => Hash)
+      end
+
+      # data/offensive/leet.yml: the digit folds both matchers use, and the
+      # rewrites only the offensive matcher uses.
+      def leet
+        @leet ||= yaml("offensive/leet.yml", "digits" => Hash, "offensive_only" => Hash)
+      end
+
+      # The entries of one data file, relative to data/ (or an absolute path).
+      # Every bundled file is read as UTF-8 explicitly: the host's locale
+      # (Encoding.default_external, US-ASCII where LANG is unset) must never
+      # decide how the lists are decoded.
+      def list(path)
+        read(path) do |file|
+          entries = File.foreach(file, chomp: true, encoding: "UTF-8").filter_map do |line|
+            entry = line.sub(/#.*/, "").strip
+            entry.freeze unless entry.empty?
+          end
+          broken!(file, "has no entries") if entries.empty?
+          entries.freeze
+        end
+      end
+
+      # [[entry, { source:, licence:, reason: }], …] for one data file: each
+      # entry with the `# source:` comment of the block it sits in (nils for
+      # an entry before any).
+      def sourced(path)
+        read(path) do |file|
+          source = { source: nil, licence: nil, reason: nil }
+          File.foreach(file, chomp: true, encoding: "UTF-8").filter_map do |line|
+            if (match = SOURCE_LINE.match(line.strip))
+              source = { source: match[:name], licence: match[:licence], reason: match[:reason] }
+              next
+            end
+            entry = line.sub(/#.*/, "").strip
+            [entry, source] unless entry.empty?
+          end
+        end
+      end
+
+      # Reads every bundled file once, so a broken install fails when the gem
+      # loads rather than on the first sign-up that needs the missing list.
+      def load!
+        reserved
+        impersonation
+        reserved_provenance
+        adjectives
+        nouns
+        available_locales.each do |locale|
+          offensive(locale)
+          allowlist(locale)
+        end
+        hate
+        leet
+        self
+      end
+
+      private
+
+      def read(relative)
+        path = File.expand_path(relative, ROOT)
+        broken!(path, "is missing") unless File.file?(path)
+
+        yield path
+      rescue JSON::ParserError, Psych::Exception => e
+        broken!(path, "can't be parsed (#{e.message.lines.first.strip})")
+      end
+
+      # A YAML mapping with at least `shape`'s keys, each of its type.
+      def yaml(relative, shape)
+        read(relative) do |path|
+          data = YAML.safe_load_file(path)
+          wrong = shape.find { |key, type| !(data.is_a?(Hash) && data[key].is_a?(type)) }
+          broken!(path, "needs #{wrong[0].inspect} (a #{wrong[1].name})") if wrong
+
+          deep_freeze(data)
+        end
+      end
+
+      def term?(entry)
+        entry.is_a?(Hash) && entry["term"].is_a?(String) && !entry["term"].empty? &&
+          entry["severity"].is_a?(Integer) && (1..4).cover?(entry["severity"]) &&
+          term_lists?(entry) && [nil, true, false].include?(entry["segment_only"])
+      end
+
+      def term_lists?(entry)
+        TERM_LISTS.all? { |key| entry[key].is_a?(Array) && entry[key].all?(String) }
+      end
+
+      def broken!(path, problem)
+        raise DataError, "usernames' bundled data is broken: #{path} #{problem}. " \
+                         "Reinstall the gem, or rebuild data/ with script/build_*.rb in a checkout."
+      end
+
+      def deep_freeze(value)
+        case value
+        when Hash then value.each_value { |item| deep_freeze(item) }
+        when Array then value.each { |item| deep_freeze(item) }
+        end
+        value.freeze
+      end
+    end
+  end
+end

data/lib/usernames/engine.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Usernames
+  # Not isolated, and no routes: the gem mounts nothing behind your back
+  # (the availability check is a generator that copies files into the app).
+  #
+  # What it does do: its config/locales load BEFORE the app's — Rails puts
+  # engines first and the application last — so any key in the app's own
+  # locale files wins; `has_username` reaches every Active Record model; the
+  # app's own routes are reserved and re-read whenever they reload; and a bad
+  # configuration value fails at boot, not at the first sign-up.
+  #
+  # The generators (`usernames:install`, `usernames:availability`,
+  # `usernames:locale`) need no wiring: Rails finds them in
+  # lib/generators/usernames/ on the load path.
+  class Engine < ::Rails::Engine
+    initializer "usernames.active_record" do
+      ActiveSupport.on_load(:active_record) { extend Usernames::Model }
+    end
+
+    # After every initializer, including the app's config/initializers/usernames.rb.
+    initializer "usernames.validate_configuration" do |app|
+      app.config.after_initialize { Usernames.configuration.validate! }
+    end
+
+    # The app's static top-level paths are reserved. They are
+    # collected lazily, on the first check after the routes load; in
+    # development the routes reload when config/routes.rb changes, and a
+    # page added there is reserved from the next check on.
+    config.after_routes_loaded { Usernames::Reserved.reset_routes! }
+  end
+end

data/lib/usernames/errors.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Usernames
+  class Error < StandardError; end
+
+  # A configuration value the gem cannot use, named in the message.
+  class ConfigurationError < Error; end
+
+  # Every generated candidate was taken (generate_username draws one batch).
+  class NoFreeNameError < Error; end
+
+  # A bundled data file (data/**) is missing or not what the gem expects.
+  # Raised when the gem loads: without its word lists the gem would check
+  # nothing at all — `admin` registrable, every slur allowed — and say nothing.
+  class DataError < Error; end
+end

data/lib/usernames/generator.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "active_support/inflector"
+
+module Usernames
+  # Default and seeded names: `quiet_otter_42`, or `jose_garcia`
+  # from "José García".
+  #
+  # Pure: it draws a batch of candidates and keeps those the rules allow; it
+  # never asks a database whether one is free. The model helpers take the
+  # batch to the database in ONE query, which is why it is a batch: twenty
+  # names at once instead of a retry loop of queries.
+  #
+  # The order inside each draw is fixed: assemble the name, shorten it
+  # to fit, and only then run the full check. Shortening can make a name
+  # the check refuses (an all-digit stem, a word cut into something else), and
+  # a check run before it would have passed a name that is not the one
+  # handed out. Digits always come from SecureRandom, never a counter, so
+  # the next name is not guessable from the last.
+  module Generator
+    # Draws per batch. Every shape, fallback and retry fits inside it.
+    DRAWS = 20
+
+    # Local parts that say nothing about the person: from
+    # info@acme.com the domain label is the better seed.
+    GENERIC_LOCAL_PARTS = %w[i me info support admin webmaster hello mail office contact team].freeze
+
+    # Below this maximum length `adjective_noun_42` cannot fit, so the
+    # compact shapes take over (as they do with no separators at all).
+    COMPACT_BELOW = 10
+
+    # How long a random-letters fallback is, within the length rule.
+    RANDOM_LETTERS = 8
+
+    class << self
+      # Up to DRAWS distinct names the rules allow, in the order drawn.
+      #
+      #   Generator.candidates(rules: rules)                  # the default shapes
+      #   Generator.candidates(from: "José García", rules: rules)
+      #
+      # With a seed: the cleaned seed, the seed with two random digits (three
+      # times) and with three (twice), then default shapes for the rest of
+      # the batch.
+      # The default shapes after a seed count their own attempts
+      # from 1, so they start at two digits as an unseeded batch does.
+      def candidates(rules:, from: nil)
+        base = from.nil? ? "" : clean(from, rules)
+        draws = base.empty? ? [] : seeded(base, rules)
+        draws += Array.new(DRAWS - draws.size) { |index| shape(index + 1, rules) }
+        allowed(draws, rules)
+      end
+
+      # One draw of the default shape, or of config.generator when the app
+      # set one (called with the attempt number, 1 up). nil when the draw
+      # cannot fit the length rule; never checked here.
+      def shape(attempt, rules)
+        callable = rules.generator.callable
+        return callable.call(attempt).to_s if callable
+
+        compact?(rules) ? compact_shape(attempt, rules) : default_shape(attempt, rules)
+      end
+
+      # A seed as a name: the useful half of an email,
+      # normalised, transliterated to ASCII, with every run of characters the
+      # rules do not allow turned into the first separator, runs of
+      # separators collapsed and the edges trimmed. May be "".
+      def clean(seed, rules)
+        text = prepare(seed, rules)
+        separators = rules.separators
+        allowed = "#{rules.lowercase? ? 'a-z' : 'A-Za-z'}0-9#{Regexp.escape(separators.join)}"
+        text = text.gsub(/[^#{allowed}]+/, separators.first.to_s)
+        return text if separators.empty?
+
+        run = "[#{Regexp.escape(separators.join)}]"
+        text.gsub(/#{run}{2,}/) { |separators_run| separators_run[0] }.gsub(/\A#{run}+|#{run}+\z/, "")
+      end
+
+      # The words of a seed, whatever the separators: "Javi R" → javi, r.
+      def words(seed, rules)
+        prepare(seed, rules).split(/[^A-Za-z0-9]+/).reject(&:empty?)
+      end
+
+      # `base` cut to at most `budget` characters, at a separator that falls
+      # inside the budget (`pedro_gonzales_armillon` → `pedro_gonzales`).
+      # nil when not even the first word fits: a name is never cut mid-word
+      # (`maximilianus` is not `maximil`), so the batch moves on to its next
+      # candidate instead.
+      def fit(base, budget, rules)
+        return if budget < 1 || base.empty?
+        return base if base.length <= budget
+
+        boundary = rules.separators.empty? ? nil : base.rindex(Regexp.union(rules.separators), budget)
+        base[0, boundary] if boundary&.positive?
+      end
+
+      # `base` + separator + random digits from `range`, the base shortened
+      # first so the whole fits the maximum length.
+      def with_digits(base, range, rules)
+        separator = rules.separators.first.to_s
+        digits = SecureRandom.random_number(range).to_s
+        stem = fit(base, rules.length.max - separator.length - digits.length, rules)
+        "#{stem}#{separator}#{digits}" if stem
+      end
+
+      private
+
+      def seeded(base, rules)
+        [fit(base, rules.length.max, rules)] +
+          Array.new(3) { with_digits(base, 10..99, rules) } +
+          Array.new(2) { with_digits(base, 100..999, rules) }
+      end
+
+      # adjective + separator + noun + separator + digits, dropping the
+      # adjective when the pair does not fit the budget the digits leave.
+      def default_shape(attempt, rules)
+        separator = rules.separators.first
+        digits = suffix(attempt)
+        base = default_base(rules, rules.length.max - separator.length - digits.length)
+        "#{base}#{separator}#{digits}" if base
+      end
+
+      def default_base(rules, budget)
+        noun = pick(rules.generator.nouns)
+        return unless noun
+
+        pair = [pick(rules.generator.adjectives), noun].compact.join(rules.separators.first)
+        [pair, noun].find { |words| words.length <= budget }
+      end
+
+      # For a short maximum or no separators: noun + digits, then the noun
+      # cut to the budget (`ott42`), then random letters when not even one
+      # letter fits beside the digits (`length: 2..2`).
+      def compact_shape(attempt, rules)
+        digits = suffix(attempt)
+        noun = pick(rules.generator.nouns)
+        budget = rules.length.max - digits.length
+        name = "#{noun[0, budget]}#{digits}" if noun && budget >= 1
+        name && name.length >= rules.length.min ? name : random_letters(rules)
+      end
+
+      def compact?(rules)
+        rules.separators.empty? || rules.length.max < COMPACT_BELOW
+      end
+
+      # Two digits for the first three draws, three for the next two, four
+      # for the next three, then six hex characters: short names
+      # first, and more room as the obvious ones turn out taken.
+      def suffix(attempt)
+        case attempt
+        when 1..3 then SecureRandom.random_number(10..99).to_s
+        when 4..5 then SecureRandom.random_number(100..999).to_s
+        when 6..8 then SecureRandom.random_number(1000..9999).to_s
+        else SecureRandom.hex(3)
+        end
+      end
+
+      def random_letters(rules)
+        length = RANDOM_LETTERS.clamp(rules.length.min, rules.length.max)
+        Array.new(length) { ("a".ord + SecureRandom.random_number(26)).chr }.join
+      end
+
+      def pick(list)
+        list[SecureRandom.random_number(list.size)] if list.any?
+      end
+
+      # Normalised, deduplicated and checked — the full check, last.
+      def allowed(draws, rules)
+        draws.each_with_object([]) do |draw, names|
+          next if draw.nil?
+
+          name = rules.normalize(draw)
+          names << name unless name.empty? || names.include?(name) || rules.failure_for(name)
+        end
+      end
+
+      # Repaired into UTF-8 first, as the normaliser does: a UTF-16 or
+      # binary seed must not reach the String operations below.
+      def prepare(seed, rules)
+        text = Normalizer.utf8(seed)
+        text = email_seed(text) if text.include?("@")
+        text = transliterate(rules.normalize(text)).delete("?")
+        rules.lowercase? ? text.downcase : text
+      end
+
+      # The local part, without a +tag, unless it is generic or empty
+      # (`@javi` is a handle, not an address): then the domain's first label.
+      # The first label, so acme.co.uk gives `acme`; the +tag is dropped
+      # (`javi+news`).
+      def email_seed(text)
+        local, _, domain = text.rpartition("@")
+        local = local.split("+", 2).first.to_s.strip
+        return local unless local.empty? || GENERIC_LOCAL_PARTS.include?(local.downcase)
+
+        domain.strip.split(".").first.to_s
+      end
+
+      # AnyAscii (gem `any_ascii`) when the app has it: it knows Cyrillic,
+      # Greek, CJK… Otherwise Active Support's transliteration, which knows
+      # Latin accents and writes `?` for the rest — dropped by the caller.
+      def transliterate(text)
+        defined?(::AnyAscii) ? ::AnyAscii.transliterate(text) : ActiveSupport::Inflector.transliterate(text)
+      end
+    end
+  end
+end