usernames 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +39 -0
- data/LICENSE.txt +21 -0
- data/README.md +362 -0
- data/SECURITY.md +42 -0
- data/config/locales/usernames.en.yml +72 -0
- data/config/locales/usernames.es.yml +51 -0
- data/config/locales/usernames.pt.yml +51 -0
- data/data/LICENSES/allprofanity.txt +24 -0
- data/data/LICENSES/big-username-blocklist.txt +25 -0
- data/data/LICENSES/corpora.txt +128 -0
- data/data/LICENSES/cuss.txt +25 -0
- data/data/LICENSES/django-registration.txt +33 -0
- data/data/LICENSES/dsojevic-profanity-list.txt +24 -0
- data/data/LICENSES/forwardemail.txt +24 -0
- data/data/LICENSES/friendly-words.txt +24 -0
- data/data/LICENSES/gitea.txt +23 -0
- data/data/LICENSES/gitlab.txt +31 -0
- data/data/LICENSES/iana-well-known-uris.txt +130 -0
- data/data/LICENSES/imsky-wordlists.txt +24 -0
- data/data/LICENSES/mottie-github-reserved-names.txt +24 -0
- data/data/LICENSES/obscenity.txt +27 -0
- data/data/LICENSES/petname.txt +218 -0
- data/data/LICENSES/rfc2142.txt +9 -0
- data/data/LICENSES/shouldbee.txt +25 -0
- data/data/LICENSES/usernames.txt +30 -0
- data/data/NOTICE +36 -0
- data/data/offensive/README.md +90 -0
- data/data/offensive/allowlist/en.txt +727 -0
- data/data/offensive/allowlist/es.txt +39 -0
- data/data/offensive/en.json +5680 -0
- data/data/offensive/es.json +1014 -0
- data/data/offensive/hate.yml +51 -0
- data/data/offensive/leet.yml +27 -0
- data/data/reserved/ALLOWLIST_EXCEPTIONS.txt +6 -0
- data/data/reserved/impersonation.txt +69 -0
- data/data/reserved/mailboxes.txt +77 -0
- data/data/reserved/system.txt +124 -0
- data/data/reserved/web.txt +962 -0
- data/data/words/adjectives.txt +367 -0
- data/data/words/animals.txt +253 -0
- data/lib/generators/usernames/availability_generator.rb +67 -0
- data/lib/generators/usernames/existing_schema.rb +85 -0
- data/lib/generators/usernames/install_generator.rb +241 -0
- data/lib/generators/usernames/install_messages.rb +63 -0
- data/lib/generators/usernames/locale_generator.rb +56 -0
- data/lib/generators/usernames/templates/add_username.rb.erb +53 -0
- data/lib/generators/usernames/templates/availability/_field.html.erb.tt +25 -0
- data/lib/generators/usernames/templates/availability/availability_controller.rb.tt +107 -0
- data/lib/generators/usernames/templates/availability/username_check_controller.js.tt +115 -0
- data/lib/generators/usernames/templates/usernames.rb +34 -0
- data/lib/tasks/usernames.rake +38 -0
- data/lib/usernames/adapters/moderate.rb +37 -0
- data/lib/usernames/audit/output.rb +51 -0
- data/lib/usernames/audit/values.rb +34 -0
- data/lib/usernames/audit.rb +230 -0
- data/lib/usernames/candidates.rb +64 -0
- data/lib/usernames/comparison/index_contract.rb +241 -0
- data/lib/usernames/comparison/index_expression.rb +183 -0
- data/lib/usernames/comparison.rb +127 -0
- data/lib/usernames/configuration.rb +197 -0
- data/lib/usernames/data.rb +205 -0
- data/lib/usernames/engine.rb +32 -0
- data/lib/usernames/errors.rb +16 -0
- data/lib/usernames/generator.rb +206 -0
- data/lib/usernames/model.rb +397 -0
- data/lib/usernames/normalizer.rb +51 -0
- data/lib/usernames/offensive/hate.rb +124 -0
- data/lib/usernames/offensive/lexicon.rb +156 -0
- data/lib/usernames/offensive/name.rb +254 -0
- data/lib/usernames/offensive.rb +136 -0
- data/lib/usernames/phrase.rb +68 -0
- data/lib/usernames/reserved/listing.rb +79 -0
- data/lib/usernames/reserved.rb +219 -0
- data/lib/usernames/result.rb +52 -0
- data/lib/usernames/rules/options.rb +175 -0
- data/lib/usernames/rules.rb +257 -0
- data/lib/usernames/unique_violation.rb +176 -0
- data/lib/usernames/validator.rb +61 -0
- data/lib/usernames/version.rb +5 -0
- data/lib/usernames/words.rb +13 -0
- data/lib/usernames.rb +138 -0
- data/sig/usernames.rbs +75 -0
- metadata +217 -0
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Usernames
|
|
4
|
+
module Offensive
|
|
5
|
+
# One language's bundled data (data/offensive/<locale>.json and its
|
|
6
|
+
# allowlist), compiled once per process into what rules a and b need.
|
|
7
|
+
#
|
|
8
|
+
# Every spelling — terms, variants, exceptions, allowlist entries — is
|
|
9
|
+
# folded exactly as names are (Offensive.fold), so the data stores
|
|
10
|
+
# `fuck` once and `fvck`, `phuck` meet it on the name side. Exemptions
|
|
11
|
+
# are kept both as written and folded: a name's R0 segment `sussex` and
|
|
12
|
+
# its R1 segment `susseks` are the same word.
|
|
13
|
+
class Lexicon
|
|
14
|
+
SEVERE = 4
|
|
15
|
+
|
|
16
|
+
# Below this length a severe term must equal a segment: `gc`, `mj`
|
|
17
|
+
# inside a segment would refuse gcastro and mjordan.
|
|
18
|
+
SUBSTRING_MINIMUM = 4
|
|
19
|
+
|
|
20
|
+
# One spelling of a term, as folded. `whole` means it is matched only
|
|
21
|
+
# as an entire segment (a short term, or one marked `segment_only`);
|
|
22
|
+
# `digits`, that it has a digit in it (`2g1c`), so it can only ever sit
|
|
23
|
+
# inside a part, never inside a segment.
|
|
24
|
+
Spelling = ::Data.define(:text, :term, :whole, :digits)
|
|
25
|
+
|
|
26
|
+
# Spellings by their first SUBSTRING_MINIMUM characters, to find every
|
|
27
|
+
# one inside a string in a single pass over it. A union Regexp of the
|
|
28
|
+
# four hundred spellings did the same job twenty times slower.
|
|
29
|
+
class Index
|
|
30
|
+
def initialize(spellings)
|
|
31
|
+
@by_prefix = spellings.group_by { |spelling| spelling.text[0, SUBSTRING_MINIMUM] }.freeze
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Whether some spelling occurring in the Whole's text makes the block
|
|
35
|
+
# answer true; the block gets the spelling and where it starts.
|
|
36
|
+
def any?(whole)
|
|
37
|
+
text = whole.text
|
|
38
|
+
whole.grams.each_with_index.any? do |gram, position|
|
|
39
|
+
spellings = @by_prefix[gram]
|
|
40
|
+
spellings&.any? do |spelling|
|
|
41
|
+
text[position, spelling.text.length] == spelling.text && yield(spelling, position)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
class << self
|
|
48
|
+
def for(locale)
|
|
49
|
+
(@lexicons ||= {})[locale.to_sym] ||= new(locale)
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def initialize(locale)
|
|
54
|
+
@allowlist = both_forms(Data.allowlist(locale))
|
|
55
|
+
@exceptions = {}
|
|
56
|
+
@tiers = Hash.new { |tiers, severity| tiers[severity] = {} }
|
|
57
|
+
@equal = {}
|
|
58
|
+
severe = []
|
|
59
|
+
Data.offensive(locale).each { |entry| add(entry, severe) }
|
|
60
|
+
compile(severe)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Rule a for this language: a whole-segment spelling that IS a unit, or
|
|
64
|
+
# any severe spelling found in a whole (see #severe_at?).
|
|
65
|
+
def severe?(name, allowed)
|
|
66
|
+
name.units.any? { |unit| whole_severe?(unit, allowed) } ||
|
|
67
|
+
name.wholes.any? do |whole|
|
|
68
|
+
@index.any?(whole) { |spelling, position| severe_at?(whole, spelling, position, allowed) }
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Rule b: a strong (or, from `minimum` 1, any) spelling that IS one of
|
|
73
|
+
# the name's segments, or a run of the words its capitals split.
|
|
74
|
+
def strong?(name, minimum, allowed)
|
|
75
|
+
table = equal_from(minimum)
|
|
76
|
+
(name.units + name.joins).any? do |unit|
|
|
77
|
+
term = table[unit]
|
|
78
|
+
term && !exempt?(unit, term, allowed)
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
private
|
|
83
|
+
|
|
84
|
+
def add(entry, severe)
|
|
85
|
+
term = entry.fetch("term")
|
|
86
|
+
severity = entry.fetch("severity")
|
|
87
|
+
@exceptions[term] = both_forms(entry.fetch("exceptions", []))
|
|
88
|
+
[term, *entry.fetch("variants", [])].map { |spelling| Offensive.fold(spelling) }.uniq.each do |text|
|
|
89
|
+
@tiers[severity][text] ||= term
|
|
90
|
+
severe << spelling(text, term, entry) if severity >= SEVERE
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def spelling(text, term, entry)
|
|
95
|
+
whole = text.length < SUBSTRING_MINIMUM || entry["segment_only"] == true
|
|
96
|
+
Spelling.new(text: text, term: term, whole: whole, digits: text.match?(/\d/))
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Whole-segment spellings are a table looked up by unit; every severe
|
|
100
|
+
# spelling of four letters or more goes in the index a whole is scanned with.
|
|
101
|
+
def compile(severe)
|
|
102
|
+
@whole_severe = severe.select(&:whole).group_by(&:text).freeze
|
|
103
|
+
@index = Index.new(severe.select { |spelling| spelling.text.length >= SUBSTRING_MINIMUM })
|
|
104
|
+
@tiers.default_proc = nil
|
|
105
|
+
@tiers.each_value(&:freeze)
|
|
106
|
+
[@tiers, @exceptions].each(&:freeze)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# a-1 for the whole-segment spellings: the unit must be the spelling.
|
|
110
|
+
def whole_severe?(unit, allowed)
|
|
111
|
+
@whole_severe[unit]&.any? { |spelling| !exempt?(unit, spelling.term, allowed) }
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# One occurrence of a severe spelling in a whole.
|
|
115
|
+
#
|
|
116
|
+
# a-2 when it straddles a separator (`fu_ck`): refused, with no
|
|
117
|
+
# exemption — such an occurrence lies inside no exempt unit — and
|
|
118
|
+
# without a-1's whole-segment special cases: a separator in the middle
|
|
119
|
+
# of `rape` reads as evasion, so `gr_ape` is refused — a documented
|
|
120
|
+
# trade-off: a real name split exactly there is rarer than the evasion.
|
|
121
|
+
#
|
|
122
|
+
# a-1 otherwise: refused unless the unit it sits in — its segment (or
|
|
123
|
+
# the word of it, between case boundaries), or for a spelling with a
|
|
124
|
+
# digit its part — is exempt. Whole-segment
|
|
125
|
+
# spellings are #whole_severe?'s, not matched as substrings.
|
|
126
|
+
# Spellings under four letters are left out of a-2 as of
|
|
127
|
+
# a-1 (none ships today); across separators they would refuse innocent
|
|
128
|
+
# joins of short segments.
|
|
129
|
+
def severe_at?(whole, spelling, position, allowed)
|
|
130
|
+
return true if whole.straddles?(spelling.text, position)
|
|
131
|
+
return false if spelling.whole
|
|
132
|
+
|
|
133
|
+
unit = spelling.digits ? whole.part_at(position) : whole.segment_at(position, spelling.text.length)
|
|
134
|
+
!exempt?(unit, spelling.term, allowed)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def exempt?(unit, term, allowed)
|
|
138
|
+
@allowlist.include?(unit) || allowed.include?(unit) || @exceptions.fetch(term).include?(unit)
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# { folded spelling => term } over every tier from `minimum` up to
|
|
142
|
+
# strong. Severe spellings are rule a's, which already caught them.
|
|
143
|
+
def equal_from(minimum)
|
|
144
|
+
@equal[minimum] ||= (minimum...SEVERE).each_with_object({}) do |severity, table|
|
|
145
|
+
@tiers.fetch(severity, {}).each { |text, term| table[text] ||= term }
|
|
146
|
+
end.freeze
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def both_forms(words)
|
|
150
|
+
words.each_with_object(Set.new) do |word, set|
|
|
151
|
+
set << word << Offensive.fold(word)
|
|
152
|
+
end.freeze
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
end
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Usernames
|
|
4
|
+
module Offensive
|
|
5
|
+
# One name, read every way the offensive matcher reads it. Built once per check.
|
|
6
|
+
#
|
|
7
|
+
# forms R0 (the key) and R1 (the key folded)
|
|
8
|
+
# segments what sits between separators and digit runs, in either form
|
|
9
|
+
# digit_parts what sits between separators only, where a digit keeps it
|
|
10
|
+
# from being a segment — the unit a spelling that itself has a
|
|
11
|
+
# digit in it (`2g1c`) is read against; segments never hold a
|
|
12
|
+
# digit, so without these such a spelling could never match
|
|
13
|
+
# wholes each form with its separators removed (see Whole)
|
|
14
|
+
# joins runs of adjacent words a case boundary split (see below)
|
|
15
|
+
#
|
|
16
|
+
# A name stored with its typed case (`case: :preserve`) also splits where
|
|
17
|
+
# a lowercase letter meets a capital, and before the last capital of a
|
|
18
|
+
# run that a lowercase letter follows: `ShitheadBob` is the segments
|
|
19
|
+
# shitheadbob, shithead and bob, `SHITHEADBob` the same, and
|
|
20
|
+
# `ScunthorpeUnited` is scunthorpeunited, scunthorpe and united. The
|
|
21
|
+
# capital is the only word boundary such a name has, and the person
|
|
22
|
+
# typed it. The glued segment stays a segment, and the whole is
|
|
23
|
+
# unchanged.
|
|
24
|
+
#
|
|
25
|
+
# Capitals also cut inside a word as often as between words
|
|
26
|
+
# (`ShitHeadBob`), so the strong terms, matched by equality, are also
|
|
27
|
+
# compared with every run of adjacent words of one segment: shithead,
|
|
28
|
+
# headbob. A run spans at most JOIN_WORDS words. A word a capital does
|
|
29
|
+
# not end (`XxShitHeadxX` reads Xx, Shit, Headx, X) makes no join that
|
|
30
|
+
# is a word: a known limit.
|
|
31
|
+
#
|
|
32
|
+
# Segments come from BOTH forms: `adm1n` is the segments adm, n as typed
|
|
33
|
+
# and admin folded, and `shithead1` keeps shithead as typed where folded it
|
|
34
|
+
# reads shitheadi.
|
|
35
|
+
class Name
|
|
36
|
+
attr_reader :key, :forms, :wholes, :segments, :digit_parts, :joins
|
|
37
|
+
|
|
38
|
+
# Where a lowercase letter meets a capital (`tB`), and where a run of
|
|
39
|
+
# capitals gives its last one to the word it starts (`DBo`).
|
|
40
|
+
CASE_BOUNDARY = /(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])/
|
|
41
|
+
|
|
42
|
+
# The most words one join of case-split words spans.
|
|
43
|
+
JOIN_WORDS = 8
|
|
44
|
+
|
|
45
|
+
# `key` is the comparison key; `canonical`, the name as stored, which
|
|
46
|
+
# differs from it only in case.
|
|
47
|
+
def initialize(key, canonical = key)
|
|
48
|
+
@key = key
|
|
49
|
+
@forms = [key, Offensive.fold(key)].uniq
|
|
50
|
+
boundaries = case_boundaries(canonical)
|
|
51
|
+
@wholes = [
|
|
52
|
+
Whole.new(key, boundaries: boundaries),
|
|
53
|
+
Whole.new(Reserved.leet(key), folded: true, boundaries: boundaries)
|
|
54
|
+
].uniq do |whole|
|
|
55
|
+
[whole.text, whole.segments]
|
|
56
|
+
end
|
|
57
|
+
@segments = @wholes.flat_map(&:segments).uniq
|
|
58
|
+
@digit_parts = @wholes.flat_map(&:parts).grep(/\d/).uniq
|
|
59
|
+
@joins = (@wholes.flat_map(&:joins) - @segments).uniq
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# What a whole-segment rule compares against: every segment, and every
|
|
63
|
+
# part a digit keeps from being one.
|
|
64
|
+
def units
|
|
65
|
+
segments + digit_parts
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
private
|
|
69
|
+
|
|
70
|
+
# The positions of the key where `canonical` changes from a lowercase
|
|
71
|
+
# letter to a capital. Positions carry over: the key is `canonical`
|
|
72
|
+
# with its ASCII letters lowercased, and the digit fold maps one
|
|
73
|
+
# character to one.
|
|
74
|
+
def case_boundaries(canonical)
|
|
75
|
+
return [].freeze if canonical == key
|
|
76
|
+
|
|
77
|
+
positions = []
|
|
78
|
+
canonical.scan(CASE_BOUNDARY) { positions << Regexp.last_match.begin(0) }
|
|
79
|
+
positions.freeze
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# A form with its separators removed (`f.u.c.k` → `fuck`), remembering
|
|
84
|
+
# for every character which parts (between separators) and which segment
|
|
85
|
+
# (between separators and digit runs) it came from. One pass over this
|
|
86
|
+
# text finds every severe spelling for rules a-1 and a-2 at once: an
|
|
87
|
+
# occurrence that straddles two parts is a-2's evasion, and one inside a
|
|
88
|
+
# part lies inside exactly one segment (a spelling without digits cannot
|
|
89
|
+
# cover a digit), which is a-1's unit.
|
|
90
|
+
#
|
|
91
|
+
# The folded whole (`folded: true`, built from the digit-folded key) is
|
|
92
|
+
# rewritten twice: each part and segment on its own, and the joined text
|
|
93
|
+
# again once the separators are gone — so a rewrite split by a separator
|
|
94
|
+
# (`p_huck`, `f_vck`, `fuuu_uuck`) still folds. A character a rewrite
|
|
95
|
+
# makes from two parts (the `f` of `p_huck`) remembers both, so whatever
|
|
96
|
+
# covers it straddles them, and an exemption never applies there.
|
|
97
|
+
#
|
|
98
|
+
# Case boundaries (see Name) cut a segment into words without making
|
|
99
|
+
# them parts: an occurrence across one (`ScunThorpe`) is still inside
|
|
100
|
+
# one segment, the glued one, and is judged by it. One that lies inside
|
|
101
|
+
# a single word is judged by that word, so `ScunthorpeUnited` is exempt
|
|
102
|
+
# where `scunthorpeunited` is not.
|
|
103
|
+
class Whole
|
|
104
|
+
# What sits between separators.
|
|
105
|
+
PART = /[^_.-]+/
|
|
106
|
+
|
|
107
|
+
attr_reader :text, :parts, :segments, :joins
|
|
108
|
+
|
|
109
|
+
def initialize(form, folded: false, boundaries: [])
|
|
110
|
+
@folded = folded
|
|
111
|
+
@boundaries = boundaries
|
|
112
|
+
@text = +""
|
|
113
|
+
@parts = []
|
|
114
|
+
@segments = []
|
|
115
|
+
@joins = []
|
|
116
|
+
@origins = [] # [first part, last part, segment, word] per character of @text
|
|
117
|
+
form.scan(PART) { add(Regexp.last_match[0], Regexp.last_match.begin(0)) }
|
|
118
|
+
rewrite_text if folded
|
|
119
|
+
@text.freeze
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# Every substring of Lexicon::SUBSTRING_MINIMUM characters, in order:
|
|
123
|
+
# what each language's Lexicon::Index looks up, cut once per name.
|
|
124
|
+
def grams
|
|
125
|
+
size = Lexicon::SUBSTRING_MINIMUM
|
|
126
|
+
@grams ||= (0..(text.length - size)).map { |position| text[position, size] }.freeze
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Whether `spelling`, found at `position` of the text, straddles a separator.
|
|
130
|
+
def straddles?(spelling, position)
|
|
131
|
+
@origins[position][0] != @origins[position + spelling.length - 1][1]
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def part_at(position)
|
|
135
|
+
@parts[@origins[position][0]]
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# The segment that `length` characters from `position` lie in: the word
|
|
139
|
+
# between case boundaries when they fit in one, the segment otherwise,
|
|
140
|
+
# or nil on a digit.
|
|
141
|
+
def segment_at(position, length = 1)
|
|
142
|
+
_first, _last, segment, word = @origins[position]
|
|
143
|
+
return @segments[word] if word && word == @origins[position + length - 1][3]
|
|
144
|
+
|
|
145
|
+
@segments[segment] if segment
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
private
|
|
149
|
+
|
|
150
|
+
# `offset`: where the part starts in the form, to place case boundaries.
|
|
151
|
+
def add(part, offset)
|
|
152
|
+
part_index = @parts.size
|
|
153
|
+
@parts << folded(part)
|
|
154
|
+
part.scan(/\d+|\D+/) do |run|
|
|
155
|
+
start = offset + Regexp.last_match.begin(0)
|
|
156
|
+
if run.getbyte(0).between?(48, 57)
|
|
157
|
+
run.length.times { @origins << [part_index, part_index, nil, nil] }
|
|
158
|
+
else
|
|
159
|
+
add_segment(run, start, part_index)
|
|
160
|
+
end
|
|
161
|
+
@text << run
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# A run of letters: one segment, and one more for each word in it when
|
|
166
|
+
# case boundaries cut it.
|
|
167
|
+
def add_segment(run, start, part_index)
|
|
168
|
+
segment = @segments.size
|
|
169
|
+
@segments << folded(run)
|
|
170
|
+
words = words_in(run, start)
|
|
171
|
+
if words.one?
|
|
172
|
+
run.length.times { @origins << [part_index, part_index, segment, nil] }
|
|
173
|
+
else
|
|
174
|
+
words.each do |word|
|
|
175
|
+
@segments << folded(word)
|
|
176
|
+
index = @segments.size - 1
|
|
177
|
+
word.length.times { @origins << [part_index, part_index, segment, index] }
|
|
178
|
+
end
|
|
179
|
+
add_joins(words)
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# Every run of two or more adjacent words short of the whole segment,
|
|
184
|
+
# each folded as one text: a rewrite can span the boundary (`PHuck`).
|
|
185
|
+
def add_joins(words)
|
|
186
|
+
(0...words.size).each do |from|
|
|
187
|
+
((from + 2)..[words.size, from + Name::JOIN_WORDS].min).each do |to|
|
|
188
|
+
@joins << folded(words[from...to].join) unless from.zero? && to == words.size
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# `run` (which starts at `start` in the form) cut at the case boundaries inside it.
|
|
194
|
+
def words_in(run, start)
|
|
195
|
+
return [run] if @boundaries.empty?
|
|
196
|
+
|
|
197
|
+
cuts = @boundaries.filter_map { |at| at - start if at > start && at < start + run.length }
|
|
198
|
+
[0, *cuts].zip([*cuts, run.length]).map { |from, to| run[from...to] }
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def folded(text)
|
|
202
|
+
@folded ? Offensive.rewrite(text) : text
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# Offensive.rewrite over the joined text, carrying every character's origins along.
|
|
206
|
+
def rewrite_text
|
|
207
|
+
return if Offensive.rewrite(@text) == @text # most names: nothing to rewrite
|
|
208
|
+
|
|
209
|
+
characters = @text.chars.zip(@origins)
|
|
210
|
+
Offensive.rewrites.each { |from, to| characters = replace(characters, from, to) }
|
|
211
|
+
characters = cut_stretches(characters)
|
|
212
|
+
@text = characters.map(&:first).join
|
|
213
|
+
@origins = characters.map(&:last)
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
# `from` → `to` left to right, as String#gsub does; each new character
|
|
217
|
+
# comes from every part the replaced ones came from.
|
|
218
|
+
def replace(characters, from, to)
|
|
219
|
+
result = []
|
|
220
|
+
index = 0
|
|
221
|
+
while index < characters.size
|
|
222
|
+
window = characters[index, from.length]
|
|
223
|
+
if window.map(&:first).join == from
|
|
224
|
+
origin = merged(window.map(&:last))
|
|
225
|
+
to.each_char { |character| result << [character, origin] }
|
|
226
|
+
index += from.length
|
|
227
|
+
else
|
|
228
|
+
result << characters[index]
|
|
229
|
+
index += 1
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
result
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
# A run of three or more of one letter becomes two; the second stands
|
|
236
|
+
# for the rest of the run, wherever it came from.
|
|
237
|
+
def cut_stretches(characters)
|
|
238
|
+
characters.chunk_while { |a, b| a.first == b.first }.flat_map do |run|
|
|
239
|
+
next run if run.size < 3 || !run.first.first.match?(/[a-z]/)
|
|
240
|
+
|
|
241
|
+
[run.first, [run[1].first, merged(run[1..].map(&:last))]]
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# A character made from several keeps the first one's segment, and a
|
|
246
|
+
# word only when they all came from the same one.
|
|
247
|
+
def merged(origins)
|
|
248
|
+
words = origins.map(&:last).uniq
|
|
249
|
+
[origins.map(&:first).min, origins.map { |origin| origin[1] }.max, origins.first[2],
|
|
250
|
+
words.size == 1 ? words.first : nil]
|
|
251
|
+
end
|
|
252
|
+
end
|
|
253
|
+
end
|
|
254
|
+
end
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "offensive/name"
|
|
4
|
+
require_relative "offensive/lexicon"
|
|
5
|
+
require_relative "offensive/hate"
|
|
6
|
+
require_relative "adapters/moderate"
|
|
7
|
+
|
|
8
|
+
module Usernames
|
|
9
|
+
# Offensive words: bundled terms in severity tiers per language,
|
|
10
|
+
# hate-number rules and optional adapters (`moderate`, any callable).
|
|
11
|
+
#
|
|
12
|
+
# The whole design is a fight between two failure modes. Match too
|
|
13
|
+
# little and `f.u.c.k` or `sh1thead_bob` gets through; match substrings
|
|
14
|
+
# naively and Scunthorpe, Penistone, `grape` and `therapist` cannot sign
|
|
15
|
+
# up. So each tier is matched as narrowly as its words allow:
|
|
16
|
+
#
|
|
17
|
+
# a-1 severe (tier 4): inside any segment that is not exempt; short
|
|
18
|
+
# terms and `segment_only` ones must BE the segment (grape, Shōta)
|
|
19
|
+
# a-2 severe, across separators: in the name with separators removed,
|
|
20
|
+
# but only where the word straddles them (`fu_ck`, `gr_ape`)
|
|
21
|
+
# b strong (tier 3; tiers 1–2 too under :mild): a whole segment
|
|
22
|
+
# c hate numbers and tokens (Offensive::Hate)
|
|
23
|
+
# d adapters
|
|
24
|
+
#
|
|
25
|
+
# Exemptions (the bundled allowlist, config.offensive.allowed, a term's own
|
|
26
|
+
# exceptions) are whole SEGMENTS, never the joined name: otherwise
|
|
27
|
+
# `scunt_horpe` would join into the exempt `scunthorpe`. They lift a–b
|
|
28
|
+
# only; hate rules and adapters are authoritative.
|
|
29
|
+
module Offensive
|
|
30
|
+
# Which tiers rule b looks at, per severity mode (nil: rule b is off).
|
|
31
|
+
EQUAL_FROM = { severe: nil, strong: 3, mild: 1 }.freeze
|
|
32
|
+
|
|
33
|
+
# A letter repeated three times or more: a stretched spelling (`fuuuuuck`).
|
|
34
|
+
STRETCH = /([a-z])\1{2,}/
|
|
35
|
+
|
|
36
|
+
class << self
|
|
37
|
+
# The kind of match — :severe, :strong, :hate_number or :adapter — or nil
|
|
38
|
+
# when the name is fine. `canonical` is a name as stored that passed
|
|
39
|
+
# every format rule and is not reserved: the words are matched on its
|
|
40
|
+
# comparison key, split also where its case changes (see Name), and
|
|
41
|
+
# adapters get it as it is; `rules` is the
|
|
42
|
+
# Usernames::Rules in force (rules.offensive carries severity, languages,
|
|
43
|
+
# allowed and adapters; rules.on_adapter_failure what a raising adapter means).
|
|
44
|
+
#
|
|
45
|
+
# `:off` turns off everything the gem bundles, hate rules included;
|
|
46
|
+
# adapters run in every mode, because an app that configured one asked
|
|
47
|
+
# for it explicitly.
|
|
48
|
+
def verdict(canonical, rules)
|
|
49
|
+
severity = rules.offensive.severity
|
|
50
|
+
unless severity == :off
|
|
51
|
+
name = Name.new(rules.comparison_key(canonical), canonical)
|
|
52
|
+
rule = lexical(name, rules, severity) || Hate.verdict(name, rules)
|
|
53
|
+
return rule if rule
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
adapter_verdict(canonical, rules)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# A name's folded form (R1): the digit folds (`sh1t` → `shit`), then
|
|
60
|
+
# #rewrite. Terms are folded the same way when they are loaded, so
|
|
61
|
+
# `phuck` in the data and `fuck` typed meet.
|
|
62
|
+
def fold(string)
|
|
63
|
+
rewrite(Reserved.leet(string))
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# The rewrites of data/offensive/leet.yml in order — `vv`→`w`, `v`→`u`,
|
|
67
|
+
# `ph`→`f`, `x`→`ks` — then every run of three or more of one letter cut
|
|
68
|
+
# to two (`fuuuuuck` → `fuuck`). Not doubles: `ass` and `book` stay.
|
|
69
|
+
def rewrite(string)
|
|
70
|
+
rewritten = rewrites.reduce(string) do |text, (from, to)|
|
|
71
|
+
text.include?(from) ? text.gsub(from, to) : text
|
|
72
|
+
end
|
|
73
|
+
rewritten.gsub(STRETCH, '\1\1')
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# [[from, to], …]: the offensive-only rewrites of data/offensive/leet.yml, in order.
|
|
77
|
+
def rewrites
|
|
78
|
+
@rewrites ||= Data.leet.fetch("offensive_only").to_a.freeze
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
private
|
|
82
|
+
|
|
83
|
+
# Rules a and b over every enabled language: any language's rejection
|
|
84
|
+
# rejects, and a severe verdict outranks a strong one.
|
|
85
|
+
def lexical(name, rules, severity)
|
|
86
|
+
allowed = allowed_for(rules)
|
|
87
|
+
# A name the app allowed in so many words is exempt as a whole. The
|
|
88
|
+
# whole name is matched exactly, so joining segments still cannot
|
|
89
|
+
# manufacture an exemption.
|
|
90
|
+
return if allowed.include?(name.key)
|
|
91
|
+
|
|
92
|
+
lexicons = rules.offensive.languages.map { |locale| Lexicon.for(locale) }
|
|
93
|
+
return :severe if lexicons.any? { |lexicon| lexicon.severe?(name, allowed) }
|
|
94
|
+
|
|
95
|
+
minimum = EQUAL_FROM.fetch(severity)
|
|
96
|
+
:strong if minimum && lexicons.any? { |lexicon| lexicon.strong?(name, minimum, allowed) }
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# config.offensive.allowed, as typed and folded (an entry `sussex` must
|
|
100
|
+
# exempt the folded segment `susseks` too).
|
|
101
|
+
def allowed_for(rules)
|
|
102
|
+
rules.compiled(:offensive_allowed) do
|
|
103
|
+
rules.offensive.allowed.each_with_object(Set.new) do |word, set|
|
|
104
|
+
key = word.strip.downcase(:ascii)
|
|
105
|
+
set << key << fold(key)
|
|
106
|
+
end.freeze
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def adapter_verdict(canonical, rules)
|
|
111
|
+
:adapter if rules.offensive.adapters.any? { |adapter| flagged?(adapter, canonical, rules) }
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# A raising adapter (a network blip, a bug) follows on_adapter_failure:
|
|
115
|
+
# :allow (the default: an optional adapter fails open) skips
|
|
116
|
+
# it, :reject refuses the name. Either way it is logged, so a broken
|
|
117
|
+
# adapter is not silently a no-op forever.
|
|
118
|
+
# Adapters receive the name as stored, typed case and all: a capital is
|
|
119
|
+
# a word boundary a text classifier can use (see Adapters::Moderate).
|
|
120
|
+
def flagged?(adapter, canonical, rules)
|
|
121
|
+
callable = adapter == :moderate ? Adapters::Moderate : adapter
|
|
122
|
+
callable.call(canonical) ? true : false
|
|
123
|
+
rescue StandardError => e
|
|
124
|
+
reject = rules.on_adapter_failure == :reject
|
|
125
|
+
log("[usernames] offensive adapter #{adapter.inspect} raised #{e.class}: #{e.message} " \
|
|
126
|
+
"(#{reject ? 'name rejected' : 'skipped'}, on_adapter_failure: #{rules.on_adapter_failure.inspect})")
|
|
127
|
+
reject
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def log(message)
|
|
131
|
+
logger = ::Rails.logger if defined?(::Rails) && ::Rails.respond_to?(:logger)
|
|
132
|
+
logger ? logger.warn(message) : warn(message)
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Usernames
|
|
4
|
+
# A piece of an error message that depends on the rules in force and on the
|
|
5
|
+
# locale: "underscores, periods and dashes", "an underscore or a period",
|
|
6
|
+
# "lowercase letters".
|
|
7
|
+
#
|
|
8
|
+
# Translated when the message is READ, not when the error is added. Active
|
|
9
|
+
# Model keeps an error's options and builds its message on every read, so a
|
|
10
|
+
# record validated in English and rendered inside `I18n.with_locale(:es)`
|
|
11
|
+
# (a mailer, a background job) must not carry an English fragment into a
|
|
12
|
+
# Spanish sentence. I18n calls any interpolation value that responds to
|
|
13
|
+
# #call; everything else (errors.details, JSON, inspect) sees the text.
|
|
14
|
+
class Phrase
|
|
15
|
+
SEPARATOR_NAMES = { "_" => :underscore, "." => :period, "-" => :dash }.freeze
|
|
16
|
+
SCOPE = %i[usernames separators].freeze
|
|
17
|
+
|
|
18
|
+
class << self
|
|
19
|
+
# "underscores", "underscores and periods", "underscores, periods and
|
|
20
|
+
# dashes" — every separator allowed, in the order they were configured.
|
|
21
|
+
def separators(separators)
|
|
22
|
+
new { join(separators, form: :plural, connector: :and) }
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# "an underscore", "an underscore or a period" — any one of them.
|
|
26
|
+
def any_separator(separators)
|
|
27
|
+
new { join(separators, form: :singular, connector: :or) }
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# "lowercase letters" or, when the typed case is kept, "letters".
|
|
31
|
+
def letters(case_mode)
|
|
32
|
+
new { I18n.t(case_mode == :lowercase ? :lowercase : :any_case, scope: %i[usernames letters]) }
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# The list sentence itself, in the current locale. Not to_sentence:
|
|
36
|
+
# Rails' English connectors carry an Oxford comma, and these sentences
|
|
37
|
+
# are written without one in all three shipped languages.
|
|
38
|
+
def join(separators, form:, connector:)
|
|
39
|
+
words = separators.map { |separator| I18n.t(form, scope: [*SCOPE, SEPARATOR_NAMES.fetch(separator)]) }
|
|
40
|
+
return words.join if words.size < 2
|
|
41
|
+
|
|
42
|
+
connectors = [*SCOPE, :connectors]
|
|
43
|
+
[words[0..-2].join(I18n.t(:words, scope: connectors)), words.last].join(I18n.t(connector, scope: connectors))
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def initialize(&translate)
|
|
48
|
+
@translate = translate
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def to_s
|
|
52
|
+
@translate.call.to_s
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# What I18n calls while interpolating.
|
|
56
|
+
def call(*)
|
|
57
|
+
to_s
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def as_json(*)
|
|
61
|
+
to_s
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def inspect
|
|
65
|
+
to_s.inspect
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Usernames
|
|
4
|
+
module Reserved
|
|
5
|
+
# `rails usernames:reserved`: every reserved word in force and where it
|
|
6
|
+
# comes from, for the day somebody asks why `x` is taken by nobody.
|
|
7
|
+
#
|
|
8
|
+
# admin impersonation · big-username-blocklist (MIT) — …
|
|
9
|
+
# recipe config.reserved
|
|
10
|
+
# pricing route /pricing
|
|
11
|
+
#
|
|
12
|
+
# A word on several lists is shown with the one that decides how it is
|
|
13
|
+
# matched: impersonation first (it is matched in every spelling), then
|
|
14
|
+
# your own lists, the bundled exact words and your routes.
|
|
15
|
+
module Listing
|
|
16
|
+
class << self
|
|
17
|
+
# [[word, source], …], sorted by word, without the words
|
|
18
|
+
# config.allowed lifts (see .allowed).
|
|
19
|
+
def entries(rules = Usernames.configuration.rules_for)
|
|
20
|
+
sources = {}
|
|
21
|
+
impersonation(rules).each { |word, source| sources[word] ||= source }
|
|
22
|
+
exact(rules).each { |word, source| sources[word] ||= source }
|
|
23
|
+
routes(rules).each { |word| sources[word] ||= "route /#{word}" }
|
|
24
|
+
lifted = allowed(rules)
|
|
25
|
+
sources.except(*lifted).sort
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# The words config.allowed lifts, sorted.
|
|
29
|
+
def allowed(rules = Usernames.configuration.rules_for)
|
|
30
|
+
keys(rules.allowed).sort
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def report(io = $stdout, rules = Usernames.configuration.rules_for)
|
|
34
|
+
words = entries(rules)
|
|
35
|
+
width = words.map { |word, _source| word.length }.max.to_i
|
|
36
|
+
io.puts "#{words.size} reserved names (impersonation words also refuse their spellings: adm1n, ad.min, admins)"
|
|
37
|
+
words.each { |word, source| io.puts " #{word.ljust(width)} #{source}" }
|
|
38
|
+
lifted = allowed(rules)
|
|
39
|
+
io.puts "Lifted by config.allowed: #{lifted.join(' ')}" if lifted.any?
|
|
40
|
+
words.size
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
private
|
|
44
|
+
|
|
45
|
+
def impersonation(rules)
|
|
46
|
+
bundled = Data.impersonation.to_set
|
|
47
|
+
keys(rules.forbidden).map do |word|
|
|
48
|
+
[word, bundled.include?(word) ? bundled_source(word) : "config.forbidden"]
|
|
49
|
+
end +
|
|
50
|
+
Reserved.app_names(rules).map { |word| [word, "impersonation · your app's name"] }
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def exact(rules)
|
|
54
|
+
bundled = Data.reserved.to_set
|
|
55
|
+
custom, shipped = keys(rules.reserved).partition { |word| !bundled.include?(word) }
|
|
56
|
+
custom.map { |word| [word, "config.reserved"] } + shipped.map { |word| [word, bundled_source(word)] }
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def routes(rules)
|
|
60
|
+
rules.reserve_routes ? Reserved.routes.to_a : []
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# "web · gitlab (MIT) — GitLab top-level and project wildcard routes"
|
|
64
|
+
def bundled_source(word)
|
|
65
|
+
provenance = Reserved.source_of(word)
|
|
66
|
+
return "bundled" unless provenance
|
|
67
|
+
|
|
68
|
+
origin = [provenance.source, ("(#{provenance.licence})" if provenance.licence)].compact.join(" ")
|
|
69
|
+
text = [provenance.category, (origin unless origin.empty?)].compact.join(" · ")
|
|
70
|
+
provenance.reason ? "#{text} — #{provenance.reason}" : text
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def keys(words)
|
|
74
|
+
words.map { |word| word.strip.downcase(:ascii) }.uniq
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|