swear_jar 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +22 -0
- data/LICENSE.txt +22 -0
- data/README.md +313 -0
- data/config/locales/en.yml +4 -0
- data/config/locales/pl.yml +4 -0
- data/lib/swear_jar/configuration.rb +56 -0
- data/lib/swear_jar/detector.rb +58 -0
- data/lib/swear_jar/entry.rb +82 -0
- data/lib/swear_jar/errors.rb +7 -0
- data/lib/swear_jar/fold_table.rb +115 -0
- data/lib/swear_jar/match.rb +10 -0
- data/lib/swear_jar/matcher.rb +289 -0
- data/lib/swear_jar/normalizer.rb +163 -0
- data/lib/swear_jar/pattern_builder.rb +221 -0
- data/lib/swear_jar/railtie.rb +36 -0
- data/lib/swear_jar/skeleton.rb +116 -0
- data/lib/swear_jar/slicer.rb +58 -0
- data/lib/swear_jar/swear_jar_validator.rb +44 -0
- data/lib/swear_jar/version.rb +5 -0
- data/lib/swear_jar/word_list.rb +167 -0
- data/lib/swear_jar.rb +27 -0
- data/locales/en/words.yml +266 -0
- data/locales/pl/words.yml +213 -0
- metadata +67 -0
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SwearJar
|
|
4
|
+
# Compiles a word list into one detection regexp (generated from a trie of stem runs),
|
|
5
|
+
# per-spelling anchored regexps for entry identification, and an allowlist regexp.
|
|
6
|
+
#
|
|
7
|
+
# @api private
|
|
8
|
+
class PatternBuilder
|
|
9
|
+
LEET = {
|
|
10
|
+
'a' => 'a4@', 'o' => 'o0', 'i' => 'i1!|', 'e' => 'e3', 's' => 's$5',
|
|
11
|
+
't' => 't7', 'l' => 'l1|', 'b' => 'b8', 'g' => 'g9'
|
|
12
|
+
}.freeze
|
|
13
|
+
CLASS_SPECIALS = ['-', ']', '[', '\\', '^'].freeze
|
|
14
|
+
# Leet symbols. They are not word characters for LEFT, so a word may start after them.
|
|
15
|
+
SYMBOLS = '!|@$'
|
|
16
|
+
LEFT = '(?<![\p{L}\p{N}*])'
|
|
17
|
+
RIGHT = '(?![\p{L}\p{N}*])'
|
|
18
|
+
JOIN = '[.\-~]*'
|
|
19
|
+
GAP = '[.\-~ ]*'
|
|
20
|
+
NEVER = /(?!)/
|
|
21
|
+
LETTER = /\p{L}/
|
|
22
|
+
DIGIT = /\p{N}/
|
|
23
|
+
PREFIXED_LOCALES = %i[pl].freeze
|
|
24
|
+
# Trie token for a run of non-letters inside a stem (e.g. the space in a multi-word entry).
|
|
25
|
+
GAP_TOKEN = :gap
|
|
26
|
+
TERMINAL = :end
|
|
27
|
+
|
|
28
|
+
attr_reader :mode, :scanner, :entry_patterns, :allow_pattern, :prefixes
|
|
29
|
+
|
|
30
|
+
# +entry_patterns+: [entry, anchored (`\G`) regexp of the stem, characters the stem can start
|
|
31
|
+
# with], one per folded spelling. A stem regexp has no prefixes: the matcher tries them.
|
|
32
|
+
# +prefixes+: locale => folded prefixes, longest first (empty for unprefixed locales).
|
|
33
|
+
# +allow_pattern+: the allowlist regexp, or nil.
|
|
34
|
+
def initialize(word_list, mode:)
|
|
35
|
+
raise ArgumentError, "unknown mode #{mode.inspect}, expected #{MODES.inspect}" unless MODES.include?(mode)
|
|
36
|
+
|
|
37
|
+
@mode = mode
|
|
38
|
+
@prefixes = folded_prefixes(word_list.prefixes)
|
|
39
|
+
spellings = word_list.entries.map { |entry| [entry, spelling_tokens(entry)] }
|
|
40
|
+
@scanner = build_scanner(spellings, mode)
|
|
41
|
+
@entry_patterns = build_entry_patterns(spellings)
|
|
42
|
+
@allow_pattern = build_allow_pattern(word_list)
|
|
43
|
+
freeze
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def inspect
|
|
47
|
+
"#<#{self.class.name} mode=#{mode.inspect} entries=#{entry_patterns.map(&:first).uniq.size}>"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
def spelling_tokens(entry)
|
|
53
|
+
where = "#{entry.source}: entry #{entry.stem.inspect}"
|
|
54
|
+
entry.spellings.map { |spelling| tokens(spelling, where) }.uniq
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Folds +word+ and splits it into runs of identical letters, [letter, count, final], with
|
|
58
|
+
# GAP_TOKEN standing for any run of non-letters between them. Only the last run is final.
|
|
59
|
+
def tokens(word, where)
|
|
60
|
+
list = []
|
|
61
|
+
Normalizer.fold(word).each_char do |char|
|
|
62
|
+
raise InvalidWordListError, "#{where}: #{word.inspect} contains a digit" if DIGIT.match?(char)
|
|
63
|
+
|
|
64
|
+
if !LETTER.match?(char)
|
|
65
|
+
list << GAP_TOKEN unless list.empty? || list.last == GAP_TOKEN
|
|
66
|
+
elsif list.last.is_a?(Array) && list.last[0] == char
|
|
67
|
+
list[-1] = [char, list.last[1] + 1, false]
|
|
68
|
+
else
|
|
69
|
+
list << [char, 1, false]
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
list.pop if list.last == GAP_TOKEN
|
|
73
|
+
raise InvalidWordListError, "#{where}: #{word.inspect} has no letters" if list.empty?
|
|
74
|
+
|
|
75
|
+
list[-1] = [*list.last.take(2), true]
|
|
76
|
+
list
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def build_scanner(spellings, mode)
|
|
80
|
+
exact, loose = spellings.partition { |entry, _| entry.exact }
|
|
81
|
+
bounded, free = loose.partition { |entry, _| mode == :words || !entry.embeddable }
|
|
82
|
+
bounded_parts = LOCALES.filter_map { |locale| locale_branch(bounded, locale, guard: :symbols) }
|
|
83
|
+
exact_trie = render_trie(exact.flat_map(&:last), guard: :symbols)
|
|
84
|
+
bounded_parts << "#{exact_trie}#{RIGHT}" if exact_trie
|
|
85
|
+
free_parts = LOCALES.filter_map { |locale| locale_branch(free, locale, guard: :run, prefixed: false) }
|
|
86
|
+
|
|
87
|
+
parts = []
|
|
88
|
+
parts << "#{LEFT}(?:#{bounded_parts.join('|')})" unless bounded_parts.empty?
|
|
89
|
+
parts << "(?:#{free_parts.join('|')})" unless free_parts.empty?
|
|
90
|
+
parts.empty? ? NEVER : Regexp.new(parts.join('|'))
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def locale_branch(spellings, locale, guard:, prefixed: true)
|
|
94
|
+
trie = render_trie(spellings.select { |entry, _| entry.locale == locale }.flat_map(&:last), guard: guard)
|
|
95
|
+
return unless trie
|
|
96
|
+
|
|
97
|
+
prefixed ? "#{prefix_group(locale)}#{trie}" : trie
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Folded prefixes per locale, longest first. Only PREFIXED_LOCALES have any.
|
|
101
|
+
def folded_prefixes(prefixes)
|
|
102
|
+
LOCALES.to_h do |locale|
|
|
103
|
+
list = PREFIXED_LOCALES.include?(locale) ? Array(prefixes[locale]) : []
|
|
104
|
+
folded = list.map { |p| Normalizer.fold(p) }.reject(&:empty?).uniq
|
|
105
|
+
[locale, folded.sort_by.with_index { |p, i| [-p.length, i] }.freeze]
|
|
106
|
+
end.freeze
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def prefix_group(locale)
|
|
110
|
+
list = @prefixes[locale]
|
|
111
|
+
list.empty? ? '' : "(?:#{list.map { |p| Regexp.escape(p) }.join('|')})?"
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def build_entry_patterns(spellings)
|
|
115
|
+
spellings.flat_map do |entry, token_lists|
|
|
116
|
+
suffix = entry.exact ? RIGHT : ''
|
|
117
|
+
token_lists.map do |list|
|
|
118
|
+
# The first run is never masked, so a stem starts with a character of its class.
|
|
119
|
+
first_chars = (LEET[list.first[0]] || list.first[0]).chars.freeze
|
|
120
|
+
[entry, Regexp.new("\\G#{render_trie([list])}#{suffix}"), first_chars].freeze
|
|
121
|
+
end
|
|
122
|
+
end.freeze
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def build_allow_pattern(word_list)
|
|
126
|
+
sources = word_list.allow_sources
|
|
127
|
+
lists = word_list.allow.map { |word| tokens(word, "#{sources.fetch(word, 'allow')}: allow word") }
|
|
128
|
+
trie = render_trie(lists, guard: :run)
|
|
129
|
+
trie && Regexp.new("\\G#{trie}")
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Builds a trie (nested Hashes keyed by token, TERMINAL marks an end) and renders it.
|
|
133
|
+
# Final runs are distinct tokens, so a node is either terminal or has children, never both.
|
|
134
|
+
# +guard+ adds a lookbehind to each root run, so a pattern searched without an anchor
|
|
135
|
+
# cannot start inside a run and rescan it (quadratic without regexp memoization):
|
|
136
|
+
# :run rejects any character of the run's class before it; :symbols rejects two leet
|
|
137
|
+
# symbols of the class (for branches behind LEFT, which a single symbol does not stop:
|
|
138
|
+
# `hello!idiot` must still match).
|
|
139
|
+
def render_trie(token_lists, guard: nil)
|
|
140
|
+
root = {}
|
|
141
|
+
token_lists.each do |list|
|
|
142
|
+
node = list.reduce(root) { |current, token| current[token] ||= {} }
|
|
143
|
+
node[TERMINAL] = true
|
|
144
|
+
end
|
|
145
|
+
render_children(root, nil, guard)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# Renders the alternatives below +node+, whose own token is +parent+ (nil at the root).
|
|
149
|
+
# Non-final runs come first so the longer spellings are tried first.
|
|
150
|
+
# Returns nil when the node has no children.
|
|
151
|
+
def render_children(node, parent, guard = nil)
|
|
152
|
+
children = node.reject { |token, _| token == TERMINAL }
|
|
153
|
+
children = children.sort_by.with_index { |(token, _), i| [final?(token) ? 1 : 0, i] }
|
|
154
|
+
branches = children.map do |token, child|
|
|
155
|
+
[lead(parent, token), "#{atom(token, parent.nil?, guard)}#{render_children(child, token)}"]
|
|
156
|
+
end
|
|
157
|
+
return if branches.empty?
|
|
158
|
+
|
|
159
|
+
leads = branches.map(&:first).uniq
|
|
160
|
+
return "#{leads.first}#{group(branches.map(&:last))}" if leads.size == 1
|
|
161
|
+
|
|
162
|
+
group(branches.map { |head, body| "#{head}#{body}" })
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def final?(token)
|
|
166
|
+
token.is_a?(Array) && token[2]
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def group(alternatives)
|
|
170
|
+
alternatives.size == 1 ? alternatives.first : "(?:#{alternatives.join('|')})"
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def lead(parent, token)
|
|
174
|
+
parent.nil? || parent == GAP_TOKEN || token == GAP_TOKEN ? '' : JOIN
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
# A run is atomic and greedy. Note: when adjacent runs have overlapping leet classes
|
|
178
|
+
# (i `[i1!|]` and l `[l1|]`), the first run swallows the shared symbols and the match fails.
|
|
179
|
+
# Masks may replace only interior runs, never the first or the last one, and stand for
|
|
180
|
+
# up to as many letters as the run has (`a**hole`).
|
|
181
|
+
def atom(token, first, guard)
|
|
182
|
+
return GAP if token == GAP_TOKEN
|
|
183
|
+
|
|
184
|
+
letter, count, final = token
|
|
185
|
+
klass = letter_class(letter)
|
|
186
|
+
return guarded_run(letter, klass, count, guard) if first
|
|
187
|
+
|
|
188
|
+
run = "(?>#{klass}#{count == 1 ? '+' : "{#{count},}"})"
|
|
189
|
+
return run if final
|
|
190
|
+
|
|
191
|
+
masks = count == 1 ? '\\*' : "\\*{1,#{count}}"
|
|
192
|
+
"(?:#{run}|#{masks})"
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# The guard looks behind the first character of the run rather than before it, so it only
|
|
196
|
+
# runs where that character already matched: the many branches of the scanner tried at
|
|
197
|
+
# every word start stay cheap.
|
|
198
|
+
def guarded_run(letter, klass, count, guard)
|
|
199
|
+
rest = count == 1 ? '*' : "{#{count - 1},}"
|
|
200
|
+
"(?>#{klass}#{guard_behind(letter, klass, guard)}#{klass}#{rest})"
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def guard_behind(letter, klass, guard)
|
|
204
|
+
case guard
|
|
205
|
+
when :run then "(?<!#{klass}{2})"
|
|
206
|
+
when :symbols
|
|
207
|
+
symbols = (LEET[letter] || letter).delete("^#{SYMBOLS}")
|
|
208
|
+
symbols.empty? ? '' : "(?<!#{class_of(symbols.chars)}{2}#{klass})"
|
|
209
|
+
else ''
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def letter_class(letter)
|
|
214
|
+
class_of((LEET[letter] || letter).chars.uniq)
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def class_of(chars)
|
|
218
|
+
"[#{chars.map { |c| CLASS_SPECIALS.include?(c) ? "\\#{c}" : c }.join}]"
|
|
219
|
+
end
|
|
220
|
+
end
|
|
221
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'rails/railtie'
|
|
4
|
+
|
|
5
|
+
module SwearJar
|
|
6
|
+
class Railtie < Rails::Railtie
|
|
7
|
+
config.swear_jar = ActiveSupport::OrderedOptions.new
|
|
8
|
+
|
|
9
|
+
# Unknown keys raise ArgumentError.
|
|
10
|
+
def self.apply_config(options)
|
|
11
|
+
return if options.empty?
|
|
12
|
+
|
|
13
|
+
options.each_key do |key|
|
|
14
|
+
raise ArgumentError, "unknown config.swear_jar option: #{key.inspect}" unless Configuration::DEFAULTS.key?(key.to_sym)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
SwearJar.configure do |configuration|
|
|
18
|
+
options.each { |key, value| configuration.public_send(:"#{key}=", value) }
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Builds the global detector at boot rather than on first use, so app servers that preload
|
|
23
|
+
# the app (Puma preload_app!) share it between workers through copy-on-write.
|
|
24
|
+
initializer 'swear_jar.configure', after: :load_config_initializers do |app|
|
|
25
|
+
Railtie.apply_config(app.config.swear_jar)
|
|
26
|
+
SwearJar.detector
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
initializer 'swear_jar.validator' do
|
|
30
|
+
require 'swear_jar/swear_jar_validator'
|
|
31
|
+
rescue LoadError => e
|
|
32
|
+
# railties does not depend on activemodel; only skip when that is what is missing.
|
|
33
|
+
raise unless e.path == 'active_model'
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'strscan'
|
|
4
|
+
|
|
5
|
+
module SwearJar
|
|
6
|
+
# Read access to one skeleton by character index, in constant time per call. Ruby indexes a
|
|
7
|
+
# non-ASCII string by scanning it from the start, so a message with many hits would cost
|
|
8
|
+
# O(length) per hit; here a non-ASCII skeleton gets a character <=> byte offset table once
|
|
9
|
+
# and all matching goes through a StringScanner at byte positions. Anchored matching also
|
|
10
|
+
# needs StringScanner: Regexp#match(string, pos) with `\G` is not reliably anchored, since for
|
|
11
|
+
# a pattern with a required substring (`{2,}` runs) Onigmo first searches the rest of the
|
|
12
|
+
# string for it. fixed_anchor lets lookbehinds see before the position.
|
|
13
|
+
#
|
|
14
|
+
# @api private
|
|
15
|
+
class Skeleton
|
|
16
|
+
NON_WORD = /[^\p{L}\p{N}*@$!|]/
|
|
17
|
+
|
|
18
|
+
attr_reader :string, :length
|
|
19
|
+
|
|
20
|
+
def initialize(string)
|
|
21
|
+
@string = string
|
|
22
|
+
@length = string.length
|
|
23
|
+
@scanner = StringScanner.new(string, fixed_anchor: true)
|
|
24
|
+
build_offsets unless string.ascii_only?
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def [](index)
|
|
28
|
+
return @string[index] unless @offsets
|
|
29
|
+
return if index >= @length
|
|
30
|
+
|
|
31
|
+
@string.byteslice(@offsets[index], @offsets[index + 1] - @offsets[index])
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def slice(first, last)
|
|
35
|
+
return @string[first...last] unless @offsets
|
|
36
|
+
|
|
37
|
+
@string.byteslice(@offsets[first], @offsets[last] - @offsets[first])
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# The end of a +pattern+ match starting exactly at +at+, or nil.
|
|
41
|
+
def match(pattern, at)
|
|
42
|
+
@scanner.pos = byte(at)
|
|
43
|
+
@scanner.skip(pattern) && char(@scanner.pos)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def literal?(string, at)
|
|
47
|
+
@scanner.pos = byte(at)
|
|
48
|
+
!@scanner.skip(string).nil?
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# [begin, end] of the first +pattern+ match at or after +from+, or nil.
|
|
52
|
+
def search(pattern, from)
|
|
53
|
+
@scanner.pos = byte(from)
|
|
54
|
+
return unless @scanner.skip_until(pattern)
|
|
55
|
+
|
|
56
|
+
[char(@scanner.pos - @scanner.matched_size), char(@scanner.pos)]
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def word_start(pos)
|
|
60
|
+
return 0 if pos.zero?
|
|
61
|
+
|
|
62
|
+
unless @offsets
|
|
63
|
+
boundary = @string.rindex(NON_WORD, pos - 1)
|
|
64
|
+
return boundary ? boundary + 1 : 0
|
|
65
|
+
end
|
|
66
|
+
index = breaks.bsearch_index { |at| at >= pos } || breaks.size
|
|
67
|
+
index.zero? ? 0 : breaks[index - 1] + 1
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# The end of the word that contains +pos+ (or +pos+ itself when it is not in a word).
|
|
71
|
+
def word_end(pos)
|
|
72
|
+
return @string.index(NON_WORD, pos) || @length unless @offsets
|
|
73
|
+
|
|
74
|
+
breaks.bsearch { |at| at >= pos } || @length
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def inspect
|
|
78
|
+
"#<#{self.class.name} length=#{@length}>"
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
private
|
|
82
|
+
|
|
83
|
+
def byte(char)
|
|
84
|
+
@offsets ? @offsets[char] : char
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def char(byte)
|
|
88
|
+
@chars ? @chars[byte] : byte
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# @offsets: character index => byte offset (plus the total size at the end);
|
|
92
|
+
# @chars: byte offset of a character => its index.
|
|
93
|
+
def build_offsets
|
|
94
|
+
@offsets = []
|
|
95
|
+
@chars = []
|
|
96
|
+
offset = 0
|
|
97
|
+
@string.each_char.with_index do |character, index|
|
|
98
|
+
@offsets << offset
|
|
99
|
+
@chars[offset] = index
|
|
100
|
+
offset += character.bytesize
|
|
101
|
+
end
|
|
102
|
+
@offsets << offset
|
|
103
|
+
@chars[offset] = @length
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Sorted indexes of the non-word characters, built on first use.
|
|
107
|
+
def breaks
|
|
108
|
+
@breaks ||= begin
|
|
109
|
+
list = []
|
|
110
|
+
@scanner.pos = 0
|
|
111
|
+
list << char(@scanner.pos - @scanner.matched_size) while @scanner.skip_until(NON_WORD)
|
|
112
|
+
list
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SwearJar
|
|
4
|
+
# Cuts match texts out of the original string. Indexing a long non-ASCII string by
|
|
5
|
+
# characters, or slicing a long piece of it, scans it from the start, so slicing each of many
|
|
6
|
+
# matches in a long message would cost O(length) per match. Matches come in order, so a
|
|
7
|
+
# cursor (character and byte offset) moves forward instead, and each slice works on a short
|
|
8
|
+
# byte window, costing only the distance from the previous match.
|
|
9
|
+
#
|
|
10
|
+
# @api private
|
|
11
|
+
class Slicer
|
|
12
|
+
# Combining marks right after a match belong to its last character.
|
|
13
|
+
TRAILING_MARKS = /\G\p{M}+/
|
|
14
|
+
# A UTF-8 character takes at most 4 bytes; extra room for the marks after a match.
|
|
15
|
+
CHAR_BYTES = 4
|
|
16
|
+
MARKS_ROOM = 64
|
|
17
|
+
|
|
18
|
+
def initialize(string)
|
|
19
|
+
@string = string
|
|
20
|
+
@char = 0
|
|
21
|
+
@byte = 0
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# [range, text] for the characters from +first+ to +last+ (exclusive), extended over the
|
|
25
|
+
# combining marks that follow. +first+ must not be before the previous call's +first+.
|
|
26
|
+
def slice(first, last)
|
|
27
|
+
skip_to(first)
|
|
28
|
+
length = last - first
|
|
29
|
+
room = (CHAR_BYTES * length) + MARKS_ROOM
|
|
30
|
+
loop do
|
|
31
|
+
window = window(room)
|
|
32
|
+
marks = TRAILING_MARKS.match(window, length)
|
|
33
|
+
if marks.nil? || marks.end(0) < window.length || @byte + room >= @string.bytesize
|
|
34
|
+
length = marks.end(0) if marks
|
|
35
|
+
return [first...(first + length), window[0, length]]
|
|
36
|
+
end
|
|
37
|
+
room *= 2
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def inspect
|
|
42
|
+
"#<#{self.class.name} at=#{@char}>"
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
def skip_to(first)
|
|
48
|
+
distance = first - @char
|
|
49
|
+
@byte += window(CHAR_BYTES * distance)[0, distance].bytesize
|
|
50
|
+
@char = first
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Up to +bytes+ bytes from the cursor, without a character cut off at the end.
|
|
54
|
+
def window(bytes)
|
|
55
|
+
@string.byteslice(@byte, bytes).scrub('')
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'swear_jar'
|
|
4
|
+
require 'active_model'
|
|
5
|
+
require 'i18n'
|
|
6
|
+
|
|
7
|
+
I18n.load_path |= Dir[File.join(SwearJar::ROOT_PATH, 'config', 'locales', '*.yml')]
|
|
8
|
+
|
|
9
|
+
# Validates that an attribute contains no profanity. Without detector options it uses
|
|
10
|
+
# +SwearJar.detector+; otherwise it uses a detector cached per option set.
|
|
11
|
+
class SwearJarValidator < ActiveModel::EachValidator
|
|
12
|
+
STANDARD_OPTIONS = %i[message if unless on allow_nil allow_blank strict].freeze
|
|
13
|
+
|
|
14
|
+
@detectors = {}
|
|
15
|
+
@mutex = Mutex.new
|
|
16
|
+
|
|
17
|
+
class << self
|
|
18
|
+
def detector_for(detector_options)
|
|
19
|
+
key = detector_options.sort_by { |name, _| name.to_s }.to_h { |name, value| [name, freeze_value(value)] }.freeze
|
|
20
|
+
@detectors[key] || @mutex.synchronize { @detectors[key] ||= SwearJar::Detector.new(**key) }
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def self.freeze_value(value)
|
|
25
|
+
value.is_a?(Array) ? value.map { |item| freeze_value(item) }.freeze : value.dup.freeze
|
|
26
|
+
end
|
|
27
|
+
private_class_method :freeze_value
|
|
28
|
+
|
|
29
|
+
def validate_each(record, attribute, value)
|
|
30
|
+
return if value.blank?
|
|
31
|
+
|
|
32
|
+
# A custom message replaces only the text: the error type stays :profane.
|
|
33
|
+
record.errors.add(attribute, :profane, **options.slice(:message)) if detector.profane?(value.to_s)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def detector
|
|
39
|
+
detector_options = options.except(*STANDARD_OPTIONS)
|
|
40
|
+
return SwearJar.detector if detector_options.empty?
|
|
41
|
+
|
|
42
|
+
self.class.detector_for(detector_options)
|
|
43
|
+
end
|
|
44
|
+
end
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'yaml'
|
|
4
|
+
|
|
5
|
+
module SwearJar
|
|
6
|
+
# @api private
|
|
7
|
+
class WordList
|
|
8
|
+
FILE_KEYS = %w[locale prefixes words allow].freeze
|
|
9
|
+
EXTRA_SOURCE = 'extra_words'
|
|
10
|
+
ALLOW_SOURCE = 'allow option'
|
|
11
|
+
GLOB_CHARS = /[*?\[{]/
|
|
12
|
+
|
|
13
|
+
# +allow_sources+: each allow word => where it came from, for error messages.
|
|
14
|
+
attr_reader :locales, :entries, :prefixes, :allow, :allow_sources
|
|
15
|
+
|
|
16
|
+
def self.load(locales: LOCALES, min_severity: :mild, categories: nil, extra_paths: [], extra_words: [], allow: [])
|
|
17
|
+
locales = validated_option(:locales, symbolize(Array(locales)), LOCALES)
|
|
18
|
+
min_severity = validated_option(:min_severity, symbolize([min_severity]), SEVERITIES).first
|
|
19
|
+
categories = validated_option(:categories, symbolize(Array(categories)), CATEGORIES) if categories
|
|
20
|
+
new(locales: locales, min_severity: min_severity, categories: categories,
|
|
21
|
+
extra_paths: extra_paths, extra_words: extra_words, allow: allow)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# An empty list is rejected too: it would build a detector that silently finds nothing.
|
|
25
|
+
def self.validated_option(name, values, allowed)
|
|
26
|
+
raise ArgumentError, "#{name} must not be empty, expected some of #{allowed.inspect}" if values.empty?
|
|
27
|
+
|
|
28
|
+
unknown = values - allowed
|
|
29
|
+
return values if unknown.empty?
|
|
30
|
+
|
|
31
|
+
raise ArgumentError, "unknown #{name} #{unknown.inspect}, expected #{allowed.inspect}"
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def self.symbolize(values)
|
|
35
|
+
values.map { |v| v.respond_to?(:to_sym) ? v.to_sym : v }
|
|
36
|
+
end
|
|
37
|
+
private_class_method :validated_option, :symbolize
|
|
38
|
+
|
|
39
|
+
def initialize(locales:, min_severity:, categories:, extra_paths:, extra_words:, allow:)
|
|
40
|
+
@locales = locales.freeze
|
|
41
|
+
sources = bundled_sources(locales) + extra_sources(extra_paths, locales) + extra_word_sources(extra_words)
|
|
42
|
+
min_rank = SEVERITIES.index(min_severity)
|
|
43
|
+
@entries = sources.flat_map { |s| s[:entries] }.select { |e| keep?(e, locales, min_rank, categories) }.freeze
|
|
44
|
+
@prefixes = collect_prefixes(sources, locales)
|
|
45
|
+
@allow_sources = collect_allow(sources, allow)
|
|
46
|
+
@allow = @allow_sources.keys.freeze
|
|
47
|
+
freeze
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def inspect
|
|
51
|
+
"#<#{self.class.name} entries=#{entries.size} allow=#{allow.size}>"
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
def keep?(entry, locales, min_rank, categories)
|
|
57
|
+
locales.include?(entry.locale) && entry.rank >= min_rank &&
|
|
58
|
+
(categories.nil? || (categories & entry.categories).any?)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def collect_prefixes(sources, locales)
|
|
62
|
+
locales.to_h do |locale|
|
|
63
|
+
list = sources.select { |s| s[:locale] == locale }.flat_map { |s| s[:prefixes] }.uniq
|
|
64
|
+
[locale, list.sort_by.with_index { |p, i| [-p.length, i] }.freeze]
|
|
65
|
+
end.freeze
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Allow words in order, without duplicates => the source of their first occurrence.
|
|
69
|
+
def collect_allow(sources, allow)
|
|
70
|
+
pairs = sources.flat_map { |s| s[:allow].map { |word| [word, s[:path]] } }
|
|
71
|
+
pairs += Array(allow).map { |word| [word.to_s.dup.freeze, ALLOW_SOURCE] }
|
|
72
|
+
pairs.each_with_object({}) { |(word, source), all| all[word] ||= source }.freeze
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def bundled_sources(locales)
|
|
76
|
+
locales.map { |locale| read_file(File.join(ROOT_PATH, 'locales', locale.to_s, 'words.yml'), locale) }
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def extra_sources(extra_paths, locales)
|
|
80
|
+
Array(extra_paths).flat_map { |pattern| extra_files(pattern.to_s) }.filter_map do |path|
|
|
81
|
+
source = read_file(path, nil)
|
|
82
|
+
source if locales.include?(source[:locale])
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# A glob may match nothing. A plain path is read as is, so a missing file raises.
|
|
87
|
+
def extra_files(pattern)
|
|
88
|
+
pattern.match?(GLOB_CHARS) ? Dir.glob(pattern).sort : [pattern]
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def extra_word_sources(extra_words)
|
|
92
|
+
Array(extra_words).map do |hash|
|
|
93
|
+
raise InvalidWordListError, "#{EXTRA_SOURCE}: entry #{hash.inspect} is not a Hash" unless hash.is_a?(Hash)
|
|
94
|
+
|
|
95
|
+
attrs = hash.transform_keys(&:to_sym)
|
|
96
|
+
locale = attrs.delete(:locale)
|
|
97
|
+
unless LOCALES.include?(locale.respond_to?(:to_sym) ? locale.to_sym : nil)
|
|
98
|
+
raise InvalidWordListError,
|
|
99
|
+
"#{EXTRA_SOURCE}: entry #{hash[:stem].inspect} needs a locale (#{LOCALES.join(' or ')})"
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
check_variants(attrs, EXTRA_SOURCE)
|
|
103
|
+
locale = locale.to_sym
|
|
104
|
+
{ locale: locale, path: EXTRA_SOURCE, prefixes: [], allow: [],
|
|
105
|
+
entries: [Entry.build(attrs, locale: locale, source: EXTRA_SOURCE)] }
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# +locale+ is nil for extra files, which must declare it.
|
|
110
|
+
def read_file(path, locale)
|
|
111
|
+
data = YAML.safe_load(File.read(path)) # rubocop:disable Style/YAMLFileRead
|
|
112
|
+
fail_in(path, 'root must be a Hash') unless data.is_a?(Hash)
|
|
113
|
+
unknown = data.keys - FILE_KEYS
|
|
114
|
+
fail_in(path, "unknown top-level keys #{unknown.inspect}") unless unknown.empty?
|
|
115
|
+
|
|
116
|
+
locale ||= file_locale(data['locale'], path)
|
|
117
|
+
{
|
|
118
|
+
locale: locale,
|
|
119
|
+
path: path,
|
|
120
|
+
prefixes: string_list(data['prefixes'], 'prefixes', path),
|
|
121
|
+
allow: string_list(data['allow'], 'allow', path),
|
|
122
|
+
entries: build_entries(data['words'], locale, path)
|
|
123
|
+
}
|
|
124
|
+
rescue Psych::Exception, SystemCallError => e
|
|
125
|
+
raise InvalidWordListError, "#{path}: #{e.message}"
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def file_locale(value, path)
|
|
129
|
+
locale = value.is_a?(String) ? value.to_sym : nil
|
|
130
|
+
return locale if LOCALES.include?(locale)
|
|
131
|
+
|
|
132
|
+
fail_in(path, "top-level locale must be one of #{LOCALES.inspect}, got #{value.inspect}")
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def string_list(value, key, path)
|
|
136
|
+
return [] if value.nil?
|
|
137
|
+
return value.map { |w| w.dup.freeze }.freeze if value.is_a?(Array) && value.all?(String)
|
|
138
|
+
|
|
139
|
+
fail_in(path, "#{key} must be an Array of strings")
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def build_entries(words, locale, path)
|
|
143
|
+
return [] if words.nil?
|
|
144
|
+
fail_in(path, 'words must be an Array') unless words.is_a?(Array)
|
|
145
|
+
|
|
146
|
+
words.map do |hash|
|
|
147
|
+
fail_in(path, "entry #{hash.inspect} must be a Hash") unless hash.is_a?(Hash)
|
|
148
|
+
check_variants(hash, path)
|
|
149
|
+
Entry.build(hash, locale: locale, source: path)
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def check_variants(hash, path)
|
|
154
|
+
key = ['variants', :variants].find { |k| hash.key?(k) }
|
|
155
|
+
return unless key
|
|
156
|
+
|
|
157
|
+
variants = hash[key]
|
|
158
|
+
return if variants.is_a?(Array) && variants.all? { |v| v.is_a?(String) && !v.strip.empty? }
|
|
159
|
+
|
|
160
|
+
fail_in(path, "entry #{(hash['stem'] || hash[:stem]).inspect}: variants must be an Array of non-blank strings")
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def fail_in(path, message)
|
|
164
|
+
raise InvalidWordListError, "#{path}: #{message}"
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
data/lib/swear_jar.rb
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'swear_jar/version'
|
|
4
|
+
require_relative 'swear_jar/errors'
|
|
5
|
+
require_relative 'swear_jar/match'
|
|
6
|
+
require_relative 'swear_jar/entry'
|
|
7
|
+
require_relative 'swear_jar/word_list'
|
|
8
|
+
require_relative 'swear_jar/fold_table'
|
|
9
|
+
require_relative 'swear_jar/normalizer'
|
|
10
|
+
require_relative 'swear_jar/pattern_builder'
|
|
11
|
+
require_relative 'swear_jar/skeleton'
|
|
12
|
+
require_relative 'swear_jar/slicer'
|
|
13
|
+
require_relative 'swear_jar/matcher'
|
|
14
|
+
require_relative 'swear_jar/detector'
|
|
15
|
+
|
|
16
|
+
module SwearJar
|
|
17
|
+
ROOT_PATH = File.expand_path('..', __dir__)
|
|
18
|
+
|
|
19
|
+
LOCALES = %i[pl en].freeze
|
|
20
|
+
SEVERITIES = %i[mild strong severe].freeze
|
|
21
|
+
CATEGORIES = %i[vulgar sexual insult slur religious].freeze
|
|
22
|
+
MODES = %i[words embedded].freeze
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
require_relative 'swear_jar/configuration'
|
|
26
|
+
|
|
27
|
+
require_relative 'swear_jar/railtie' if defined?(Rails::Railtie)
|