kotoshu 0.9.1 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms.rb +1 -0
- data/lib/kotoshu/dictionary/base.rb +32 -5
- data/lib/kotoshu/dictionary/custom.rb +3 -0
- data/lib/kotoshu/dictionary/hunspell.rb +5 -2
- data/lib/kotoshu/dictionary/plain_text.rb +2 -0
- data/lib/kotoshu/dictionary/unix_words.rb +2 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +25 -6
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +30 -57
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +1 -0
- data/lib/kotoshu/version.rb +1 -1
- metadata +3 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 44cb12b37d8bf72fa6febd503bdcd713aa569d64e4c648f02c422e4d362ab891
|
|
4
|
+
data.tar.gz: b42724adb7b58ca46c5494770cff1fb6c821b68272a545cd24e1f1ac152b3799
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 17486ce07cb10da6c26bfd643aa7fc0d7d2ec7fb28cb8a406a6e3eb0a3612931a1db33badc577624e9e566902ab3e3f4a104a1ef10c174b7629651c3aeb11522
|
|
7
|
+
data.tar.gz: 66f0dba2a4a727d700ba8c4c7d386380c2d77380b2b8cd7f4943dae377bff1db751bd1ca7f5dcb2075a65139a453ccdf4395c2331874d1b2e783fcd7400be53f
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Algorithms
|
|
5
|
+
# Soundex phonetic coding.
|
|
6
|
+
#
|
|
7
|
+
# Soundex is a phonetic algorithm developed by Robert C. Russell
|
|
8
|
+
# and Margaret King Odell in the early 1900s. A word maps to a
|
|
9
|
+
# four-character code (letter + 3 digits); words that sound alike
|
|
10
|
+
# share a code, which is what {Suggestions::Strategies::PhoneticStrategy}
|
|
11
|
+
# matches on.
|
|
12
|
+
#
|
|
13
|
+
# Coding rules (the gem's exact semantics — first letter verbatim,
|
|
14
|
+
# H/W never reset the previous code, non-ASCII stripped after
|
|
15
|
+
# upcase, letter-less words code to the empty string):
|
|
16
|
+
#
|
|
17
|
+
# - The first letter is kept verbatim and never coded.
|
|
18
|
+
# - B/P/F/V code 1, C/S/K/G/J/Q/X/Z code 2, D/T code 3, L codes 4,
|
|
19
|
+
# M/N code 5, R codes 6; vowels and H/W code 0 (skipped).
|
|
20
|
+
# H and W do NOT reset the previous code; every other coded 0
|
|
21
|
+
# letter does.
|
|
22
|
+
# - Adjacent letters sharing a code collapse to one digit.
|
|
23
|
+
# - The code is truncated to and zero-padded to exactly 4 chars.
|
|
24
|
+
#
|
|
25
|
+
# @example
|
|
26
|
+
# Soundex.code("Robert") # => "R163"
|
|
27
|
+
# Soundex.code("Rupert") # => "R163"
|
|
28
|
+
# Soundex.code("Ashcraft") # => "A261"
|
|
29
|
+
# Soundex.code("é") # => "" (no A-Z letters remain)
|
|
30
|
+
module Soundex
|
|
31
|
+
module_function
|
|
32
|
+
|
|
33
|
+
# Calculate the Soundex code for a word.
|
|
34
|
+
#
|
|
35
|
+
# @param word [String, nil] The word
|
|
36
|
+
# @return [String] The Soundex code (letter + 3 digits), or ""
|
|
37
|
+
# when the word has no A-Z letters
|
|
38
|
+
def code(word)
|
|
39
|
+
return "" if word.nil? || word.empty?
|
|
40
|
+
|
|
41
|
+
letters = word.upcase.gsub(/[^A-Z]/, "")
|
|
42
|
+
return "" if letters.empty?
|
|
43
|
+
|
|
44
|
+
# Keep first letter. The code is built into one mutable
|
|
45
|
+
# buffer (`<<`) instead of `code += digit`, which allocated
|
|
46
|
+
# a fresh String per encoded letter — the phonetic sweep
|
|
47
|
+
# pays this once per dictionary word.
|
|
48
|
+
code = +letters[0]
|
|
49
|
+
|
|
50
|
+
prev_code = encode(code)
|
|
51
|
+
i = 1
|
|
52
|
+
length = letters.length
|
|
53
|
+
|
|
54
|
+
while code.length < 4 && i < length
|
|
55
|
+
encoded = encode(letters[i])
|
|
56
|
+
|
|
57
|
+
# Add code if different from previous (ignore h and w)
|
|
58
|
+
code << encoded if encoded != "0" && encoded != prev_code
|
|
59
|
+
|
|
60
|
+
prev_code = encoded if encoded != "0"
|
|
61
|
+
i += 1
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Pad with zeros if needed
|
|
65
|
+
code.ljust(4, "0")[0...4]
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Soundex encoding table.
|
|
69
|
+
#
|
|
70
|
+
# @param char [String] The character
|
|
71
|
+
# @return [String] The encoded digit or "0" for no code
|
|
72
|
+
def encode(char)
|
|
73
|
+
case char.upcase
|
|
74
|
+
when "B", "P", "F", "V"
|
|
75
|
+
"1"
|
|
76
|
+
when "C", "S", "K", "G", "J", "Q", "X", "Z"
|
|
77
|
+
"2"
|
|
78
|
+
when "D", "T"
|
|
79
|
+
"3"
|
|
80
|
+
when "L"
|
|
81
|
+
"4"
|
|
82
|
+
when "M", "N"
|
|
83
|
+
"5"
|
|
84
|
+
when "R"
|
|
85
|
+
"6"
|
|
86
|
+
else
|
|
87
|
+
"0"
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
data/lib/kotoshu/algorithms.rb
CHANGED
|
@@ -18,5 +18,6 @@ module Kotoshu
|
|
|
18
18
|
autoload :PhonetSuggest, "kotoshu/algorithms/phonet_suggest"
|
|
19
19
|
autoload :Capitalization, "kotoshu/algorithms/capitalization"
|
|
20
20
|
autoload :EditDistance, "kotoshu/algorithms/edit_distance"
|
|
21
|
+
autoload :Soundex, "kotoshu/algorithms/soundex"
|
|
21
22
|
end
|
|
22
23
|
end
|
|
@@ -149,18 +149,45 @@ module Kotoshu
|
|
|
149
149
|
nil
|
|
150
150
|
end
|
|
151
151
|
|
|
152
|
+
# The sweep invariants for this dictionary's word list — char
|
|
153
|
+
# lengths, Soundex codes and length buckets — built once at the
|
|
154
|
+
# first suggestion sweep and shared by every later sweep (see
|
|
155
|
+
# {Suggestions::SweepIndex}).
|
|
156
|
+
#
|
|
157
|
+
# The index derives only from {#words}. Backends that mutate the
|
|
158
|
+
# word list (their {#add_word} / {#remove_word} implementations)
|
|
159
|
+
# call {#reset_sweep_index} so the memo cannot go stale; backends
|
|
160
|
+
# that cannot mutate after load (e.g. CSpell's frozen trie) never
|
|
161
|
+
# need to.
|
|
162
|
+
#
|
|
163
|
+
# @return [Suggestions::SweepIndex] The memoized sweep index
|
|
164
|
+
def sweep_index
|
|
165
|
+
@sweep_index ||= Suggestions::SweepIndex.build(words)
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# Drop the memoized {#sweep_index} so the next sweep rebuilds it
|
|
169
|
+
# from the current word list. Mutating backends call this after
|
|
170
|
+
# every successful word-list change.
|
|
171
|
+
#
|
|
172
|
+
# @return [self] Self for chaining
|
|
173
|
+
def reset_sweep_index
|
|
174
|
+
@sweep_index = nil
|
|
175
|
+
self
|
|
176
|
+
end
|
|
177
|
+
|
|
152
178
|
# Return words whose length is in [min_length, max_length].
|
|
153
179
|
#
|
|
154
|
-
#
|
|
155
|
-
#
|
|
156
|
-
#
|
|
157
|
-
#
|
|
180
|
+
# Uses the memoized {#sweep_index} length buckets — O(buckets in
|
|
181
|
+
# range) instead of an O(n) scan — and restores word-list order,
|
|
182
|
+
# exactly the set and order the historical whole-list filter
|
|
183
|
+
# produced. Subclasses that maintain their own length index
|
|
184
|
+
# (e.g. PlainText) keep their override.
|
|
158
185
|
#
|
|
159
186
|
# @param min_length [Integer] Minimum length (inclusive)
|
|
160
187
|
# @param max_length [Integer] Maximum length (inclusive)
|
|
161
188
|
# @return [Array<String>] Words in the length range
|
|
162
189
|
def find_by_length_range(min_length:, max_length:)
|
|
163
|
-
|
|
190
|
+
sweep_index.words_in_length_range(min_length, max_length)
|
|
164
191
|
end
|
|
165
192
|
|
|
166
193
|
# Get the number of words in the dictionary.
|
|
@@ -88,6 +88,7 @@ module Kotoshu
|
|
|
88
88
|
|
|
89
89
|
@words << lookup_word
|
|
90
90
|
@word_set[lookup_word] = true
|
|
91
|
+
reset_sweep_index
|
|
91
92
|
|
|
92
93
|
true
|
|
93
94
|
end
|
|
@@ -104,6 +105,7 @@ module Kotoshu
|
|
|
104
105
|
|
|
105
106
|
@word_set.delete(lookup_word)
|
|
106
107
|
@words.delete(lookup_word)
|
|
108
|
+
reset_sweep_index
|
|
107
109
|
|
|
108
110
|
true
|
|
109
111
|
end
|
|
@@ -121,6 +123,7 @@ module Kotoshu
|
|
|
121
123
|
def clear
|
|
122
124
|
@words.clear
|
|
123
125
|
@word_set.clear
|
|
126
|
+
reset_sweep_index
|
|
124
127
|
self
|
|
125
128
|
end
|
|
126
129
|
|
|
@@ -313,6 +313,7 @@ module Kotoshu
|
|
|
313
313
|
|
|
314
314
|
word_key = word.downcase
|
|
315
315
|
@word_index[word_key] = flags
|
|
316
|
+
reset_sweep_index
|
|
316
317
|
|
|
317
318
|
true
|
|
318
319
|
end
|
|
@@ -320,12 +321,14 @@ module Kotoshu
|
|
|
320
321
|
# Remove a word from the dictionary.
|
|
321
322
|
#
|
|
322
323
|
# @param word [String] The word to remove
|
|
323
|
-
# @return [Boolean] True if removed
|
|
324
|
+
# @return [Boolean] True if the word was removed
|
|
324
325
|
def remove_word(word)
|
|
325
326
|
return false if word.nil? || word.empty?
|
|
326
327
|
|
|
327
328
|
word_key = word.downcase
|
|
328
|
-
|
|
329
|
+
removed = @word_index.delete(word_key)
|
|
330
|
+
reset_sweep_index unless removed.nil?
|
|
331
|
+
!removed.nil?
|
|
329
332
|
end
|
|
330
333
|
|
|
331
334
|
# Get all words in the dictionary.
|
|
@@ -114,6 +114,7 @@ module Kotoshu
|
|
|
114
114
|
@words << lookup_word
|
|
115
115
|
@word_set[lookup_word] = true
|
|
116
116
|
(@length_index[lookup_word.length] ||= []) << lookup_word
|
|
117
|
+
reset_sweep_index
|
|
117
118
|
|
|
118
119
|
true
|
|
119
120
|
end
|
|
@@ -131,6 +132,7 @@ module Kotoshu
|
|
|
131
132
|
@word_set.delete(lookup_word)
|
|
132
133
|
@words.delete(lookup_word)
|
|
133
134
|
@length_index[lookup_word.length]&.delete(lookup_word)
|
|
135
|
+
reset_sweep_index
|
|
134
136
|
|
|
135
137
|
true
|
|
136
138
|
end
|
|
@@ -104,6 +104,7 @@ module Kotoshu
|
|
|
104
104
|
|
|
105
105
|
@words << lookup_word
|
|
106
106
|
@word_set[lookup_word] = true
|
|
107
|
+
reset_sweep_index
|
|
107
108
|
|
|
108
109
|
true
|
|
109
110
|
end
|
|
@@ -120,6 +121,7 @@ module Kotoshu
|
|
|
120
121
|
|
|
121
122
|
@word_set.delete(lookup_word)
|
|
122
123
|
@words.delete(lookup_word)
|
|
124
|
+
reset_sweep_index
|
|
123
125
|
|
|
124
126
|
true
|
|
125
127
|
end
|
|
@@ -35,17 +35,15 @@ module Kotoshu
|
|
|
35
35
|
|
|
36
36
|
return create_suggestion_set([]) if word.length < n
|
|
37
37
|
|
|
38
|
-
all_words = dictionary_words(context)
|
|
39
|
-
|
|
40
38
|
# Get n-grams for input word
|
|
41
39
|
word_ngrams = extract_ngrams(word, n)
|
|
42
40
|
word_length = word.length
|
|
43
41
|
|
|
44
42
|
# Calculate n-gram similarity for each dictionary word
|
|
45
43
|
results = {}
|
|
46
|
-
|
|
44
|
+
each_word_with_length(context) do |dict_word, dict_length|
|
|
47
45
|
next if dict_word == word
|
|
48
|
-
next if
|
|
46
|
+
next if dict_length < n
|
|
49
47
|
|
|
50
48
|
# Length pre-gate: similarity is a Jaccard coefficient
|
|
51
49
|
# over n-gram multisets, so it is at most
|
|
@@ -54,8 +52,9 @@ module Kotoshu
|
|
|
54
52
|
# already sits under min_similarity can never pass the
|
|
55
53
|
# real test, so skip it before extracting a single
|
|
56
54
|
# n-gram. Mathematically the same set of words passes
|
|
57
|
-
# (same gate as ngram.rs in kotoshu-rs).
|
|
58
|
-
|
|
55
|
+
# (same gate as ngram.rs in kotoshu-rs). The length is
|
|
56
|
+
# read from the dictionary's memoized sweep index before
|
|
57
|
+
# any per-word work when the backend maintains one.
|
|
59
58
|
min_grams = (dict_length < word_length ? dict_length : word_length) - (n - 1)
|
|
60
59
|
max_grams = (dict_length > word_length ? dict_length : word_length) - (n - 1)
|
|
61
60
|
next if min_grams.to_f / max_grams < min_sim
|
|
@@ -92,6 +91,26 @@ module Kotoshu
|
|
|
92
91
|
|
|
93
92
|
private
|
|
94
93
|
|
|
94
|
+
# Yield each dictionary word paired with its char length, in
|
|
95
|
+
# word-list order. Over a Dictionary::Base backend the pairs
|
|
96
|
+
# come from the dictionary's memoized sweep index
|
|
97
|
+
# ({Dictionary::Base#sweep_index}) — the length gate reads the
|
|
98
|
+
# indexed value before any per-word work, and the sweep never
|
|
99
|
+
# copies the word list. Ad-hoc Hash/Array dictionaries compute
|
|
100
|
+
# the length live, exactly as before.
|
|
101
|
+
#
|
|
102
|
+
# @param context [Context] The suggestion context
|
|
103
|
+
# @yield [String, Integer] Each word and its char length
|
|
104
|
+
# @return [void]
|
|
105
|
+
def each_word_with_length(context)
|
|
106
|
+
dictionary = context.dictionary
|
|
107
|
+
if dictionary.is_a?(Kotoshu::Dictionary::Base)
|
|
108
|
+
dictionary.sweep_index.each_with_length { |*pair| yield(*pair) }
|
|
109
|
+
else
|
|
110
|
+
dictionary_words(context).each { |dict_word| yield dict_word, dict_word.length }
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
95
114
|
# Extract n-grams from a word.
|
|
96
115
|
#
|
|
97
116
|
# @param word [String] The word
|
|
@@ -34,17 +34,13 @@ module Kotoshu
|
|
|
34
34
|
algorithm = get_config(:algorithm, :soundex)
|
|
35
35
|
max_dist = 2
|
|
36
36
|
|
|
37
|
-
all_words = dictionary_words(context)
|
|
38
|
-
|
|
39
37
|
# Get phonetic code for input word
|
|
40
38
|
word_code = phonetic_code(word, algorithm)
|
|
41
39
|
|
|
42
40
|
# Find words with same phonetic code
|
|
43
41
|
results = []
|
|
44
|
-
|
|
42
|
+
each_phonetic_pair(context, algorithm) do |dict_word, dict_code|
|
|
45
43
|
next if dict_word == word
|
|
46
|
-
|
|
47
|
-
dict_code = phonetic_code(dict_word, algorithm)
|
|
48
44
|
next unless dict_code == word_code
|
|
49
45
|
|
|
50
46
|
dist = edit_distance(word, dict_word)
|
|
@@ -70,6 +66,29 @@ module Kotoshu
|
|
|
70
66
|
|
|
71
67
|
private
|
|
72
68
|
|
|
69
|
+
# Yield each dictionary word paired with its phonetic code, in
|
|
70
|
+
# word-list order. For Soundex over a Dictionary::Base backend
|
|
71
|
+
# the codes come from the dictionary's memoized sweep index
|
|
72
|
+
# ({Dictionary::Base#sweep_index}) — one Soundex per word for
|
|
73
|
+
# the dictionary's lifetime instead of one per sweep. Every
|
|
74
|
+
# other combination (ad-hoc Hash/Array dictionaries, Metaphone)
|
|
75
|
+
# computes each code live, exactly as before.
|
|
76
|
+
#
|
|
77
|
+
# @param context [Context] The suggestion context
|
|
78
|
+
# @param algorithm [Symbol] The phonetic algorithm
|
|
79
|
+
# @yield [String, String] Each word and its phonetic code
|
|
80
|
+
# @return [void]
|
|
81
|
+
def each_phonetic_pair(context, algorithm)
|
|
82
|
+
dictionary = context.dictionary
|
|
83
|
+
if algorithm == :soundex && dictionary.is_a?(Kotoshu::Dictionary::Base)
|
|
84
|
+
dictionary.sweep_index.each_with_soundex { |*pair| yield(*pair) }
|
|
85
|
+
else
|
|
86
|
+
dictionary_words(context).each do |dict_word|
|
|
87
|
+
yield dict_word, phonetic_code(dict_word, algorithm)
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
73
92
|
# Get phonetic code for a word.
|
|
74
93
|
#
|
|
75
94
|
# @param word [String] The word
|
|
@@ -89,7 +108,10 @@ module Kotoshu
|
|
|
89
108
|
# Calculate Soundex code for a word.
|
|
90
109
|
#
|
|
91
110
|
# Soundex is a phonetic algorithm developed by Robert C. Russell
|
|
92
|
-
# and Margaret King Odell in the early 1900s.
|
|
111
|
+
# and Margaret King Odell in the early 1900s. The implementation
|
|
112
|
+
# lives in {Algorithms::Soundex} and is shared with
|
|
113
|
+
# {Suggestions::SweepIndex}, which memoizes the code per
|
|
114
|
+
# dictionary word.
|
|
93
115
|
#
|
|
94
116
|
# @param word [String] The word
|
|
95
117
|
# @return [String] The Soundex code (letter + 3 digits)
|
|
@@ -97,58 +119,9 @@ module Kotoshu
|
|
|
97
119
|
# @example
|
|
98
120
|
# soundex_code("Robert") # => "R163"
|
|
99
121
|
# soundex_code("Rupert") # => "R163"
|
|
100
|
-
# soundex_code("Ashcraft") # => "
|
|
122
|
+
# soundex_code("Ashcraft") # => "A261"
|
|
101
123
|
def soundex_code(word)
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
letters = word.upcase.gsub(/[^A-Z]/, "")
|
|
105
|
-
return "" if letters.empty?
|
|
106
|
-
|
|
107
|
-
# Keep first letter. The code is built into one mutable
|
|
108
|
-
# buffer (`<<`) instead of `code += digit`, which allocated
|
|
109
|
-
# a fresh String per encoded letter — the phonetic sweep
|
|
110
|
-
# pays this once per dictionary word.
|
|
111
|
-
code = +letters[0]
|
|
112
|
-
|
|
113
|
-
prev_code = soundex_encode(code)
|
|
114
|
-
i = 1
|
|
115
|
-
length = letters.length
|
|
116
|
-
|
|
117
|
-
while code.length < 4 && i < length
|
|
118
|
-
encoded = soundex_encode(letters[i])
|
|
119
|
-
|
|
120
|
-
# Add code if different from previous (ignore h and w)
|
|
121
|
-
code << encoded if encoded != "0" && encoded != prev_code
|
|
122
|
-
|
|
123
|
-
prev_code = encoded if encoded != "0"
|
|
124
|
-
i += 1
|
|
125
|
-
end
|
|
126
|
-
|
|
127
|
-
# Pad with zeros if needed
|
|
128
|
-
code.ljust(4, "0")[0...4]
|
|
129
|
-
end
|
|
130
|
-
|
|
131
|
-
# Soundex encoding table.
|
|
132
|
-
#
|
|
133
|
-
# @param char [String] The character
|
|
134
|
-
# @return [String] The encoded digit or "0" for no code
|
|
135
|
-
def soundex_encode(char)
|
|
136
|
-
case char.upcase
|
|
137
|
-
when "B", "P", "F", "V"
|
|
138
|
-
"1"
|
|
139
|
-
when "C", "S", "K", "G", "J", "Q", "X", "Z"
|
|
140
|
-
"2"
|
|
141
|
-
when "D", "T"
|
|
142
|
-
"3"
|
|
143
|
-
when "L"
|
|
144
|
-
"4"
|
|
145
|
-
when "M", "N"
|
|
146
|
-
"5"
|
|
147
|
-
when "R"
|
|
148
|
-
"6"
|
|
149
|
-
else
|
|
150
|
-
"0"
|
|
151
|
-
end
|
|
124
|
+
Algorithms::Soundex.code(word)
|
|
152
125
|
end
|
|
153
126
|
|
|
154
127
|
# Calculate Metaphone code for a word.
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Suggestions
|
|
5
|
+
# Per-dictionary sweep invariants, built once at the first
|
|
6
|
+
# suggestion sweep and kept for the dictionary's lifetime.
|
|
7
|
+
#
|
|
8
|
+
# Every strategy used to walk the whole word list per sweep and
|
|
9
|
+
# compute per-word values before its gate — the length window, the
|
|
10
|
+
# Soundex code, the n-gram length bound — so a full dictionary paid
|
|
11
|
+
# for work on words a sweep was about to skip. Those per-word
|
|
12
|
+
# values never change with the query: they are computed here one
|
|
13
|
+
# time (char lengths, packed Soundex codes, length buckets), and
|
|
14
|
+
# the sweeps touch only the entries their gates admit.
|
|
15
|
+
#
|
|
16
|
+
# Soundex codes are the 4-character {Algorithms::Soundex} strings
|
|
17
|
+
# (frozen); two letter-less words both code to +""+ and compare
|
|
18
|
+
# equal, exactly as the live computation does. Lengths are char
|
|
19
|
+
# counts. +by_length+ groups word indices by length in original
|
|
20
|
+
# order so the edit-distance sweep can iterate only its ±2 window;
|
|
21
|
+
# {#indices_in_length_range} restores word-list order because ties
|
|
22
|
+
# in the strategies' ranking sorts are decided by input order.
|
|
23
|
+
#
|
|
24
|
+
# The index derives only from the word list it was built over.
|
|
25
|
+
# Dictionary backends memoize it (see
|
|
26
|
+
# {Dictionary::Base#sweep_index}) and invalidate the memo wherever
|
|
27
|
+
# they mutate the word list ({Dictionary::Base#add_word} /
|
|
28
|
+
# {Dictionary::Base#remove_word} implementations).
|
|
29
|
+
#
|
|
30
|
+
# This mirrors SweepIndex in kotoshu-rs
|
|
31
|
+
# (kotoshu/src/suggest/sweep_index.rs).
|
|
32
|
+
class SweepIndex
|
|
33
|
+
# @return [Array<String>] The words in list order (as captured at
|
|
34
|
+
# build time)
|
|
35
|
+
attr_reader :words
|
|
36
|
+
|
|
37
|
+
# Build an index over a word list.
|
|
38
|
+
#
|
|
39
|
+
# @param words [Array<String>] Dictionary words in list order
|
|
40
|
+
# @return [SweepIndex] The built index
|
|
41
|
+
def self.build(words)
|
|
42
|
+
lengths = Array.new(words.length)
|
|
43
|
+
soundex_codes = Array.new(words.length)
|
|
44
|
+
by_length = {}
|
|
45
|
+
words.each_with_index do |word, idx|
|
|
46
|
+
length = word.length
|
|
47
|
+
lengths[idx] = length
|
|
48
|
+
soundex_codes[idx] = Algorithms::Soundex.code(word).freeze
|
|
49
|
+
(by_length[length] ||= []) << idx
|
|
50
|
+
end
|
|
51
|
+
new(words, lengths, soundex_codes, by_length)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Create an index from prebuilt structures.
|
|
55
|
+
#
|
|
56
|
+
# @param words [Array<String>] The indexed words in list order
|
|
57
|
+
# @param lengths [Array<Integer>] Char length per word index
|
|
58
|
+
# @param soundex_codes [Array<String>] Soundex code per word index
|
|
59
|
+
# @param by_length [Hash<Integer, Array<Integer>>] Length => word
|
|
60
|
+
# indices in list order
|
|
61
|
+
def initialize(words, lengths, soundex_codes, by_length)
|
|
62
|
+
@words = words
|
|
63
|
+
@lengths = lengths
|
|
64
|
+
@soundex_codes = soundex_codes
|
|
65
|
+
@by_length = by_length
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Char length of the word at +idx+.
|
|
69
|
+
#
|
|
70
|
+
# @param idx [Integer] Word index
|
|
71
|
+
# @return [Integer] Char count
|
|
72
|
+
def length(idx)
|
|
73
|
+
@lengths[idx]
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Soundex code of the word at +idx+ — equal codes compare equal,
|
|
77
|
+
# including two letter-less empty codes.
|
|
78
|
+
#
|
|
79
|
+
# @param idx [Integer] Word index
|
|
80
|
+
# @return [String] The frozen 4-char (or empty) Soundex code
|
|
81
|
+
def soundex(idx)
|
|
82
|
+
@soundex_codes[idx]
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Word indices whose length is within +min..max+, every word in
|
|
86
|
+
# exactly one bucket, each bucket in original order.
|
|
87
|
+
#
|
|
88
|
+
# @param min [Integer] Minimum length (inclusive)
|
|
89
|
+
# @param max [Integer] Maximum length (inclusive)
|
|
90
|
+
# @return [Array<Integer>] Matching indices in word-list order
|
|
91
|
+
def indices_in_length_range(min, max)
|
|
92
|
+
hits = []
|
|
93
|
+
(min..max).each do |length|
|
|
94
|
+
bucket = @by_length[length]
|
|
95
|
+
hits.concat(bucket) if bucket
|
|
96
|
+
end
|
|
97
|
+
# Restore the word-list order the strategies' candidate arrays
|
|
98
|
+
# were built in before this index existed — ties in the ranking
|
|
99
|
+
# sort are decided by input order, so this must not drift.
|
|
100
|
+
hits.sort!
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Words whose length is within +min..max+, in word-list order —
|
|
104
|
+
# exactly the set and order {Dictionary::Base#find_by_length_range}
|
|
105
|
+
# historically returned.
|
|
106
|
+
#
|
|
107
|
+
# @param min [Integer] Minimum length (inclusive)
|
|
108
|
+
# @param max [Integer] Maximum length (inclusive)
|
|
109
|
+
# @return [Array<String>] Matching words in word-list order
|
|
110
|
+
def words_in_length_range(min, max)
|
|
111
|
+
indices_in_length_range(min, max).map { |idx| @words[idx] }
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Yield every word with its memoized char length.
|
|
115
|
+
#
|
|
116
|
+
# @yield [String, Integer] Each word and its char count
|
|
117
|
+
# @return [void]
|
|
118
|
+
def each_with_length
|
|
119
|
+
@words.each_with_index do |word, idx|
|
|
120
|
+
yield word, @lengths[idx]
|
|
121
|
+
end
|
|
122
|
+
nil
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Yield every word with its memoized Soundex code.
|
|
126
|
+
#
|
|
127
|
+
# @yield [String, String] Each word and its Soundex code
|
|
128
|
+
# @return [void]
|
|
129
|
+
def each_with_soundex
|
|
130
|
+
@words.each_with_index do |word, idx|
|
|
131
|
+
yield word, @soundex_codes[idx]
|
|
132
|
+
end
|
|
133
|
+
nil
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
data/lib/kotoshu/suggestions.rb
CHANGED
|
@@ -10,6 +10,7 @@ module Kotoshu
|
|
|
10
10
|
autoload :SemanticCascade, "kotoshu/suggestions/semantic_cascade"
|
|
11
11
|
autoload :Suggestion, "kotoshu/suggestions/suggestion"
|
|
12
12
|
autoload :SuggestionSet, "kotoshu/suggestions/suggestion_set"
|
|
13
|
+
autoload :SweepIndex, "kotoshu/suggestions/sweep_index"
|
|
13
14
|
|
|
14
15
|
# Strategies sub-namespace.
|
|
15
16
|
module Strategies
|
data/lib/kotoshu/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: kotoshu
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.9.
|
|
4
|
+
version: 0.9.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
@@ -204,6 +204,7 @@ files:
|
|
|
204
204
|
- lib/kotoshu/algorithms/ngram_suggest.rb
|
|
205
205
|
- lib/kotoshu/algorithms/permutations.rb
|
|
206
206
|
- lib/kotoshu/algorithms/phonet_suggest.rb
|
|
207
|
+
- lib/kotoshu/algorithms/soundex.rb
|
|
207
208
|
- lib/kotoshu/algorithms/suggest.rb
|
|
208
209
|
- lib/kotoshu/analyzers.rb
|
|
209
210
|
- lib/kotoshu/analyzers/semantic_analyzer.rb
|
|
@@ -457,6 +458,7 @@ files:
|
|
|
457
458
|
- lib/kotoshu/suggestions/strategies/word_index.rb
|
|
458
459
|
- lib/kotoshu/suggestions/suggestion.rb
|
|
459
460
|
- lib/kotoshu/suggestions/suggestion_set.rb
|
|
461
|
+
- lib/kotoshu/suggestions/sweep_index.rb
|
|
460
462
|
- lib/kotoshu/tasks.rb
|
|
461
463
|
- lib/kotoshu/tasks/check_task.rb
|
|
462
464
|
- lib/kotoshu/validators.rb
|