kotoshu 1.0.7 → 1.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2e0e6dc04a08737e455f97e405334cc8f4f620b94241fffd902c6b6e3c5ca695
|
|
4
|
+
data.tar.gz: 413bf5bde3121c709fb94ab345bafcef58880ae2c1b408275ba4a6e8d3d206f8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 0b329177f96ed12abe9c34f4f2186a9c98511aa67e6d9904bba16f66ae321620a91fd93580e2267cc0c17fa2ea3dddcb48f467aaf0e645107aa6137963b11213
|
|
7
|
+
data.tar.gz: 7c32da93203d90e90eda84b62900f7d2b1cb3f21feda9a8b8d89b7f8ab561e119e046054f25b27a02fc6775896efaf1e5b550cc51d50efe46ee6dc80608996e7
|
|
@@ -146,13 +146,19 @@ module Kotoshu
|
|
|
146
146
|
# CommonWordsLoader autoloaded via Kotoshu::Data
|
|
147
147
|
data = Data::CommonWordsLoader.load_from_frequency_file(frequency_file)
|
|
148
148
|
|
|
149
|
-
{
|
|
149
|
+
payload = {
|
|
150
150
|
frequency_path: frequency_file,
|
|
151
151
|
tiers: data[:tiers],
|
|
152
152
|
metadata: metadata,
|
|
153
153
|
full_list: data[:full_list] || [],
|
|
154
154
|
ranks: data[:ranks] || {}
|
|
155
155
|
}
|
|
156
|
+
# Optional transliteration sidecar ({lang}/translit.json — the
|
|
157
|
+
# kelly repo's romanization retrieval keys): fold-normalized
|
|
158
|
+
# Latin key -> native words. Absent for most languages.
|
|
159
|
+
translit_file = File.join(language_dir(language_code), "translit.json")
|
|
160
|
+
payload[:translit] = JSON.parse(File.read(translit_file)) if File.exist?(translit_file)
|
|
161
|
+
payload
|
|
156
162
|
end
|
|
157
163
|
|
|
158
164
|
protected
|
|
@@ -196,6 +202,19 @@ module Kotoshu
|
|
|
196
202
|
}
|
|
197
203
|
write_metadata(metadata_path, metadata)
|
|
198
204
|
|
|
205
|
+
# Best-effort romanization sidecar (the kelly repo's
|
|
206
|
+
# {lang}.translit.json): consumed by the cross-script
|
|
207
|
+
# suggestion channel. Absent for most languages — a 404 is
|
|
208
|
+
# not an error, the channel simply stays inert.
|
|
209
|
+
begin
|
|
210
|
+
translit_url = "https://raw.githubusercontent.com/#{GITHUB_REPO}/#{GITHUB_BRANCH}/data/#{language_code}.translit.json"
|
|
211
|
+
translit_response = download_url(translit_url)
|
|
212
|
+
JSON.parse(translit_response)
|
|
213
|
+
File.binwrite(File.join(dest_path, "translit.json"), translit_response)
|
|
214
|
+
rescue StandardError
|
|
215
|
+
nil
|
|
216
|
+
end
|
|
217
|
+
|
|
199
218
|
# Load and return the data
|
|
200
219
|
load_cached(language_code)
|
|
201
220
|
end
|
|
@@ -58,6 +58,12 @@ module Kotoshu
|
|
|
58
58
|
data_for(language_code)[:full_list]
|
|
59
59
|
end
|
|
60
60
|
|
|
61
|
+
# Optional romanization sidecar (kelly {lang}.translit.json):
|
|
62
|
+
# fold-normalized Latin key -> native words. Empty when absent.
|
|
63
|
+
def translit_for(language_code)
|
|
64
|
+
data_for(language_code)[:translit] || {}
|
|
65
|
+
end
|
|
66
|
+
|
|
61
67
|
# word.downcase => rank (1 = most frequent). Empty when unknown.
|
|
62
68
|
#
|
|
63
69
|
# @param language_code [String]
|
|
@@ -107,7 +113,8 @@ module Kotoshu
|
|
|
107
113
|
return {
|
|
108
114
|
tiers: cache_result[:tiers],
|
|
109
115
|
full_list: cache_result[:full_list] || [],
|
|
110
|
-
ranks: cache_result[:ranks] || {}
|
|
116
|
+
ranks: cache_result[:ranks] || {},
|
|
117
|
+
translit: cache_result[:translit] || {}
|
|
111
118
|
}
|
|
112
119
|
end
|
|
113
120
|
|
|
@@ -103,6 +103,25 @@ module Kotoshu
|
|
|
103
103
|
|
|
104
104
|
candidates.delete(word_lower)
|
|
105
105
|
|
|
106
|
+
# Cross-script retrieval (the romanization sidecar): a
|
|
107
|
+
# Latin-typed query on a non-Latin language reaches native
|
|
108
|
+
# words through fold-normalized romanization keys (mrhb ->
|
|
109
|
+
# مرحبا). Exact key ranks as a strong match (1); its
|
|
110
|
+
# single-deletion neighborhood (mrhaba -> mrhb) as 2. No
|
|
111
|
+
# frequency-list baseline can do this at all.
|
|
112
|
+
translit_hits = {}
|
|
113
|
+
if @translit && !@translit.empty? &&
|
|
114
|
+
word_lower.ascii_only? && word_lower.match?(/[a-z]/)
|
|
115
|
+
(@translit[word_lower] || []).each { |w| translit_hits[w] ||= 1 }
|
|
116
|
+
word_lower.length.times do |i|
|
|
117
|
+
v = word_lower[0...i] + word_lower[(i + 1)..]
|
|
118
|
+
next if v.empty?
|
|
119
|
+
|
|
120
|
+
(@translit[v] || []).each { |w| translit_hits[w] ||= 2 }
|
|
121
|
+
end
|
|
122
|
+
translit_hits.each_key { |w| candidates.add(w) }
|
|
123
|
+
end
|
|
124
|
+
|
|
106
125
|
# TRUE edit distance per candidate (with early exit) — the
|
|
107
126
|
# deletion-level approximation misranked same-distance words
|
|
108
127
|
# and cost 18pp of English top-1 (plan C6/C9). Sort by
|
|
@@ -125,7 +144,9 @@ module Kotoshu
|
|
|
125
144
|
# ranking distance switches.
|
|
126
145
|
typed_fold = fold_word(word_lower)
|
|
127
146
|
scored = candidates.filter_map do |cand|
|
|
128
|
-
dist = if
|
|
147
|
+
dist = if (td = translit_hits[cand])
|
|
148
|
+
td
|
|
149
|
+
elsif fold_scoring?
|
|
129
150
|
cand_fold = @folded_words&.[](cand) || fold_word(cand)
|
|
130
151
|
folded_distance(typed_fold, cand_fold, max_dist + 1)
|
|
131
152
|
else
|
|
@@ -192,6 +213,7 @@ module Kotoshu
|
|
|
192
213
|
if freq_words && !freq_words.empty?
|
|
193
214
|
@dictionary = freq_words
|
|
194
215
|
@ranks = @frequency_provider.ranks_for(@language_code)
|
|
216
|
+
@translit = @frequency_provider.translit_for(@language_code)
|
|
195
217
|
elsif @dictionary.nil?
|
|
196
218
|
@dictionary = context.dictionary
|
|
197
219
|
end
|
data/lib/kotoshu/version.rb
CHANGED