kotoshu 1.0.7-aarch64-linux → 1.0.8-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 51aa9c31d7eedc5d4e4bcae93ff0cf3c6d10dfd7f350b35ce6db91c89a10851b
4
- data.tar.gz: c064ad3481bb49e4df500845c099ff01c86aa8359e9a000ec0ca4eb31cb6a4b8
3
+ metadata.gz: 752d8003d8ca8648a08115886d1ccf9300adf15c932364689dcc2659c438734f
4
+ data.tar.gz: 051fbbe3a29865a6846d8704bc0bfc6a933f1f216d34191a5df99316b288fca4
5
5
  SHA512:
6
- metadata.gz: f7b9b6f2eb04f15fe6079012d704bf4b76bc8f4bc56decff0939e28b6104e757bd16044bf712aaffe17717a2e57b0ab66f02662d3b2ef15cf87e8ddf6f3dcee5
7
- data.tar.gz: 47604780a8e9d39ac681da6c613bb068a4f0e5bdfcd5d061662288f8601faad64e9fb5f7fb7ac2635e538260230aebb722b549b38ca8963d4fd8d4460e3dc4c1
6
+ metadata.gz: e1b4007839cb2d33e0aca19016128e20fb1eeb8ff8c9926b71fda5a54b44711c62145eed2eec2a85725f0aa4b3aa880bc09b2b878209b58de5c5e1407b307e2d
7
+ data.tar.gz: 87a5eca886a5c52d6b0369d7f0db3789e158859028b996b82a7fccf668dcf52df2ae6e87aff2768999599a0ae1cd43ecc06d79a4a5feb6d99835d0fdf67286f0
@@ -146,13 +146,19 @@ module Kotoshu
146
146
  # CommonWordsLoader autoloaded via Kotoshu::Data
147
147
  data = Data::CommonWordsLoader.load_from_frequency_file(frequency_file)
148
148
 
149
- {
149
+ payload = {
150
150
  frequency_path: frequency_file,
151
151
  tiers: data[:tiers],
152
152
  metadata: metadata,
153
153
  full_list: data[:full_list] || [],
154
154
  ranks: data[:ranks] || {}
155
155
  }
156
+ # Optional transliteration sidecar ({lang}/translit.json — the
157
+ # kelly repo's romanization retrieval keys): fold-normalized
158
+ # Latin key -> native words. Absent for most languages.
159
+ translit_file = File.join(language_dir(language_code), "translit.json")
160
+ payload[:translit] = JSON.parse(File.read(translit_file)) if File.exist?(translit_file)
161
+ payload
156
162
  end
157
163
 
158
164
  protected
@@ -196,6 +202,19 @@ module Kotoshu
196
202
  }
197
203
  write_metadata(metadata_path, metadata)
198
204
 
205
+ # Best-effort romanization sidecar (the kelly repo's
206
+ # {lang}.translit.json): consumed by the cross-script
207
+ # suggestion channel. Absent for most languages — a 404 is
208
+ # not an error, the channel simply stays inert.
209
+ begin
210
+ translit_url = "https://raw.githubusercontent.com/#{GITHUB_REPO}/#{GITHUB_BRANCH}/data/#{language_code}.translit.json"
211
+ translit_response = download_url(translit_url)
212
+ JSON.parse(translit_response)
213
+ File.binwrite(File.join(dest_path, "translit.json"), translit_response)
214
+ rescue StandardError
215
+ nil
216
+ end
217
+
199
218
  # Load and return the data
200
219
  load_cached(language_code)
201
220
  end
@@ -58,6 +58,12 @@ module Kotoshu
58
58
  data_for(language_code)[:full_list]
59
59
  end
60
60
 
61
+ # Optional romanization sidecar (kelly {lang}.translit.json):
62
+ # fold-normalized Latin key -> native words. Empty when absent.
63
+ def translit_for(language_code)
64
+ data_for(language_code)[:translit] || {}
65
+ end
66
+
61
67
  # word.downcase => rank (1 = most frequent). Empty when unknown.
62
68
  #
63
69
  # @param language_code [String]
@@ -107,7 +113,8 @@ module Kotoshu
107
113
  return {
108
114
  tiers: cache_result[:tiers],
109
115
  full_list: cache_result[:full_list] || [],
110
- ranks: cache_result[:ranks] || {}
116
+ ranks: cache_result[:ranks] || {},
117
+ translit: cache_result[:translit] || {}
111
118
  }
112
119
  end
113
120
 
@@ -103,6 +103,25 @@ module Kotoshu
103
103
 
104
104
  candidates.delete(word_lower)
105
105
 
106
+ # Cross-script retrieval (the romanization sidecar): a
107
+ # Latin-typed query on a non-Latin language reaches native
108
+ # words through fold-normalized romanization keys (mrhb ->
109
+ # مرحبا). Exact key ranks as a strong match (1); its
110
+ # single-deletion neighborhood (mrhaba -> mrhb) as 2. No
111
+ # frequency-list baseline can do this at all.
112
+ translit_hits = {}
113
+ if @translit && !@translit.empty? &&
114
+ word_lower.ascii_only? && word_lower.match?(/[a-z]/)
115
+ (@translit[word_lower] || []).each { |w| translit_hits[w] ||= 1 }
116
+ word_lower.length.times do |i|
117
+ v = word_lower[0...i] + word_lower[(i + 1)..]
118
+ next if v.empty?
119
+
120
+ (@translit[v] || []).each { |w| translit_hits[w] ||= 2 }
121
+ end
122
+ translit_hits.each_key { |w| candidates.add(w) }
123
+ end
124
+
106
125
  # TRUE edit distance per candidate (with early exit) — the
107
126
  # deletion-level approximation misranked same-distance words
108
127
  # and cost 18pp of English top-1 (plan C6/C9). Sort by
@@ -125,7 +144,9 @@ module Kotoshu
125
144
  # ranking distance switches.
126
145
  typed_fold = fold_word(word_lower)
127
146
  scored = candidates.filter_map do |cand|
128
- dist = if fold_scoring?
147
+ dist = if (td = translit_hits[cand])
148
+ td
149
+ elsif fold_scoring?
129
150
  cand_fold = @folded_words&.[](cand) || fold_word(cand)
130
151
  folded_distance(typed_fold, cand_fold, max_dist + 1)
131
152
  else
@@ -192,6 +213,7 @@ module Kotoshu
192
213
  if freq_words && !freq_words.empty?
193
214
  @dictionary = freq_words
194
215
  @ranks = @frequency_provider.ranks_for(@language_code)
216
+ @translit = @frequency_provider.translit_for(@language_code)
195
217
  elsif @dictionary.nil?
196
218
  @dictionary = context.dictionary
197
219
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Kotoshu
4
- VERSION = '1.0.7'
4
+ VERSION = '1.0.8'
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: kotoshu
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.7
4
+ version: 1.0.8
5
5
  platform: aarch64-linux
6
6
  authors:
7
7
  - Ribose Inc.