kabosu 0.6.10.2-aarch64-linux → 0.6.11.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/kabosu/3.1/kabosu.so +0 -0
- data/lib/kabosu/3.2/kabosu.so +0 -0
- data/lib/kabosu/3.3/kabosu.so +0 -0
- data/lib/kabosu/3.4/kabosu.so +0 -0
- data/lib/kabosu/4.0/kabosu.so +0 -0
- data/lib/kabosu/dict_manager.rb +102 -24
- data/lib/kabosu/morpheme_list.rb +5 -5
- data/lib/kabosu/pos_matcher.rb +7 -4
- data/lib/kabosu/tasks.rake +15 -16
- data/lib/kabosu/version.rb +1 -1
- data/lib/kabosu.rb +28 -66
- metadata +32 -17
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6310222bef7890fd2ae5882d39f312a656d2bbfbb115d41a9dff80ac94b11932
|
|
4
|
+
data.tar.gz: f6d831203d5a58c208f7fee1b21f0036b9d0c344fc16652f8cf1e5fe7ef38e46
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f945a86f63e3694fb8bb4a199dbb83470977ed77f1e0d0ae41f66482986e9ff4d9acf148affb77e7e7a4487f512924f54884f92d0155d85cb0787494dd9d825f
|
|
7
|
+
data.tar.gz: 3a85e6084607c95ff81d0a1807813c625b1a4d9d84ae28c21626ae97d8b501b1a38131306cd38b8cd2939d1fb0e8acbe763cb415c9cb7fc80c44d5a027b775fc
|
data/lib/kabosu/3.1/kabosu.so
CHANGED
|
Binary file
|
data/lib/kabosu/3.2/kabosu.so
CHANGED
|
Binary file
|
data/lib/kabosu/3.3/kabosu.so
CHANGED
|
Binary file
|
data/lib/kabosu/3.4/kabosu.so
CHANGED
|
Binary file
|
data/lib/kabosu/4.0/kabosu.so
CHANGED
|
Binary file
|
data/lib/kabosu/dict_manager.rb
CHANGED
|
@@ -8,8 +8,8 @@ module Kabosu
|
|
|
8
8
|
class DictManager
|
|
9
9
|
EDITIONS = %w[small core full].freeze
|
|
10
10
|
EDITION_PRIORITY = %w[full core small].freeze
|
|
11
|
-
GITHUB_REPO = "WorksApplications/SudachiDict"
|
|
12
|
-
GITHUB_API = "https://api.github.com"
|
|
11
|
+
GITHUB_REPO = "WorksApplications/SudachiDict".freeze
|
|
12
|
+
GITHUB_API = "https://api.github.com".freeze
|
|
13
13
|
|
|
14
14
|
class DictNotFound < StandardError; end
|
|
15
15
|
class DownloadError < StandardError; end
|
|
@@ -42,23 +42,40 @@ module Kabosu
|
|
|
42
42
|
dic_path = File.join(dest_dir, "system_#{edition}.dic")
|
|
43
43
|
|
|
44
44
|
if File.exist?(dic_path)
|
|
45
|
-
|
|
45
|
+
warn "Already installed: #{dic_path}"
|
|
46
46
|
return dic_path
|
|
47
47
|
end
|
|
48
48
|
|
|
49
|
-
url = release_asset_url(version, edition)
|
|
50
|
-
zip_path = File.join(@dir, "sudachi-dictionary-#{version}-#{edition}.zip")
|
|
51
|
-
|
|
52
49
|
FileUtils.mkdir_p(@dir)
|
|
53
|
-
download(url, zip_path)
|
|
54
|
-
extract(zip_path, @dir)
|
|
55
|
-
FileUtils.rm_f(zip_path)
|
|
56
50
|
|
|
57
|
-
|
|
58
|
-
|
|
51
|
+
# SudachiDict switched to Python-only release assets in v20260723:
|
|
52
|
+
# https://github.com/WorksApplications/SudachiDict/releases/tag/v20260723
|
|
53
|
+
# The legacy `sudachi-dictionary-{version}-{edition}.zip` is gone; the
|
|
54
|
+
# new releases ship `sudachidict_{edition}-{version}-py3-none-any.whl`
|
|
55
|
+
# (a PEP 427 wheel). Pick whichever exists, preferring the legacy zip
|
|
56
|
+
# when both are present (older releases kept both formats in flight).
|
|
57
|
+
sources = pick_release_sources(version, edition)
|
|
58
|
+
raise DownloadError, "No downloadable assets for #{version}/#{edition}" if sources.empty?
|
|
59
|
+
|
|
60
|
+
sources.each do |source|
|
|
61
|
+
begin
|
|
62
|
+
download(source.fetch(:url), source.fetch(:archive_path))
|
|
63
|
+
extract(source.fetch(:archive_path), dest_dir, edition: edition)
|
|
64
|
+
FileUtils.rm_f(source.fetch(:archive_path))
|
|
65
|
+
break
|
|
66
|
+
rescue DownloadError => e
|
|
67
|
+
FileUtils.rm_f(source.fetch(:archive_path)) if File.exist?(source.fetch(:archive_path))
|
|
68
|
+
# Try the next candidate (e.g. wheel if the zip 404s). If this was
|
|
69
|
+
# the last one, surface the original error.
|
|
70
|
+
raise if sources.last.equal?(source)
|
|
71
|
+
|
|
72
|
+
warn " falling back: #{e.message}"
|
|
73
|
+
end
|
|
59
74
|
end
|
|
60
75
|
|
|
61
|
-
|
|
76
|
+
raise DownloadError, "Expected #{dic_path} after extraction, but file not found" unless File.exist?(dic_path)
|
|
77
|
+
|
|
78
|
+
warn "Installed: #{dic_path}"
|
|
62
79
|
dic_path
|
|
63
80
|
end
|
|
64
81
|
|
|
@@ -88,7 +105,7 @@ module Kabosu
|
|
|
88
105
|
results = []
|
|
89
106
|
return results unless Dir.exist?(@dir)
|
|
90
107
|
|
|
91
|
-
Dir.glob(File.join(@dir, "sudachi-dictionary-*")).
|
|
108
|
+
Dir.glob(File.join(@dir, "sudachi-dictionary-*")).reverse.each do |version_dir|
|
|
92
109
|
next unless File.directory?(version_dir)
|
|
93
110
|
|
|
94
111
|
version = File.basename(version_dir).sub("sudachi-dictionary-", "")
|
|
@@ -113,6 +130,7 @@ module Kabosu
|
|
|
113
130
|
edition = validate_edition(edition)
|
|
114
131
|
match = candidates.find { |d| d[:edition] == edition }
|
|
115
132
|
raise DictNotFound, "No #{edition} dictionary installed" unless match
|
|
133
|
+
|
|
116
134
|
return match[:path]
|
|
117
135
|
end
|
|
118
136
|
|
|
@@ -140,14 +158,14 @@ module Kabosu
|
|
|
140
158
|
|
|
141
159
|
targets.each do |d|
|
|
142
160
|
FileUtils.rm_f(d[:path])
|
|
143
|
-
|
|
161
|
+
warn "Removed: #{d[:path]}"
|
|
144
162
|
|
|
145
163
|
# Clean up empty version directories
|
|
146
164
|
version_dir = File.dirname(d[:path])
|
|
147
165
|
dics_remaining = Dir.glob(File.join(version_dir, "system_*.dic"))
|
|
148
166
|
if dics_remaining.empty?
|
|
149
167
|
FileUtils.rm_rf(version_dir)
|
|
150
|
-
|
|
168
|
+
warn "Removed empty directory: #{version_dir}"
|
|
151
169
|
end
|
|
152
170
|
end
|
|
153
171
|
end
|
|
@@ -178,15 +196,61 @@ module Kabosu
|
|
|
178
196
|
unless EDITIONS.include?(edition)
|
|
179
197
|
raise ArgumentError, "Unknown edition '#{edition}'. Must be one of: #{EDITIONS.join(", ")}"
|
|
180
198
|
end
|
|
199
|
+
|
|
181
200
|
edition
|
|
182
201
|
end
|
|
183
202
|
|
|
203
|
+
# Look up the GitHub release for `version` and return an ordered list of
|
|
204
|
+
# candidate download sources. The legacy `sudachi-dictionary-{version}-{edition}.zip`
|
|
205
|
+
# is preferred when present (matches the layout every other consumer
|
|
206
|
+
# assumes); otherwise we fall back to the SudachiDict Python wheel,
|
|
207
|
+
# which is what v20260723+ ship and which carries `system.dic` inside
|
|
208
|
+
# `sudachidict_{edition}/resources/`.
|
|
209
|
+
#
|
|
210
|
+
# On GitHub API failure we degrade to the legacy zip URL alone: an old
|
|
211
|
+
# release that 404s on the zip *also* 404s on the wheel, so falling back
|
|
212
|
+
# to "guessed" candidates costs nothing and keeps the code path simple
|
|
213
|
+
# for offline / no-network callers.
|
|
214
|
+
def pick_release_sources(version, edition)
|
|
215
|
+
candidates = []
|
|
216
|
+
zip = release_asset_url(version, edition)
|
|
217
|
+
candidates << { url: zip, archive_path: File.join(@dir, "sudachi-dictionary-#{version}-#{edition}.zip"), format: :zip }
|
|
218
|
+
|
|
219
|
+
begin
|
|
220
|
+
release = fetch_release(version)
|
|
221
|
+
if release
|
|
222
|
+
wheel_name = "sudachidict_#{edition}-#{version}-py3-none-any.whl"
|
|
223
|
+
wheel = release["assets"].to_a.find { |a| a["name"] == wheel_name }
|
|
224
|
+
if wheel && wheel["browser_download_url"]
|
|
225
|
+
candidates << {
|
|
226
|
+
url: wheel.fetch("browser_download_url"),
|
|
227
|
+
archive_path: File.join(@dir, wheel_name),
|
|
228
|
+
format: :wheel
|
|
229
|
+
}
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
rescue DownloadError
|
|
233
|
+
# API failure is fine: keep the legacy-zip candidate; the user will
|
|
234
|
+
# see a clean 404 if neither asset actually exists on disk.
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
candidates
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def fetch_release(version)
|
|
241
|
+
uri = URI("#{GITHUB_API}/repos/#{GITHUB_REPO}/releases/tags/v#{version}")
|
|
242
|
+
response = http_get(uri, headers: { "Accept" => "application/json" })
|
|
243
|
+
return nil unless response.is_a?(Net::HTTPSuccess)
|
|
244
|
+
|
|
245
|
+
JSON.parse(response.body)
|
|
246
|
+
end
|
|
247
|
+
|
|
184
248
|
def release_asset_url(version, edition)
|
|
185
249
|
"https://github.com/#{GITHUB_REPO}/releases/download/v#{version}/sudachi-dictionary-#{version}-#{edition}.zip"
|
|
186
250
|
end
|
|
187
251
|
|
|
188
252
|
def download(url, dest)
|
|
189
|
-
|
|
253
|
+
warn "Downloading #{url}..."
|
|
190
254
|
uri = resolve_redirects(URI(url))
|
|
191
255
|
|
|
192
256
|
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
|
|
@@ -202,20 +266,22 @@ module Kabosu
|
|
|
202
266
|
response.read_body do |chunk|
|
|
203
267
|
f.write(chunk)
|
|
204
268
|
written += chunk.bytesize
|
|
205
|
-
if total
|
|
269
|
+
if total&.positive?
|
|
206
270
|
pct = (written * 100 / total).clamp(0, 100)
|
|
207
|
-
|
|
271
|
+
done_mb = (written.to_f / 1024 / 1024).round(1)
|
|
272
|
+
total_mb = (total.to_f / 1024 / 1024).round(1)
|
|
273
|
+
$stderr.print "\r #{done_mb} / #{total_mb} MB (#{pct}%)"
|
|
208
274
|
end
|
|
209
275
|
end
|
|
210
276
|
end
|
|
211
277
|
|
|
212
|
-
|
|
278
|
+
warn "\r #{(written.to_f / 1024 / 1024).round(1)} MB downloaded"
|
|
213
279
|
end
|
|
214
280
|
end
|
|
215
281
|
end
|
|
216
282
|
|
|
217
283
|
def resolve_redirects(uri, limit: 5)
|
|
218
|
-
raise DownloadError, "Too many redirects" if limit
|
|
284
|
+
raise DownloadError, "Too many redirects" if limit.zero?
|
|
219
285
|
|
|
220
286
|
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
|
|
221
287
|
response = http.request(Net::HTTP::Head.new(uri))
|
|
@@ -229,7 +295,7 @@ module Kabosu
|
|
|
229
295
|
end
|
|
230
296
|
|
|
231
297
|
def http_get(uri, headers: {}, redirect_limit: 5)
|
|
232
|
-
raise DownloadError, "Too many redirects" if redirect_limit
|
|
298
|
+
raise DownloadError, "Too many redirects" if redirect_limit.zero?
|
|
233
299
|
|
|
234
300
|
http = Net::HTTP.new(uri.host, uri.port)
|
|
235
301
|
http.use_ssl = (uri.scheme == "https")
|
|
@@ -247,15 +313,27 @@ module Kabosu
|
|
|
247
313
|
end
|
|
248
314
|
end
|
|
249
315
|
|
|
250
|
-
def extract(zip_path, dest_dir)
|
|
251
|
-
|
|
316
|
+
def extract(zip_path, dest_dir, edition: nil)
|
|
317
|
+
warn "Extracting..."
|
|
252
318
|
Zip::File.open(zip_path) do |archive|
|
|
253
319
|
archive.each do |entry|
|
|
254
|
-
|
|
320
|
+
# Wheels (`sudachidict_{edition}/resources/system.dic`) carry the
|
|
321
|
+
# edition in the top-level directory; legacy zips name the file
|
|
322
|
+
# `system_{edition}.dic` already. When the entry is a `system.dic`
|
|
323
|
+
# inside a wheel and `edition:` was passed, land it at
|
|
324
|
+
# `dest_dir/system_{edition}.dic` so `find`/`installed` keep working
|
|
325
|
+
# without a separate code path.
|
|
326
|
+
target_name = entry.name
|
|
327
|
+
if edition && File.basename(entry.name) == "system.dic"
|
|
328
|
+
target_name = "system_#{edition}.dic"
|
|
329
|
+
end
|
|
330
|
+
|
|
331
|
+
target = File.join(dest_dir, target_name)
|
|
255
332
|
# Guard against zip-slip — refuse entries that escape dest_dir.
|
|
256
333
|
unless File.expand_path(target).start_with?(File.expand_path(dest_dir) + File::SEPARATOR)
|
|
257
334
|
raise DownloadError, "Refusing to extract entry outside dest_dir: #{entry.name}"
|
|
258
335
|
end
|
|
336
|
+
|
|
259
337
|
FileUtils.mkdir_p(File.dirname(target))
|
|
260
338
|
entry.extract(target) { true } # overwrite existing
|
|
261
339
|
end
|
data/lib/kabosu/morpheme_list.rb
CHANGED
|
@@ -7,7 +7,7 @@ module Kabosu
|
|
|
7
7
|
def initialize(source_or_morphemes, internal_cost: nil)
|
|
8
8
|
@source = source_or_morphemes if lazy_source?(source_or_morphemes)
|
|
9
9
|
@morphemes = @source ? Array.new(@source.size) : source_or_morphemes
|
|
10
|
-
@internal_cost = internal_cost ||
|
|
10
|
+
@internal_cost = internal_cost || @source&.internal_cost
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
def each(&block)
|
|
@@ -62,7 +62,7 @@ module Kabosu
|
|
|
62
62
|
end
|
|
63
63
|
|
|
64
64
|
def surfaces
|
|
65
|
-
return @source.surfaces if @source
|
|
65
|
+
return @source.surfaces if @source.respond_to?(:surfaces)
|
|
66
66
|
|
|
67
67
|
map(&:surface)
|
|
68
68
|
end
|
|
@@ -96,9 +96,7 @@ module Kabosu
|
|
|
96
96
|
# source. Falls back to a Ruby implementation for already-materialized
|
|
97
97
|
# lists so the method is always safe to call.
|
|
98
98
|
def group_morphemes
|
|
99
|
-
if @source
|
|
100
|
-
return @source.group_morphemes
|
|
101
|
-
end
|
|
99
|
+
return @source.group_morphemes if @source.respond_to?(:group_morphemes)
|
|
102
100
|
|
|
103
101
|
groups = []
|
|
104
102
|
each do |m|
|
|
@@ -131,10 +129,12 @@ module Kabosu
|
|
|
131
129
|
|
|
132
130
|
def clause_boundary?(morpheme)
|
|
133
131
|
return false unless morpheme
|
|
132
|
+
|
|
134
133
|
pos = morpheme.part_of_speech
|
|
135
134
|
return true if pos[0] == "助詞" &&
|
|
136
135
|
%w[ながら たら ば と のに から ので けれど けど つつ なり や か かどうか とも].include?(morpheme.surface)
|
|
137
136
|
return true if pos[0] == "助詞" && pos[1] == "接続助詞" && morpheme.surface == "が"
|
|
137
|
+
|
|
138
138
|
false
|
|
139
139
|
end
|
|
140
140
|
|
data/lib/kabosu/pos_matcher.rb
CHANGED
|
@@ -48,13 +48,15 @@ module Kabosu
|
|
|
48
48
|
|
|
49
49
|
# Union: matches if either matcher matches.
|
|
50
50
|
def |(other)
|
|
51
|
-
a
|
|
51
|
+
a = self
|
|
52
|
+
b = other
|
|
52
53
|
PosMatcher.new { |pos| a.match?(pos) || b.match?(pos) }
|
|
53
54
|
end
|
|
54
55
|
|
|
55
56
|
# Intersection: matches if both matchers match.
|
|
56
57
|
def &(other)
|
|
57
|
-
a
|
|
58
|
+
a = self
|
|
59
|
+
b = other
|
|
58
60
|
PosMatcher.new { |pos| a.match?(pos) && b.match?(pos) }
|
|
59
61
|
end
|
|
60
62
|
|
|
@@ -64,7 +66,8 @@ module Kabosu
|
|
|
64
66
|
end
|
|
65
67
|
|
|
66
68
|
def difference(other)
|
|
67
|
-
a
|
|
69
|
+
a = self
|
|
70
|
+
b = other
|
|
68
71
|
PosMatcher.new { |pos| a.match?(pos) && !b.match?(pos) }
|
|
69
72
|
end
|
|
70
73
|
|
|
@@ -95,7 +98,7 @@ module Kabosu
|
|
|
95
98
|
end
|
|
96
99
|
|
|
97
100
|
def self.proper_nouns
|
|
98
|
-
@proper_nouns ||= new([
|
|
101
|
+
@proper_nouns ||= new(%w[名詞 固有名詞])
|
|
99
102
|
end
|
|
100
103
|
|
|
101
104
|
private
|
data/lib/kabosu/tasks.rake
CHANGED
|
@@ -5,53 +5,54 @@ namespace :kabosu do
|
|
|
5
5
|
|
|
6
6
|
desc "Install the core dictionary (default). VERSION=YYYYMMDD to pin a specific release."
|
|
7
7
|
task :install do
|
|
8
|
-
Kabosu::DictManager.new.install("core", version: ENV
|
|
8
|
+
Kabosu::DictManager.new.install("core", version: ENV.fetch("VERSION", nil))
|
|
9
9
|
end
|
|
10
10
|
|
|
11
11
|
namespace :install do
|
|
12
12
|
desc "Install the small dictionary. VERSION=YYYYMMDD to pin a specific release."
|
|
13
13
|
task :small do
|
|
14
|
-
Kabosu::DictManager.new.install("small", version: ENV
|
|
14
|
+
Kabosu::DictManager.new.install("small", version: ENV.fetch("VERSION", nil))
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
desc "Install the core dictionary. VERSION=YYYYMMDD to pin a specific release."
|
|
18
18
|
task :core do
|
|
19
|
-
Kabosu::DictManager.new.install("core", version: ENV
|
|
19
|
+
Kabosu::DictManager.new.install("core", version: ENV.fetch("VERSION", nil))
|
|
20
20
|
end
|
|
21
21
|
|
|
22
22
|
desc "Install the full dictionary. VERSION=YYYYMMDD to pin a specific release."
|
|
23
23
|
task :full do
|
|
24
|
-
Kabosu::DictManager.new.install("full", version: ENV
|
|
24
|
+
Kabosu::DictManager.new.install("full", version: ENV.fetch("VERSION", nil))
|
|
25
25
|
end
|
|
26
26
|
end
|
|
27
27
|
|
|
28
|
-
desc "Install a dictionary only if a matching one isn't already on disk.
|
|
28
|
+
desc "Install a dictionary only if a matching one isn't already on disk. " \
|
|
29
|
+
"EDITION=core|small|full (default core), VERSION=YYYYMMDD optional."
|
|
29
30
|
task :install_if_missing do
|
|
30
31
|
edition = ENV["EDITION"] || "core"
|
|
31
|
-
Kabosu::DictManager.new.install_if_missing(edition, version: ENV
|
|
32
|
+
Kabosu::DictManager.new.install_if_missing(edition, version: ENV.fetch("VERSION", nil))
|
|
32
33
|
end
|
|
33
34
|
|
|
34
35
|
# ── Remove ──
|
|
35
36
|
|
|
36
37
|
desc "Remove all installed dictionaries, or a specific one with EDITION=small|core|full and/or VERSION=YYYYMMDD"
|
|
37
38
|
task :remove do
|
|
38
|
-
Kabosu::DictManager.new.remove(edition: ENV
|
|
39
|
+
Kabosu::DictManager.new.remove(edition: ENV.fetch("EDITION", nil), version: ENV.fetch("VERSION", nil))
|
|
39
40
|
end
|
|
40
41
|
|
|
41
42
|
namespace :remove do
|
|
42
43
|
desc "Remove the small dictionary."
|
|
43
44
|
task :small do
|
|
44
|
-
Kabosu::DictManager.new.remove(edition: "small", version: ENV
|
|
45
|
+
Kabosu::DictManager.new.remove(edition: "small", version: ENV.fetch("VERSION", nil))
|
|
45
46
|
end
|
|
46
47
|
|
|
47
48
|
desc "Remove the core dictionary."
|
|
48
49
|
task :core do
|
|
49
|
-
Kabosu::DictManager.new.remove(edition: "core", version: ENV
|
|
50
|
+
Kabosu::DictManager.new.remove(edition: "core", version: ENV.fetch("VERSION", nil))
|
|
50
51
|
end
|
|
51
52
|
|
|
52
53
|
desc "Remove the full dictionary."
|
|
53
54
|
task :full do
|
|
54
|
-
Kabosu::DictManager.new.remove(edition: "full", version: ENV
|
|
55
|
+
Kabosu::DictManager.new.remove(edition: "full", version: ENV.fetch("VERSION", nil))
|
|
55
56
|
end
|
|
56
57
|
end
|
|
57
58
|
|
|
@@ -82,11 +83,9 @@ namespace :kabosu do
|
|
|
82
83
|
|
|
83
84
|
desc "Show the path to the best available dictionary. EDITION=small|core|full to be specific."
|
|
84
85
|
task :path do
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
exit 1
|
|
90
|
-
end
|
|
86
|
+
puts Kabosu::DictManager.new.find(edition: ENV.fetch("EDITION", nil))
|
|
87
|
+
rescue Kabosu::DictManager::DictNotFound => e
|
|
88
|
+
warn e.message
|
|
89
|
+
exit 1
|
|
91
90
|
end
|
|
92
91
|
end
|
data/lib/kabosu/version.rb
CHANGED
data/lib/kabosu.rb
CHANGED
|
@@ -66,25 +66,17 @@ module Kabosu
|
|
|
66
66
|
DEFAULT_CONFIG_PATH = File.expand_path("kabosu/resources/sudachi.json", __dir__).freeze
|
|
67
67
|
|
|
68
68
|
class << self
|
|
69
|
-
|
|
69
|
+
alias _new new
|
|
70
70
|
|
|
71
71
|
def new(config: nil, system_dict: nil, user_dicts: nil)
|
|
72
|
-
unless config.nil? || config.is_a?(String)
|
|
73
|
-
|
|
74
|
-
end
|
|
75
|
-
unless system_dict.nil? || system_dict.is_a?(String)
|
|
76
|
-
raise ArgumentError, "system_dict must be a String or nil"
|
|
77
|
-
end
|
|
72
|
+
raise ArgumentError, "config must be a String or nil" unless config.nil? || config.is_a?(String)
|
|
73
|
+
raise ArgumentError, "system_dict must be a String or nil" unless system_dict.nil? || system_dict.is_a?(String)
|
|
78
74
|
unless user_dicts.nil? || user_dicts.is_a?(Array)
|
|
79
75
|
raise ArgumentError, "user_dicts must be an Array<String> or nil"
|
|
80
76
|
end
|
|
81
|
-
if user_dicts&.any? { !_1.is_a?(String) }
|
|
82
|
-
raise ArgumentError, "user_dicts must contain only String values"
|
|
83
|
-
end
|
|
77
|
+
raise ArgumentError, "user_dicts must contain only String values" if user_dicts&.any? { !_1.is_a?(String) }
|
|
84
78
|
|
|
85
|
-
if config.nil? && system_dict.nil?
|
|
86
|
-
raise ArgumentError, "either config or system_dict is required"
|
|
87
|
-
end
|
|
79
|
+
raise ArgumentError, "either config or system_dict is required" if config.nil? && system_dict.nil?
|
|
88
80
|
|
|
89
81
|
# Default to the sudachi.json bundled with this gem when only
|
|
90
82
|
# system_dict is given. sudachi.rs's own default config path is
|
|
@@ -113,9 +105,7 @@ module Kabosu
|
|
|
113
105
|
|
|
114
106
|
def map_dictionary_init_error(error, config:, system_dict:)
|
|
115
107
|
message = error.message
|
|
116
|
-
if config && system_dict.nil?
|
|
117
|
-
ConfigError.new(message)
|
|
118
|
-
elsif message.match?(/config|setting\.json|json/i)
|
|
108
|
+
if (config && system_dict.nil?) || message.match?(/config|setting\.json|json/i)
|
|
119
109
|
ConfigError.new(message)
|
|
120
110
|
else
|
|
121
111
|
DictionaryError.new(message)
|
|
@@ -127,40 +117,33 @@ module Kabosu
|
|
|
127
117
|
end
|
|
128
118
|
end
|
|
129
119
|
|
|
130
|
-
|
|
131
|
-
|
|
120
|
+
alias _create create
|
|
121
|
+
alias _lookup lookup
|
|
132
122
|
|
|
133
123
|
def create(**options)
|
|
134
124
|
unknown = options.keys - %i[mode fields debug projection]
|
|
135
|
-
raise ArgumentError, "unknown keyword(s): #{unknown.join(
|
|
125
|
+
raise ArgumentError, "unknown keyword(s): #{unknown.join(", ")}" unless unknown.empty?
|
|
136
126
|
|
|
137
127
|
mode = options.fetch(:mode, MODE_C)
|
|
138
128
|
fields = options.fetch(:fields, nil)
|
|
139
129
|
debug = options.fetch(:debug, false)
|
|
140
130
|
projection = options.fetch(:projection, nil)
|
|
141
131
|
|
|
142
|
-
unless fields.nil? || fields.is_a?(Array)
|
|
143
|
-
raise ArgumentError, "fields must be an Array<String|Symbol> or nil"
|
|
144
|
-
end
|
|
132
|
+
raise ArgumentError, "fields must be an Array<String|Symbol> or nil" unless fields.nil? || fields.is_a?(Array)
|
|
145
133
|
if fields&.any? { !(_1.is_a?(String) || _1.is_a?(Symbol)) }
|
|
146
134
|
raise ArgumentError, "fields must contain only String or Symbol values"
|
|
147
135
|
end
|
|
148
|
-
|
|
149
|
-
raise ArgumentError, "debug must be true or false"
|
|
150
|
-
end
|
|
136
|
+
raise ArgumentError, "debug must be true or false" unless [true, false].include?(debug)
|
|
151
137
|
|
|
152
|
-
unless projection.nil?
|
|
153
|
-
raise NotImplementedError, "projection is not supported yet"
|
|
154
|
-
end
|
|
138
|
+
raise NotImplementedError, "projection is not supported yet" unless projection.nil?
|
|
155
139
|
|
|
156
140
|
mode_str = Kabosu.__send__(:normalize_mode, mode)
|
|
157
141
|
_create(mode_str, fields, debug)
|
|
158
142
|
end
|
|
159
143
|
|
|
160
144
|
def lookup(text)
|
|
161
|
-
unless text.is_a?(String)
|
|
162
|
-
|
|
163
|
-
end
|
|
145
|
+
raise ArgumentError, "text must be a String" unless text.is_a?(String)
|
|
146
|
+
|
|
164
147
|
MorphemeList.new(_lookup(text))
|
|
165
148
|
rescue RuntimeError => e
|
|
166
149
|
raise LookupError.new(e.message), cause: e
|
|
@@ -170,12 +153,10 @@ module Kabosu
|
|
|
170
153
|
# ── Tokenizer: wrap output in MorphemeList ──
|
|
171
154
|
|
|
172
155
|
class Tokenizer
|
|
173
|
-
|
|
156
|
+
alias _tokenize tokenize
|
|
174
157
|
|
|
175
158
|
def tokenize(text)
|
|
176
|
-
unless text.is_a?(String)
|
|
177
|
-
raise ArgumentError, "text must be a String"
|
|
178
|
-
end
|
|
159
|
+
raise ArgumentError, "text must be a String" unless text.is_a?(String)
|
|
179
160
|
|
|
180
161
|
batch = _tokenize(text)
|
|
181
162
|
cost = batch.respond_to?(:internal_cost) ? batch.internal_cost : nil
|
|
@@ -186,12 +167,11 @@ module Kabosu
|
|
|
186
167
|
end
|
|
187
168
|
|
|
188
169
|
class Morpheme
|
|
189
|
-
|
|
170
|
+
alias _split split
|
|
190
171
|
|
|
191
172
|
def split(mode: MODE_C, add_single: true)
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
end
|
|
173
|
+
raise ArgumentError, "add_single must be true or false" unless [true, false].include?(add_single)
|
|
174
|
+
|
|
195
175
|
mode_str = Kabosu.__send__(:normalize_mode, mode)
|
|
196
176
|
MorphemeList.new(_split(mode_str, nil, add_single))
|
|
197
177
|
rescue RuntimeError => e
|
|
@@ -200,29 +180,15 @@ module Kabosu
|
|
|
200
180
|
end
|
|
201
181
|
|
|
202
182
|
def self.split_sentences(text, limit: nil, with_checker: false, ranges: false, dictionary: nil)
|
|
203
|
-
unless text.is_a?(String)
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
unless
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
if limit && limit < 1
|
|
210
|
-
raise ArgumentError, "limit must be greater than 0"
|
|
211
|
-
end
|
|
212
|
-
unless with_checker == true || with_checker == false
|
|
213
|
-
raise ArgumentError, "with_checker must be true or false"
|
|
214
|
-
end
|
|
215
|
-
unless ranges == true || ranges == false
|
|
216
|
-
raise ArgumentError, "ranges must be true or false"
|
|
217
|
-
end
|
|
218
|
-
unless dictionary.nil? || dictionary.is_a?(String)
|
|
219
|
-
raise ArgumentError, "dictionary must be a String path or nil"
|
|
220
|
-
end
|
|
183
|
+
raise ArgumentError, "text must be a String" unless text.is_a?(String)
|
|
184
|
+
raise ArgumentError, "limit must be an Integer or nil" unless limit.nil? || limit.is_a?(Integer)
|
|
185
|
+
raise ArgumentError, "limit must be greater than 0" if limit && limit < 1
|
|
186
|
+
raise ArgumentError, "with_checker must be true or false" unless [true, false].include?(with_checker)
|
|
187
|
+
raise ArgumentError, "ranges must be true or false" unless [true, false].include?(ranges)
|
|
188
|
+
raise ArgumentError, "dictionary must be a String path or nil" unless dictionary.nil? || dictionary.is_a?(String)
|
|
221
189
|
|
|
222
190
|
dict_path = nil
|
|
223
|
-
if with_checker
|
|
224
|
-
dict_path = dictionary || Dictionary.path
|
|
225
|
-
end
|
|
191
|
+
dict_path = dictionary || Dictionary.path if with_checker
|
|
226
192
|
|
|
227
193
|
if ranges
|
|
228
194
|
_split_sentences_with_ranges(text, limit, dict_path).map do |(start, finish, sentence)|
|
|
@@ -244,12 +210,8 @@ module Kabosu
|
|
|
244
210
|
# Kabosu.tokenize("東京都に住んでいる", tokenizer: tok)
|
|
245
211
|
#
|
|
246
212
|
def self.tokenize(text, tokenizer:)
|
|
247
|
-
unless text.is_a?(String)
|
|
248
|
-
|
|
249
|
-
end
|
|
250
|
-
unless tokenizer.is_a?(Tokenizer)
|
|
251
|
-
raise ArgumentError, "tokenizer must be a Kabosu::Tokenizer"
|
|
252
|
-
end
|
|
213
|
+
raise ArgumentError, "text must be a String" unless text.is_a?(String)
|
|
214
|
+
raise ArgumentError, "tokenizer must be a Kabosu::Tokenizer" unless tokenizer.is_a?(Tokenizer)
|
|
253
215
|
|
|
254
216
|
batch = tokenizer.__send__(:_tokenize, text)
|
|
255
217
|
cost = batch.respond_to?(:internal_cost) ? batch.internal_cost : nil
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: kabosu
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.6.
|
|
4
|
+
version: 0.6.11.2
|
|
5
5
|
platform: aarch64-linux
|
|
6
6
|
authors:
|
|
7
7
|
- davafons
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-08-20 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rubyzip
|
|
@@ -25,61 +25,75 @@ dependencies:
|
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
26
|
version: '2.3'
|
|
27
27
|
- !ruby/object:Gem::Dependency
|
|
28
|
-
name:
|
|
28
|
+
name: benchmark
|
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - ">="
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: '0'
|
|
34
|
+
type: :development
|
|
35
|
+
prerelease: false
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
+
requirements:
|
|
38
|
+
- - ">="
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: '0'
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: minitest
|
|
29
43
|
requirement: !ruby/object:Gem::Requirement
|
|
30
44
|
requirements:
|
|
31
45
|
- - "~>"
|
|
32
46
|
- !ruby/object:Gem::Version
|
|
33
|
-
version: '
|
|
47
|
+
version: '5.0'
|
|
34
48
|
type: :development
|
|
35
49
|
prerelease: false
|
|
36
50
|
version_requirements: !ruby/object:Gem::Requirement
|
|
37
51
|
requirements:
|
|
38
52
|
- - "~>"
|
|
39
53
|
- !ruby/object:Gem::Version
|
|
40
|
-
version: '
|
|
54
|
+
version: '5.0'
|
|
41
55
|
- !ruby/object:Gem::Dependency
|
|
42
|
-
name: rake
|
|
56
|
+
name: rake
|
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
|
44
58
|
requirements:
|
|
45
59
|
- - "~>"
|
|
46
60
|
- !ruby/object:Gem::Version
|
|
47
|
-
version: '
|
|
61
|
+
version: '13.0'
|
|
48
62
|
type: :development
|
|
49
63
|
prerelease: false
|
|
50
64
|
version_requirements: !ruby/object:Gem::Requirement
|
|
51
65
|
requirements:
|
|
52
66
|
- - "~>"
|
|
53
67
|
- !ruby/object:Gem::Version
|
|
54
|
-
version: '
|
|
68
|
+
version: '13.0'
|
|
55
69
|
- !ruby/object:Gem::Dependency
|
|
56
|
-
name:
|
|
70
|
+
name: rake-compiler
|
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
|
58
72
|
requirements:
|
|
59
|
-
- - "
|
|
73
|
+
- - "~>"
|
|
60
74
|
- !ruby/object:Gem::Version
|
|
61
|
-
version: '
|
|
75
|
+
version: '1.2'
|
|
62
76
|
type: :development
|
|
63
77
|
prerelease: false
|
|
64
78
|
version_requirements: !ruby/object:Gem::Requirement
|
|
65
79
|
requirements:
|
|
66
|
-
- - "
|
|
80
|
+
- - "~>"
|
|
67
81
|
- !ruby/object:Gem::Version
|
|
68
|
-
version: '
|
|
82
|
+
version: '1.2'
|
|
69
83
|
- !ruby/object:Gem::Dependency
|
|
70
|
-
name:
|
|
84
|
+
name: rubocop
|
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
|
72
86
|
requirements:
|
|
73
87
|
- - "~>"
|
|
74
88
|
- !ruby/object:Gem::Version
|
|
75
|
-
version: '
|
|
89
|
+
version: '1.0'
|
|
76
90
|
type: :development
|
|
77
91
|
prerelease: false
|
|
78
92
|
version_requirements: !ruby/object:Gem::Requirement
|
|
79
93
|
requirements:
|
|
80
94
|
- - "~>"
|
|
81
95
|
- !ruby/object:Gem::Version
|
|
82
|
-
version: '
|
|
96
|
+
version: '1.0'
|
|
83
97
|
description: Kabosu provides Ruby bindings for sudachi.rs, a Rust implementation of
|
|
84
98
|
the Sudachi Japanese morphological analyzer.
|
|
85
99
|
email:
|
|
@@ -109,7 +123,8 @@ files:
|
|
|
109
123
|
homepage: https://github.com/davafons/kabosu
|
|
110
124
|
licenses:
|
|
111
125
|
- Apache-2.0
|
|
112
|
-
metadata:
|
|
126
|
+
metadata:
|
|
127
|
+
rubygems_mfa_required: 'true'
|
|
113
128
|
post_install_message:
|
|
114
129
|
rdoc_options: []
|
|
115
130
|
require_paths:
|