kotoshu-server 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6fa24971e66e791bd277c8fe878890693c6a624a3de1c488dbb7f5b38f629f0a
-  data.tar.gz: eabc7a8dd8124cf20a1c05b32257e9fdfdcdebeb72ac6aceae249c2aba692976
+  metadata.gz: 4a1120bdda9dcd7ecc468c95d5386127efefa6509e7aa1a03f2fa12a5e28c33f
+  data.tar.gz: b8cbe5d0db18715009576fd104be80bb06e35ba1b730cbd2564178755a05e6a1
 SHA512:
-  metadata.gz: fac8b861344a723119bf84e7c078c772d491d974cb37a9338b2100acb0e28cf9e17d83e06b4f18aad41ecba9d2ec78bc6249575830b485f03f6f1200ab58daa0
-  data.tar.gz: bd7c11018743035167858f90f8a7b70aa29efdfb6313496b3f6d06a1e297931985e6259625d3b43b34a5236d03179f58851a33f388f9f5a6e2cbcb0701ab745b
+  metadata.gz: 6c001de9ec24f78e48eac1c400b0ecf9038b47fdeed7115ad3ed0635e100adc7b3ae9e241a7474c8956636f1690f3dea39b425c125d122577aa67dfd097af2a4
+  data.tar.gz: 8696ca46e3728fbd2b2f32b3e9468be4e544290d83badb6837b2133bb76700fd7241c687446b7926e9d0a485eb95fbae22d44183503da64cb84cada4227080d7

data/README.md

@@ -12,9 +12,12 @@ See `TODO.impl/64-http-api-and-sdks.md` for the full plan.
 
 ## Install
 
-**Run from source.** The published `kotoshu-server 0.1.0` gem is
-empty (a gemspec file-list bug, fixed on main). Use source until the
-owner republishes 0.1.1:
+```bash
+gem install kotoshu-server   # >= 0.1.1 (the 0.1.0 gem was empty — a gemspec file-list bug)
+kotoshu-server
+```
+
+Or run from source:
 
 ```bash
 cd kotoshu-server && bundle install && bundle exec exe/kotoshu-server
@@ -71,6 +74,24 @@ the default fluency tier (the full tier is far larger), plus
 one-time model load on the first model-enabled request. Listing the
 language in `KOTOSHU_SERVER_MODEL_LANGS` warms it at boot instead.
 
+## Language detection
+
+`POST /v1/detect` reports which engine answered in the `engine`
+field:
+
+- `"lid-176"` — the 176-language lid.176 model served through the
+  kotoshu native extension (kotoshu >= 0.10.0). The model is set up
+  lazily on the first detect — one download, never at boot, and
+  never at all under `KOTOSHU_OFFLINE=1` without a cache (Docker
+  defaults to offline).
+- `"heuristic"` — the 7-language character-set heuristic (en, de,
+  es, fr, pt, ru, ja), the fallback whenever lid-176 cannot serve:
+  kotoshu < 0.10.0, a pure-Ruby install or `KOTOSHU_BACKEND=ruby`,
+  or the model missing after a failed setup.
+
+Set `KOTOSHU_DETECT=heuristic` to pin the heuristic regardless of
+the installed gem — the 0.1.1 behavior, byte for byte.
+
 ## Endpoints
 
 | Method | Path | Body | Returns |
@@ -81,7 +102,7 @@ language in `KOTOSHU_SERVER_MODEL_LANGS` warms it at boot instead.
 | `GET` | `/v1/languages` | — | `{ cached, supported, model }` |
 | `POST` | `/v1/check` | `{ text, language?, format?, model? }` | `{ file, word_count, errors: [...] }` |
 | `POST` | `/v1/suggest` | `{ word, language?, max? }` | `{ word, suggestions: [...] }` |
-| `POST` | `/v1/detect` | `{ text }` | `{ language, confidence }` |
+| `POST` | `/v1/detect` | `{ text }` | `{ language, confidence, engine }` |
 
 Each `error` and `suggestion` mirrors the lutaml-model serialization
 (`word`, `distance`, `confidence`, `source`).
@@ -129,6 +150,7 @@ Healthcheck probes `/v1/health` every 30s.
 | `KOTOSHU_SERVER_LAZY` | `0` | Skip pre-warm; load on first request |
 | `KOTOSHU_SERVER_DEFAULT_LANG` | `en` | When client omits `language` |
 | `KOTOSHU_SERVER_LOG_LEVEL` | `info` | `debug`/`info`/`warn`/`error` |
+| `KOTOSHU_DETECT` | `auto` | `heuristic` pins /v1/detect to the heuristic engine |
 | `KOTOSHU_OFFLINE` | `1` (in Docker) | Never trigger downloads |
 
 ## OpenAPI

data/lib/kotoshu/server/app.rb

@@ -12,7 +12,9 @@ module Kotoshu
     class ModelConfigError < StandardError; end
 
     class App < Sinatra::Base
-      VERSION = "0.1.0".freeze
+      # The release version, from lib/kotoshu/server/version.rb —
+      # the only source of truth (the release workflow bumps it).
+      VERSION = Kotoshu::Server::VERSION
 
       # Semantic models (tiers, registry, confidence cascade) ship in
       # kotoshu 0.7.0. The gemspec keeps its `kotoshu ~> 0.6`
@@ -20,6 +22,14 @@ module Kotoshu
       # server validates the *installed* gem at boot instead.
       MODEL_MIN_KOTOSHU = Gem::Version.new("0.7.0")
 
+      # Native language identification (plan 106) ships in kotoshu
+      # 0.10.0: Kotoshu.detect_language returning a
+      # Language::Detection backed by the lid-176 model through the
+      # native extension. Same policy as MODEL_MIN_KOTOSHU — the
+      # gemspec floor stays `kotoshu ~> 0.6` (older gems keep the
+      # heuristic), so the installed gem is checked per request.
+      DETECT_MIN_KOTOSHU = Gem::Version.new("0.10.0")
+
       # Languages to set up at boot, from KOTOSHU_SERVER_LANGUAGES
       # (space separated). Single source of truth for the env var: the
       # boot pre-warm and /v1/health both read it here.
@@ -148,6 +158,99 @@ module Kotoshu
         end
       end
 
+      # ---- Language detection (/v1/detect) ----
+
+      @lid_setup_mutex = Mutex.new
+      @lid_setup_attempted = false
+
+      # Whether the installed kotoshu gem carries the lid-176
+      # detection surface (0.10.0+): Kotoshu.detect_language returning
+      # a Language::Detection, plus setup_lid / LidDetector.available?.
+      #
+      # @return [Boolean]
+      def self.lid_supported?
+        Gem::Version.new(Kotoshu::VERSION) >= DETECT_MIN_KOTOSHU
+      end
+
+      # Whether KOTOSHU_DETECT=heuristic pins /v1/detect to the
+      # 7-language heuristic, bypassing the gem engine selection (the
+      # 0.1.1 behavior, e.g. to reproduce earlier results).
+      #
+      # @return [Boolean]
+      def self.heuristic_detect?
+        ENV.fetch("KOTOSHU_DETECT", nil) == "heuristic"
+      end
+
+      # Detect for /v1/detect: the language code, its score, and the
+      # engine that served — "lid-176" or "heuristic".
+      #
+      # Engine selection, in order:
+      # 1. KOTOSHU_DETECT=heuristic — the heuristic, pinned.
+      # 2. kotoshu < 0.10.0 — the heuristic, as in 0.1.1; the server
+      #    keeps working on old gems.
+      # 3. Otherwise the gem's own selection: lid-176 through the
+      #    native extension when it is built, the backend is not
+      #    explicitly ruby, and the artifact pair is cached; the
+      #    heuristic otherwise (KOTOSHU_BACKEND=ruby, pure-Ruby
+      #    install, missing model).
+      #
+      # The lid model is set up lazily on the first detect reaching
+      # case 3 — one download, off the boot path, honoring
+      # KOTOSHU_OFFLINE through the gem; a failed setup (offline with
+      # no cache, no registry entry, checksum mismatch) logs once and
+      # detection answers from whatever the gem can serve.
+      #
+      # @param text [String] the text to analyze
+      # @return [Array(String, Float, String)] code (nil when the
+      #   heuristic is uncertain), score in [0, 1], engine name
+      def self.detect_language_with_engine(text)
+        return heuristic_detection(text) if heuristic_detect? || !lid_supported?
+
+        ensure_lid_setup!
+        detection = Kotoshu.detect_language(text)
+        [detection.code, detection.score, lid_engine]
+      end
+
+      # The heuristic detection (Language::Detector, the same engine
+      # /v1/detect served in 0.1.1, unchanged across gem versions).
+      #
+      # @param text [String]
+      # @return [Array(String, Float, String)]
+      def self.heuristic_detection(text)
+        code, confidence = Kotoshu::Language::Detector.detect_with_confidence(text)
+        [code, confidence, "heuristic"]
+      end
+
+      # Which engine the gem serves detect from: lid-176 when the
+      # native model is loadable, the heuristic otherwise. Asked
+      # after a detect, so it always names the engine behind the
+      # returned code.
+      #
+      # @return [String]
+      def self.lid_engine
+        Kotoshu::Language::LidDetector.available? ? "lid-176" : "heuristic"
+      end
+
+      # Run Kotoshu.setup_lid once per process, on the first detect.
+      # Idempotent in the gem; the once-guard keeps concurrent
+      # requests from racing the download. Any failure is logged and
+      # swallowed — detect then falls back inside the gem to whatever
+      # is cached (typically the heuristic).
+      #
+      # @return [void]
+      def self.ensure_lid_setup!
+        @lid_setup_mutex.synchronize do
+          return if @lid_setup_attempted
+
+          @lid_setup_attempted = true
+          Kotoshu.setup_lid
+        end
+      rescue StandardError => e
+        Logger.new($stderr).warn(
+          "lid model setup failed, falling back to the heuristic: #{e.class}: #{e.message}"
+        )
+      end
+
       # ---- Semantic analyzers (memoized per language + model file) ----
 
       @semantic_analyzers = {}
@@ -265,9 +368,9 @@ module Kotoshu
         text = body["text"]
         halt_with_error(400, "missing 'text'") unless text.is_a?(String)
 
-        lang, confidence = Kotoshu.detect_language_with_confidence(text)
+        language, confidence, engine = self.class.detect_language_with_engine(text)
         content_type :json
-        { language: lang, confidence: confidence }.to_json
+        { language: language, confidence: confidence, engine: engine }.to_json
       end
 
       # ---- Error handling ----

data/lib/kotoshu/server/version.rb

@@ -2,6 +2,6 @@
 
 module Kotoshu
   module Server
-    VERSION = "0.1.1"
+    VERSION = "0.1.2"
   end
 end

data/openapi.yaml

@@ -236,6 +236,13 @@ components:
       properties:
         language: { type: string, nullable: true }
         confidence: { type: number, format: float }
+        engine:
+          type: string
+          enum: [lid-176, heuristic]
+          description: >
+            Which engine served the detection: the 176-language lid-176
+            model (kotoshu >= 0.10.0 with the native extension and the
+            model set up) or the 7-language heuristic.
     Error:
       type: object
       properties:

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kotoshu-server
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-09-06 00:00:00.000000000 Z
+date: 2026-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kotoshu