cjk_index 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b8780b3265cb3e02580edd66360721d6a9ea245c3b8465c49e499e0b1bb6d420
-  data.tar.gz: a9c1233d9e57d170ba90c25e7e0ecf9af6b10e89640584855dbe6fae5b358418
+  metadata.gz: 615a0f6d44d6e04e21f66aeb15778e39adf566ec078f11c69ceecf41473e1388
+  data.tar.gz: 7a106a769e2cc57e382727c8ada1afbba691fbdb7620c0f487eddbe31610abf8
 SHA512:
-  metadata.gz: 205fff662f662ae191a3bb437351531473bcbb8943758d13b7b6593aeabb425814fb8a4cc779f5a028b3d0ff2c6c895ab90efb49ed6d9a6423d31f63f62bb8ed
-  data.tar.gz: 4a82cf20daf35f34bb5596645e373e4c1902722f5fd91c6541a77c8039f6d97a97d8421c68d20761c29914140d4e8201aa3d5409b883458ff2417deb633dde6b
+  metadata.gz: 2783f62f61930fbd740261d552d7b8092b098b1e15a4fe28741b37f766b9670f5e66045c994da3bcff2b16644f28772b681b8f6fd70b28440f8dd58571e0759e
+  data.tar.gz: c1faf175e55309bf62568d7215bee743fe7a4f0942d9621e1ad8f846e9082393dd51c38ef2ef141bdada33ce887a253d4ae533e812abe6e07cf59423901e3638

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 1.1.1 (2026-10-28)
+
+- Variation selectors (U+FE00–U+FE0F and the ideographic U+E0100–U+E01EF)
+  are dropped when folding: 「珊」 followed by an ideographic variation
+  selector is 「珊」. Before, the selector broke a CJK run in two, so
+  「珊瑚」 did not find 「珊\u{E0100}瑚」, and a table file of your own with
+  such a cell was rejected as "not one character". `highlight` and
+  `excerpt` keep the selector inside the mark with its character. Rebuild
+  indexes whose text has variation selectors.
+
 ## 1.1.0 (2026-10-28)
 
 - `variants` also takes table files of your own: one group per line,

data/README.md

@@ -24,8 +24,8 @@ even when it is an exact title. cjk_index instead:
   stores their positions, so a query matches only where its bigrams are
   adjacent and in order: an exact substring match, with no dictionary or
   native extension (the same idea as [kensaku](https://github.com/whispcat/kensaku));
-- folds variants on both sides: NFKC, lowercase, katakana → hiragana,
-  old-form kanji → modern form (「圖」→「図」), 々 expanded (「佐々木」→「佐佐木」),
+- folds variants on both sides: NFKC, lowercase, variation selectors
+  dropped, katakana → hiragana, old-form kanji → modern form (「圖」→「図」), 々 expanded (「佐々木」→「佐佐木」),
   and optionally Traditional ↔ Simplified Chinese (from OpenCC) and old
   Hangul in the Private Use Area;
 - ranks with BM25 per field, with field boosts;
@@ -386,8 +386,10 @@ cjk_index:
   variants: [ja, _data/itaiji.tsv]
 ```
 
-Each cell is folded first (NFKC, lowercase, katakana → hiragana), and the
-table listed first still picks the form a group folds to. The index records
+Each cell is folded first (NFKC, lowercase, variation selectors dropped,
+katakana → hiragana), so a cell written as an ideographic variation sequence
+(珊 + U+E0100) is the character itself, and the table listed first still
+picks the form a group folds to. The index records
 the entry as written (`_data/itaiji.tsv`), not the full path, and its digest
 covers the file's contents, so editing the file means rebuilding the index.
 To read such an index from another folder in Ruby, pass the normalizer:

data/lib/cjk_index/normalizer.rb

@@ -5,9 +5,10 @@ require "json"
 
 module CJKIndex
   # Folds text so that spelling variants a reader treats as "the same" compare
-  # equal: NFKC (full-width -> half-width etc.), lowercase, katakana ->
-  # hiragana, character variants from the selected tables, and the iteration
-  # mark 々 expanded to the preceding character.
+  # equal: NFKC (full-width -> half-width etc.), lowercase, variation
+  # selectors dropped (珊 followed by an ideographic variation selector is 珊),
+  # katakana -> hiragana, character variants from the selected tables, and
+  # the iteration mark 々 expanded to the preceding character.
   #
   # The same rules run in the browser (see Runtime), so an index built here
   # and a query typed there are folded identically.
@@ -52,6 +53,11 @@ module CJKIndex
     KATAKANA_LAST = 0x30F6
     KANA_OFFSET = 0x60
     ITERATION_MARK = "々"
+    # Variation selectors (VS1-VS16 and the ideographic VS17-VS256) only pick
+    # a glyph of the character before them: 珊 with or without U+E0100 is the
+    # same character to a reader, so they are dropped, not kept as a run of
+    # their own between two CJK characters. In String#delete form.
+    VARIATION_SELECTORS = "\u{FE00}-\u{FE0F}\u{E0100}-\u{E01EF}"
 
     def self.default
       @default ||= new
@@ -121,14 +127,16 @@ module CJKIndex
                            "#{tables.inspect} (an older cjk_index?); rebuild it"
     end
 
-    # Character by character: katakana to hiragana, then the variant tables,
-    # then 々 to the (folded) character before it. Done as passes over the
+    # Character by character: variation selectors dropped, katakana to
+    # hiragana, then the variant tables, then 々 to the (folded) character
+    # before it. Done as passes over the
     # whole string, which gives the same result as one loop over the
     # characters (the loop in runtime.js) and is several times faster in Ruby.
     def normalize(text)
       return "" if text.nil?
 
       s = text.to_s.unicode_normalize(:nfkc).downcase
+      s = s.delete(VARIATION_SELECTORS)
       s = s.tr(KATAKANA, HIRAGANA)
       # 々 repeats the folded character before it, which may be several
       # characters (ko): fold one character at a time, as runtime.js does

data/lib/cjk_index/runtime.js

@@ -59,6 +59,10 @@
   // Score multiplier for matches found by prefix or typo expansion.
   var EXPANDED = 0.6;
 
+  // Variation selectors (VS1-VS16, VS17-VS256): dropped, as
+  // CJKIndex::Normalizer::VARIATION_SELECTORS in Ruby.
+  var SELECTOR = /^[\ufe00-\ufe0f\u{e0100}-\u{e01ef}]$/u;
+
   // Folds one character; prev is the previous folded character (for 々).
   function foldChar(ch, prev) {
     var c = ch.codePointAt(0);
@@ -73,6 +77,7 @@
     var out = '';
     var prev = null;
     for (var ch of String(text).normalize('NFKC').toLowerCase()) {
+      if (SELECTOR.test(ch)) continue;
       prev = foldChar(ch, prev);
       out += prev;
     }
@@ -534,11 +539,16 @@
     // Each folded code point remembers which original characters it came
     // from. A character may fold to several (ko: one Private Use Area
     // character -> a jamo sequence), and a cluster may compose into fewer.
+    // A character that folds to nothing (a variation selector) belongs to
+    // the one before it: a match on 珊 marks its selector too.
     function emit(str, from, to) {
+      var before = norm.length;
       for (var c of str.normalize('NFKC').toLowerCase()) {
+        if (SELECTOR.test(c)) continue;
         prev = foldChar(c, prev);
         for (var p of prev) { norm.push(p); start.push(from); end.push(to); }
       }
+      if (norm.length === before && end.length) end[end.length - 1] = to;
     }
     for (var i = 0, j; i < chars.length; i = j) {
       for (j = i + 1; j < chars.length && JOINS_PREVIOUS.test(chars[j]); j++);

data/lib/cjk_index/searcher.rb

@@ -353,19 +353,23 @@ module CJKIndex
     chars = text.to_s.chars
     norm = +""
     origin = []
+    ends = []
     chars.each_with_index do |ch, i|
       folded = normalizer.normalize(ch)
       # 々 depends on the previous character, so fold it in context.
       folded = norm[-1] if folded == Normalizer::ITERATION_MARK && !norm.empty?
+      # A character that folds to nothing (a variation selector) belongs to
+      # the one before it: a match on 珊 marks its selector too.
+      ends[-1] = i + 1 if folded.empty? && !ends.empty?
       norm << folded
-      folded.length.times { origin << i }
+      folded.length.times { origin << i; ends << i + 1 }
     end
     needles = normalizer.normalize(query).split(Tokenizer::SEPARATOR).reject(&:empty?)
     ranges = needles.flat_map do |needle|
       found = []
       from = 0
       while (at = norm.index(needle, from))
-        found << [origin[at], origin[at + needle.length - 1] + 1]
+        found << [origin[at], ends[at + needle.length - 1]]
         from = at + needle.length
       end
       found

data/lib/cjk_index/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CJKIndex
-  VERSION = "1.1.0"
+  VERSION = "1.1.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cjk_index
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors:
 - Satoru Nakamura