cjk_index 1.0.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c9ffe634562d740738f6d9b9cf97ade5f5ab6d32a80b6bfd349496435d9e88b
-  data.tar.gz: 6cc0aebdedd14b07cdcf9dda3dcc9b849b80d135149444f057daed4cc620c004
+  metadata.gz: 615a0f6d44d6e04e21f66aeb15778e39adf566ec078f11c69ceecf41473e1388
+  data.tar.gz: 7a106a769e2cc57e382727c8ada1afbba691fbdb7620c0f487eddbe31610abf8
 SHA512:
-  metadata.gz: ee41c34d99efe6e17f8936f3211b62aaab6fdafa54b714f685458c641a56263a46b506de9f703a4cd1c6f470c32e0d336b1914547655a433641ecbf9cdc5fcc0
-  data.tar.gz: 608805cba3a84e8c1bb8425e09c8ff3bdcb2bd4da6443f6ca63eeb6d2e3952e75b0166db871e853383d3531cb40d7f48052cc70540f2ae7fd8bd5ae372424af7
+  metadata.gz: 2783f62f61930fbd740261d552d7b8092b098b1e15a4fe28741b37f766b9670f5e66045c994da3bcff2b16644f28772b681b8f6fd70b28440f8dd58571e0759e
+  data.tar.gz: c1faf175e55309bf62568d7215bee743fe7a4f0942d9621e1ad8f846e9082393dd51c38ef2ef141bdada33ce887a253d4ae533e812abe6e07cf59423901e3638

data/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## 1.1.1 (2026-10-28)
+
+- Variation selectors (U+FE00–U+FE0F and the ideographic U+E0100–U+E01EF)
+  are dropped when folding: 「珊」 followed by an ideographic variation
+  selector is 「珊」. Before, the selector broke a CJK run in two, so
+  「珊瑚」 did not find 「珊\u{E0100}瑚」, and a table file of your own with
+  such a cell was rejected as "not one character". `highlight` and
+  `excerpt` keep the selector inside the mark with its character. Rebuild
+  indexes whose text has variation selectors.
+
+## 1.1.0 (2026-10-28)
+
+- `variants` also takes table files of your own: one group per line,
+  tab-separated, the form to fold to first (the format of the itaiji lists
+  that collection databases keep). Paths are relative to the site's folder
+  (Jekyll, Bridgetown) or the current folder; the index records the path as
+  written and a digest of the contents. `Builder.load` and `Builder.read`
+  take `normalizer:` to read such an index from another folder.
+- `Searcher.new(builder)` folds with the builder's normalizer.
+
 ## 1.0.0 (2026-10-28)
 
 Indexes built with 0.x should be rebuilt: the `ja` table changed, and the

data/README.md

@@ -24,8 +24,8 @@ even when it is an exact title. cjk_index instead:
   stores their positions, so a query matches only where its bigrams are
   adjacent and in order: an exact substring match, with no dictionary or
   native extension (the same idea as [kensaku](https://github.com/whispcat/kensaku));
-- folds variants on both sides: NFKC, lowercase, katakana → hiragana,
-  old-form kanji → modern form (「圖」→「図」), 々 expanded (「佐々木」→「佐佐木」),
+- folds variants on both sides: NFKC, lowercase, variation selectors
+  dropped, katakana → hiragana, old-form kanji → modern form (「圖」→「図」), 々 expanded (「佐々木」→「佐佐木」),
   and optionally Traditional ↔ Simplified Chinese (from OpenCC) and old
   Hangul in the Private Use Area;
 - ranks with BM25 per field, with field boosts;
@@ -366,6 +366,37 @@ by Unicode normalization. `ko` is for old Hangul stored in the Private Use
 Area by HWP and the Hanyang fonts; a query typed in standard jamo then finds
 and highlights it.
 
+### A table of your own
+
+Any entry of `variants` that is not one of the names above is a table file
+of your own, relative to the site's folder (Jekyll, Bridgetown) or the
+current folder (CLI, Rake, Ruby). One group per line, tab-separated: the
+form to fold to, then its variants, one character each. This is the format
+of the itaiji lists that collection databases often keep already.
+
+```text
+# itaiji.tsv
+与	與
+宴	醼	讌
+嶋	嶌
+```
+
+```yaml
+cjk_index:
+  variants: [ja, _data/itaiji.tsv]
+```
+
+Each cell is folded first (NFKC, lowercase, variation selectors dropped,
+katakana → hiragana), so a cell written as an ideographic variation sequence
+(珊 + U+E0100) is the character itself, and the table listed first still
+picks the form a group folds to. The index records
+the entry as written (`_data/itaiji.tsv`), not the full path, and its digest
+covers the file's contents, so editing the file means rebuilding the index.
+To read such an index from another folder in Ruby, pass the normalizer:
+`Builder.load(json, normalizer: CJKIndex::Normalizer.new(tables: [...], base: dir))`.
+Wide lists merge more than spelling: a list that puts 燕 with 宴 makes
+a search for 燕 (swallow) find 宴 (banquet). Check a list against your own material first.
+
 Browser script size, gzipped: no table 6.3 KB, `ja` 9.0 KB, `ja, zh-safe`
 24.1 KB, `ja, zh` 25.9 KB, `ja, ko` 36.1 KB.
 

data/lib/cjk_index/bridgetown.rb

@@ -11,7 +11,7 @@ module CJKIndex
   #   Bridgetown.configure do |config|
   #     init :cjk_index,
   #          output: "cjk-index",               # folder inside the output (default)
-  #          variants: ["ja"],                  # variant tables: ja (default), zh
+  #          variants: ["ja"],                  # variant tables: ja (default), zh-safe, zh, ko, or a file
   #          indexes: [
   #            { name: "posts",                 # -> output/cjk-index/posts.json
   #              collection: "posts",           # a collection (or a list of them), or...
@@ -37,7 +37,7 @@ module CJKIndex
     def write(site, config)
       config = stringify(config)
       output = config.fetch("output", DEFAULT_OUTPUT).to_s.gsub(%r{\A/+|/+\z}, "")
-      normalizer = Normalizer.new(tables: Array(config.fetch("variants", Normalizer::DEFAULT_TABLES)))
+      normalizer = Normalizer.new(tables: Array(config.fetch("variants", Normalizer::DEFAULT_TABLES)), base: site.root_dir)
       put(site, output, RUNTIME_NAME, Runtime.source(normalizer: normalizer))
       Array(config["indexes"]).each do |spec|
         builder = build(site, stringify(spec), normalizer)

data/lib/cjk_index/builder.rb

@@ -129,7 +129,9 @@ module CJKIndex
     # Reads an index written by to_json (String) or to_h back, to update it.
     # Reads formats 2, 3 and 4; a split index is read with read.
     # A value filter keeps the form of each value that the index stored.
-    def self.load(index)
+    # normalizer: needed when the index uses a table file of your own and
+    # the current folder is not the one it was built from.
+    def self.load(index, normalizer: nil)
       data = index.is_a?(String) ? JSON.parse(index) : index
       unless data.is_a?(Hash) && FORMATS.include?(data["format"])
         raise ArgumentError, "cjk_index: unsupported index format #{data.is_a?(Hash) ? data['format'].inspect : data.class}"
@@ -137,7 +139,7 @@ module CJKIndex
       raise ArgumentError, "cjk_index: a split index is read with Builder.read(directory)" unless data["tokens"]
 
       filters = (data["filters"] || {}).transform_values { |f| f.slice("kind", "separator") }
-      normalizer = Normalizer.new(tables: data["variants"])
+      normalizer ||= Normalizer.new(tables: data["variants"])
       normalizer.check!(data)
       builder = new(fields: data["fields"], boosts: data["fields"].zip(data["boosts"]).to_h,
                     normalizer: normalizer, filters: filters)
@@ -147,8 +149,8 @@ module CJKIndex
 
     # Reads an index from a file (written by write or to_json) or from the
     # directory of a split index (written by write_split), to update it.
-    def self.read(path)
-      load(Builder.read_data(path))
+    def self.read(path, normalizer: nil)
+      load(Builder.read_data(path), normalizer: normalizer)
     end
 
     # The index at path as one Hash with "tokens", for a file or a split

data/lib/cjk_index/cli.rb

@@ -73,7 +73,7 @@ module CJKIndex
           name, spec = parse_filter(v)
           opts[:filters][name] = spec
         end
-        o.on("--variants LIST", Array, "variant tables: ja (default), zh-safe, zh, ko") { |v| opts[:variants] = v }
+        o.on("--variants LIST", Array, "variant tables: ja (default), zh-safe, zh, ko, or table files") { |v| opts[:variants] = v }
         o.on("-o", "--output PATH", "index file (default: standard output)") { |v| output = v }
         o.on("--runtime PATH", "also write the browser script for these variants") { |v| runtime = v }
       end
@@ -196,7 +196,7 @@ module CJKIndex
       output = nil
       OptionParser.new do |o|
         o.banner = "Usage: cjk_index runtime [options]"
-        o.on("--variants LIST", Array, "variant tables: ja (default), zh-safe, zh, ko; must match the index") { |v| variants = v }
+        o.on("--variants LIST", Array, "variant tables: ja (default), zh-safe, zh, ko, or table files; must match the index") { |v| variants = v }
         o.on("-o", "--output PATH", "script file (default: standard output)") { |v| output = v }
       end.parse!(argv)
       raise Error, "too many arguments: #{argv.join(' ')}" unless argv.empty?

data/lib/cjk_index/jekyll.rb

@@ -9,7 +9,7 @@ module CJKIndex
   #
   #   cjk_index:
   #     output: assets/cjk-index          # where files are written (default)
-  #     variants: [ja]                    # variant tables: ja (default), zh (Traditional -> Simplified)
+  #     variants: [ja]                    # variant tables: ja (default), zh-safe, zh, ko, or a file of your own
   #     indexes:
   #       - name: posts                   # -> assets/cjk-index/posts.json
   #         collection: posts             # a Jekyll collection, or...
@@ -64,7 +64,7 @@ module CJKIndex
         return unless config.is_a?(Hash)
 
         output = config.fetch("output", DEFAULT_OUTPUT).sub(%r{\A/+}, "").sub(%r{/+\z}, "")
-        normalizer = Normalizer.new(tables: Array(config.fetch("variants", Normalizer::DEFAULT_TABLES)))
+        normalizer = Normalizer.new(tables: Array(config.fetch("variants", Normalizer::DEFAULT_TABLES)), base: site.source)
         site.static_files << GeneratedFile.new(site, output, RUNTIME_NAME, Runtime.source(normalizer: normalizer))
         Array(config["indexes"]).each do |spec|
           builder = build(site, spec, normalizer)

data/lib/cjk_index/normalizer.rb

@@ -5,9 +5,10 @@ require "json"
 
 module CJKIndex
   # Folds text so that spelling variants a reader treats as "the same" compare
-  # equal: NFKC (full-width -> half-width etc.), lowercase, katakana ->
-  # hiragana, character variants from the selected tables, and the iteration
-  # mark 々 expanded to the preceding character.
+  # equal: NFKC (full-width -> half-width etc.), lowercase, variation
+  # selectors dropped (珊 followed by an ideographic variation selector is 珊),
+  # katakana -> hiragana, character variants from the selected tables, and
+  # the iteration mark 々 expanded to the preceding character.
   #
   # The same rules run in the browser (see Runtime), so an index built here
   # and a query typed there are folded identically.
@@ -24,10 +25,19 @@ module CJKIndex
   # With several tables, characters linked by any table fold to one form,
   # preferring the target of the table listed first: with [ja, zh],
   # 圖, 図 and 图 all become 図.
+  # Any other entry names a table file of your own, relative to base (the
+  # site's folder for Jekyll and Bridgetown, the current folder otherwise).
+  # The index records the entry as written, not the resolved path.
   #
   # Table format: UTF-8 lines "variant<TAB>folded", optionally followed by
   # another tab and a comment column; lines starting with "#" are comments.
   # "folded" may be several characters.
+  #
+  # Your own table is one group per line, tab-separated: the form to fold to,
+  # then its variants, one character each (the format of the itaiji lists
+  # that collection databases keep). Each cell is folded first (NFKC,
+  # lowercase, katakana -> hiragana), so a compatibility ideograph such as
+  # U+F900 joins the group of the character NFKC makes of it.
   class Normalizer
     DATA_DIR = File.expand_path("../../data", __dir__)
     TABLES = {
@@ -43,16 +53,48 @@ module CJKIndex
     KATAKANA_LAST = 0x30F6
     KANA_OFFSET = 0x60
     ITERATION_MARK = "々"
+    # Variation selectors (VS1-VS16 and the ideographic VS17-VS256) only pick
+    # a glyph of the character before them: 珊 with or without U+E0100 is the
+    # same character to a reader, so they are dropped, not kept as a run of
+    # their own between two CJK characters. In String#delete form.
+    VARIATION_SELECTORS = "\u{FE00}-\u{FE0F}\u{E0100}-\u{E01EF}"
 
     def self.default
       @default ||= new
     end
 
-    def self.read_table(name)
-      file = TABLES.fetch(name) { raise ArgumentError, "unknown variant table #{name.inspect} (#{TABLES.keys.join(', ')})" }
-      File.readlines(File.join(DATA_DIR, file), chomp: true, encoding: "UTF-8")
-          .reject { |line| line.empty? || line.start_with?("#") }
-          .map { |line| line.split("\t")[0, 2] }
+    # Pairs [variant, folded] of a built-in table, or of a table file of your
+    # own (a name that is not built in, resolved against base).
+    def self.read_table(name, base: Dir.pwd)
+      return read_groups(name, File.expand_path(name, base)) unless TABLES.key?(name)
+
+      lines(File.join(DATA_DIR, TABLES[name])).map { |_, line| line.split("\t")[0, 2] }
+    end
+
+    def self.lines(path) # :nodoc:
+      File.readlines(path, chomp: true, encoding: "UTF-8").each_with_index
+          .reject { |line, _| line.strip.empty? || line.start_with?("#") }
+          .map { |line, i| [i + 1, line.delete_prefix("\uFEFF")] }
+    end
+
+    def self.read_groups(name, path) # :nodoc:
+      unless File.file?(path)
+        raise ArgumentError, "unknown variant table #{name.inspect}: not one of #{TABLES.keys.join(', ')}, " \
+                             "and no file at #{path}"
+      end
+
+      plain = new(tables: [])
+      lines(path).flat_map do |number, line|
+        target, *variants = line.split("\t").map(&:strip).reject(&:empty?).map { |cell| plain.normalize(cell) }
+        variants = variants.uniq.reject { |v| v == target } # NFKC may already fold a cell (㍿)
+        variants.each do |v|
+          next if v.length == 1
+
+          raise ArgumentError, "#{name}:#{number}: #{v.inspect} is not one character; " \
+                               "only the first cell (the form to fold to) may be longer"
+        end
+        variants.map { |v| [v, target] }
+      end
     end
 
     attr_reader :tables, :variants, :digest
@@ -60,8 +102,9 @@ module CJKIndex
     KATAKANA = "#{KATAKANA_FIRST.chr(Encoding::UTF_8)}-#{KATAKANA_LAST.chr(Encoding::UTF_8)}"
     HIRAGANA = "#{(KATAKANA_FIRST - KANA_OFFSET).chr(Encoding::UTF_8)}-#{(KATAKANA_LAST - KANA_OFFSET).chr(Encoding::UTF_8)}"
 
-    def initialize(tables: DEFAULT_TABLES)
+    def initialize(tables: DEFAULT_TABLES, base: Dir.pwd)
       @tables = tables.map(&:to_s).uniq.freeze
+      @base = base
       @variants = build_variants.freeze
       keys = @variants.keys.select { |k| k.length == 1 }
       @variant_chars = keys.empty? ? nil : Regexp.new("[#{keys.map { |k| format('\\u{%x}', k.ord) }.join}]")
@@ -84,14 +127,16 @@ module CJKIndex
                            "#{tables.inspect} (an older cjk_index?); rebuild it"
     end
 
-    # Character by character: katakana to hiragana, then the variant tables,
-    # then 々 to the (folded) character before it. Done as passes over the
+    # Character by character: variation selectors dropped, katakana to
+    # hiragana, then the variant tables, then 々 to the (folded) character
+    # before it. Done as passes over the
     # whole string, which gives the same result as one loop over the
     # characters (the loop in runtime.js) and is several times faster in Ruby.
     def normalize(text)
       return "" if text.nil?
 
       s = text.to_s.unicode_normalize(:nfkc).downcase
+      s = s.delete(VARIATION_SELECTORS)
       s = s.tr(KATAKANA, HIRAGANA)
       # 々 repeats the folded character before it, which may be several
       # characters (ko): fold one character at a time, as runtime.js does
@@ -122,7 +167,7 @@ module CJKIndex
       end
       rank = {}
       @tables.each_with_index do |name, priority|
-        self.class.read_table(name).each do |from, to|
+        self.class.read_table(name, base: @base).each do |from, to|
           parent[find.call(from)] = find.call(to)
           rank[to] = [rank.fetch(to, [priority, to.ord]), [priority, to.ord]].min
         end

data/lib/cjk_index/runtime.js

@@ -59,6 +59,10 @@
   // Score multiplier for matches found by prefix or typo expansion.
   var EXPANDED = 0.6;
 
+  // Variation selectors (VS1-VS16, VS17-VS256): dropped, as
+  // CJKIndex::Normalizer::VARIATION_SELECTORS in Ruby.
+  var SELECTOR = /^[\ufe00-\ufe0f\u{e0100}-\u{e01ef}]$/u;
+
   // Folds one character; prev is the previous folded character (for 々).
   function foldChar(ch, prev) {
     var c = ch.codePointAt(0);
@@ -73,6 +77,7 @@
     var out = '';
     var prev = null;
     for (var ch of String(text).normalize('NFKC').toLowerCase()) {
+      if (SELECTOR.test(ch)) continue;
       prev = foldChar(ch, prev);
       out += prev;
     }
@@ -534,11 +539,16 @@
     // Each folded code point remembers which original characters it came
     // from. A character may fold to several (ko: one Private Use Area
     // character -> a jamo sequence), and a cluster may compose into fewer.
+    // A character that folds to nothing (a variation selector) belongs to
+    // the one before it: a match on 珊 marks its selector too.
     function emit(str, from, to) {
+      var before = norm.length;
       for (var c of str.normalize('NFKC').toLowerCase()) {
+        if (SELECTOR.test(c)) continue;
         prev = foldChar(c, prev);
         for (var p of prev) { norm.push(p); start.push(from); end.push(to); }
       }
+      if (norm.length === before && end.length) end[end.length - 1] = to;
     }
     for (var i = 0, j; i < chars.length; i = j) {
       for (j = i + 1; j < chars.length && JOINS_PREVIOUS.test(chars[j]); j++);

data/lib/cjk_index/searcher.rb

@@ -57,7 +57,7 @@ module CJKIndex
         end
       end
 
-      @normalizer = normalizer || Normalizer.new(tables: data["variants"])
+      @normalizer = normalizer || @builder&.normalizer || Normalizer.new(tables: data["variants"])
       @normalizer.check!(data)
 
       if data["shards"] && !@load_shard
@@ -353,19 +353,23 @@ module CJKIndex
     chars = text.to_s.chars
     norm = +""
     origin = []
+    ends = []
     chars.each_with_index do |ch, i|
       folded = normalizer.normalize(ch)
       # 々 depends on the previous character, so fold it in context.
       folded = norm[-1] if folded == Normalizer::ITERATION_MARK && !norm.empty?
+      # A character that folds to nothing (a variation selector) belongs to
+      # the one before it: a match on 珊 marks its selector too.
+      ends[-1] = i + 1 if folded.empty? && !ends.empty?
       norm << folded
-      folded.length.times { origin << i }
+      folded.length.times { origin << i; ends << i + 1 }
     end
     needles = normalizer.normalize(query).split(Tokenizer::SEPARATOR).reject(&:empty?)
     ranges = needles.flat_map do |needle|
       found = []
       from = 0
       while (at = norm.index(needle, from))
-        found << [origin[at], origin[at + needle.length - 1] + 1]
+        found << [origin[at], ends[at + needle.length - 1]]
         from = at + needle.length
       end
       found

data/lib/cjk_index/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CJKIndex
-  VERSION = "1.0.0"
+  VERSION = "1.1.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cjk_index
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.1
 platform: ruby
 authors:
 - Satoru Nakamura