cjk_index 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c9ffe634562d740738f6d9b9cf97ade5f5ab6d32a80b6bfd349496435d9e88b
-  data.tar.gz: 6cc0aebdedd14b07cdcf9dda3dcc9b849b80d135149444f057daed4cc620c004
+  metadata.gz: b8780b3265cb3e02580edd66360721d6a9ea245c3b8465c49e499e0b1bb6d420
+  data.tar.gz: a9c1233d9e57d170ba90c25e7e0ecf9af6b10e89640584855dbe6fae5b358418
 SHA512:
-  metadata.gz: ee41c34d99efe6e17f8936f3211b62aaab6fdafa54b714f685458c641a56263a46b506de9f703a4cd1c6f470c32e0d336b1914547655a433641ecbf9cdc5fcc0
-  data.tar.gz: 608805cba3a84e8c1bb8425e09c8ff3bdcb2bd4da6443f6ca63eeb6d2e3952e75b0166db871e853383d3531cb40d7f48052cc70540f2ae7fd8bd5ae372424af7
+  metadata.gz: 205fff662f662ae191a3bb437351531473bcbb8943758d13b7b6593aeabb425814fb8a4cc779f5a028b3d0ff2c6c895ab90efb49ed6d9a6423d31f63f62bb8ed
+  data.tar.gz: 4a82cf20daf35f34bb5596645e373e4c1902722f5fd91c6541a77c8039f6d97a97d8421c68d20761c29914140d4e8201aa3d5409b883458ff2417deb633dde6b

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 1.1.0 (2026-10-28)
+
+- `variants` also takes table files of your own: one group per line,
+  tab-separated, the form to fold to first (the format of the itaiji lists
+  that collection databases keep). Paths are relative to the site's folder
+  (Jekyll, Bridgetown) or the current folder; the index records the path as
+  written and a digest of the contents. `Builder.load` and `Builder.read`
+  take `normalizer:` to read such an index from another folder.
+- `Searcher.new(builder)` folds with the builder's normalizer.
+
 ## 1.0.0 (2026-10-28)
 
 Indexes built with 0.x should be rebuilt: the `ja` table changed, and the

data/README.md

@@ -366,6 +366,35 @@ by Unicode normalization. `ko` is for old Hangul stored in the Private Use
 Area by HWP and the Hanyang fonts; a query typed in standard jamo then finds
 and highlights it.
 
+### A table of your own
+
+Any entry of `variants` that is not one of the names above is a table file
+of your own, relative to the site's folder (Jekyll, Bridgetown) or the
+current folder (CLI, Rake, Ruby). One group per line, tab-separated: the
+form to fold to, then its variants, one character each. This is the format
+of the itaiji lists that collection databases often keep already.
+
+```text
+# itaiji.tsv
+与	與
+宴	醼	讌
+嶋	嶌
+```
+
+```yaml
+cjk_index:
+  variants: [ja, _data/itaiji.tsv]
+```
+
+Each cell is folded first (NFKC, lowercase, katakana → hiragana), and the
+table listed first still picks the form a group folds to. The index records
+the entry as written (`_data/itaiji.tsv`), not the full path, and its digest
+covers the file's contents, so editing the file means rebuilding the index.
+To read such an index from another folder in Ruby, pass the normalizer:
+`Builder.load(json, normalizer: CJKIndex::Normalizer.new(tables: [...], base: dir))`.
+Wide lists merge more than spelling: a list that puts 燕 with 宴 makes
+a search for 燕 (swallow) find 宴 (banquet). Check a list against your own material first.
+
 Browser script size, gzipped: no table 6.3 KB, `ja` 9.0 KB, `ja, zh-safe`
 24.1 KB, `ja, zh` 25.9 KB, `ja, ko` 36.1 KB.
 

data/lib/cjk_index/bridgetown.rb

@@ -11,7 +11,7 @@ module CJKIndex
   #   Bridgetown.configure do |config|
   #     init :cjk_index,
   #          output: "cjk-index",               # folder inside the output (default)
-  #          variants: ["ja"],                  # variant tables: ja (default), zh
+  #          variants: ["ja"],                  # variant tables: ja (default), zh-safe, zh, ko, or a file
   #          indexes: [
   #            { name: "posts",                 # -> output/cjk-index/posts.json
   #              collection: "posts",           # a collection (or a list of them), or...
@@ -37,7 +37,7 @@ module CJKIndex
     def write(site, config)
       config = stringify(config)
       output = config.fetch("output", DEFAULT_OUTPUT).to_s.gsub(%r{\A/+|/+\z}, "")
-      normalizer = Normalizer.new(tables: Array(config.fetch("variants", Normalizer::DEFAULT_TABLES)))
+      normalizer = Normalizer.new(tables: Array(config.fetch("variants", Normalizer::DEFAULT_TABLES)), base: site.root_dir)
       put(site, output, RUNTIME_NAME, Runtime.source(normalizer: normalizer))
       Array(config["indexes"]).each do |spec|
         builder = build(site, stringify(spec), normalizer)

data/lib/cjk_index/builder.rb

@@ -129,7 +129,9 @@ module CJKIndex
     # Reads an index written by to_json (String) or to_h back, to update it.
     # Reads formats 2, 3 and 4; a split index is read with read.
     # A value filter keeps the form of each value that the index stored.
-    def self.load(index)
+    # normalizer: needed when the index uses a table file of your own and
+    # the current folder is not the one it was built from.
+    def self.load(index, normalizer: nil)
       data = index.is_a?(String) ? JSON.parse(index) : index
       unless data.is_a?(Hash) && FORMATS.include?(data["format"])
         raise ArgumentError, "cjk_index: unsupported index format #{data.is_a?(Hash) ? data['format'].inspect : data.class}"
@@ -137,7 +139,7 @@ module CJKIndex
       raise ArgumentError, "cjk_index: a split index is read with Builder.read(directory)" unless data["tokens"]
 
       filters = (data["filters"] || {}).transform_values { |f| f.slice("kind", "separator") }
-      normalizer = Normalizer.new(tables: data["variants"])
+      normalizer ||= Normalizer.new(tables: data["variants"])
       normalizer.check!(data)
       builder = new(fields: data["fields"], boosts: data["fields"].zip(data["boosts"]).to_h,
                     normalizer: normalizer, filters: filters)
@@ -147,8 +149,8 @@ module CJKIndex
 
     # Reads an index from a file (written by write or to_json) or from the
     # directory of a split index (written by write_split), to update it.
-    def self.read(path)
-      load(Builder.read_data(path))
+    def self.read(path, normalizer: nil)
+      load(Builder.read_data(path), normalizer: normalizer)
     end
 
     # The index at path as one Hash with "tokens", for a file or a split

data/lib/cjk_index/cli.rb

@@ -73,7 +73,7 @@ module CJKIndex
           name, spec = parse_filter(v)
           opts[:filters][name] = spec
         end
-        o.on("--variants LIST", Array, "variant tables: ja (default), zh-safe, zh, ko") { |v| opts[:variants] = v }
+        o.on("--variants LIST", Array, "variant tables: ja (default), zh-safe, zh, ko, or table files") { |v| opts[:variants] = v }
         o.on("-o", "--output PATH", "index file (default: standard output)") { |v| output = v }
         o.on("--runtime PATH", "also write the browser script for these variants") { |v| runtime = v }
       end
@@ -196,7 +196,7 @@ module CJKIndex
       output = nil
       OptionParser.new do |o|
         o.banner = "Usage: cjk_index runtime [options]"
-        o.on("--variants LIST", Array, "variant tables: ja (default), zh-safe, zh, ko; must match the index") { |v| variants = v }
+        o.on("--variants LIST", Array, "variant tables: ja (default), zh-safe, zh, ko, or table files; must match the index") { |v| variants = v }
         o.on("-o", "--output PATH", "script file (default: standard output)") { |v| output = v }
       end.parse!(argv)
       raise Error, "too many arguments: #{argv.join(' ')}" unless argv.empty?

data/lib/cjk_index/jekyll.rb

@@ -9,7 +9,7 @@ module CJKIndex
   #
   #   cjk_index:
   #     output: assets/cjk-index          # where files are written (default)
-  #     variants: [ja]                    # variant tables: ja (default), zh (Traditional -> Simplified)
+  #     variants: [ja]                    # variant tables: ja (default), zh-safe, zh, ko, or a file of your own
   #     indexes:
   #       - name: posts                   # -> assets/cjk-index/posts.json
   #         collection: posts             # a Jekyll collection, or...
@@ -64,7 +64,7 @@ module CJKIndex
         return unless config.is_a?(Hash)
 
         output = config.fetch("output", DEFAULT_OUTPUT).sub(%r{\A/+}, "").sub(%r{/+\z}, "")
-        normalizer = Normalizer.new(tables: Array(config.fetch("variants", Normalizer::DEFAULT_TABLES)))
+        normalizer = Normalizer.new(tables: Array(config.fetch("variants", Normalizer::DEFAULT_TABLES)), base: site.source)
         site.static_files << GeneratedFile.new(site, output, RUNTIME_NAME, Runtime.source(normalizer: normalizer))
         Array(config["indexes"]).each do |spec|
           builder = build(site, spec, normalizer)

data/lib/cjk_index/normalizer.rb

@@ -24,10 +24,19 @@ module CJKIndex
   # With several tables, characters linked by any table fold to one form,
   # preferring the target of the table listed first: with [ja, zh],
   # 圖, 図 and 图 all become 図.
+  # Any other entry names a table file of your own, relative to base (the
+  # site's folder for Jekyll and Bridgetown, the current folder otherwise).
+  # The index records the entry as written, not the resolved path.
   #
   # Table format: UTF-8 lines "variant<TAB>folded", optionally followed by
   # another tab and a comment column; lines starting with "#" are comments.
   # "folded" may be several characters.
+  #
+  # Your own table is one group per line, tab-separated: the form to fold to,
+  # then its variants, one character each (the format of the itaiji lists
+  # that collection databases keep). Each cell is folded first (NFKC,
+  # lowercase, katakana -> hiragana), so a compatibility ideograph such as
+  # U+F900 joins the group of the character NFKC makes of it.
   class Normalizer
     DATA_DIR = File.expand_path("../../data", __dir__)
     TABLES = {
@@ -48,11 +57,38 @@ module CJKIndex
       @default ||= new
     end
 
-    def self.read_table(name)
-      file = TABLES.fetch(name) { raise ArgumentError, "unknown variant table #{name.inspect} (#{TABLES.keys.join(', ')})" }
-      File.readlines(File.join(DATA_DIR, file), chomp: true, encoding: "UTF-8")
-          .reject { |line| line.empty? || line.start_with?("#") }
-          .map { |line| line.split("\t")[0, 2] }
+    # Pairs [variant, folded] of a built-in table, or of a table file of your
+    # own (a name that is not built in, resolved against base).
+    def self.read_table(name, base: Dir.pwd)
+      return read_groups(name, File.expand_path(name, base)) unless TABLES.key?(name)
+
+      lines(File.join(DATA_DIR, TABLES[name])).map { |_, line| line.split("\t")[0, 2] }
+    end
+
+    def self.lines(path) # :nodoc:
+      File.readlines(path, chomp: true, encoding: "UTF-8").each_with_index
+          .reject { |line, _| line.strip.empty? || line.start_with?("#") }
+          .map { |line, i| [i + 1, line.delete_prefix("\uFEFF")] }
+    end
+
+    def self.read_groups(name, path) # :nodoc:
+      unless File.file?(path)
+        raise ArgumentError, "unknown variant table #{name.inspect}: not one of #{TABLES.keys.join(', ')}, " \
+                             "and no file at #{path}"
+      end
+
+      plain = new(tables: [])
+      lines(path).flat_map do |number, line|
+        target, *variants = line.split("\t").map(&:strip).reject(&:empty?).map { |cell| plain.normalize(cell) }
+        variants = variants.uniq.reject { |v| v == target } # NFKC may already fold a cell (㍿)
+        variants.each do |v|
+          next if v.length == 1
+
+          raise ArgumentError, "#{name}:#{number}: #{v.inspect} is not one character; " \
+                               "only the first cell (the form to fold to) may be longer"
+        end
+        variants.map { |v| [v, target] }
+      end
     end
 
     attr_reader :tables, :variants, :digest
@@ -60,8 +96,9 @@ module CJKIndex
     KATAKANA = "#{KATAKANA_FIRST.chr(Encoding::UTF_8)}-#{KATAKANA_LAST.chr(Encoding::UTF_8)}"
     HIRAGANA = "#{(KATAKANA_FIRST - KANA_OFFSET).chr(Encoding::UTF_8)}-#{(KATAKANA_LAST - KANA_OFFSET).chr(Encoding::UTF_8)}"
 
-    def initialize(tables: DEFAULT_TABLES)
+    def initialize(tables: DEFAULT_TABLES, base: Dir.pwd)
       @tables = tables.map(&:to_s).uniq.freeze
+      @base = base
       @variants = build_variants.freeze
       keys = @variants.keys.select { |k| k.length == 1 }
       @variant_chars = keys.empty? ? nil : Regexp.new("[#{keys.map { |k| format('\\u{%x}', k.ord) }.join}]")
@@ -122,7 +159,7 @@ module CJKIndex
       end
       rank = {}
       @tables.each_with_index do |name, priority|
-        self.class.read_table(name).each do |from, to|
+        self.class.read_table(name, base: @base).each do |from, to|
           parent[find.call(from)] = find.call(to)
           rank[to] = [rank.fetch(to, [priority, to.ord]), [priority, to.ord]].min
         end

data/lib/cjk_index/searcher.rb

@@ -57,7 +57,7 @@ module CJKIndex
         end
       end
 
-      @normalizer = normalizer || Normalizer.new(tables: data["variants"])
+      @normalizer = normalizer || @builder&.normalizer || Normalizer.new(tables: data["variants"])
       @normalizer.check!(data)
 
       if data["shards"] && !@load_shard

data/lib/cjk_index/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CJKIndex
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cjk_index
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Satoru Nakamura