cjk_index 0.1.0

data/lib/cjk_index/builder.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "json"
+
+module CJKIndex
+  # Builds a prebuilt inverted index that the browser runtime loads as JSON.
+  #
+  #   builder = CJKIndex::Builder.new(fields: %w[title creator], boosts: { "title" => 3 })
+  #   builder.add("item1.html", "title" => "東京帝國大學", "creator" => "…")
+  #   File.write("search-index.json", builder.to_json)
+  #
+  # Format (version 2):
+  #   {
+  #     "format":  "cjk_index/2",
+  #     "variants": ["ja"],
+  #     "fields":  ["title", "creator"],
+  #     "boosts":  [3, 1],
+  #     "refs":    ["item1.html", ...],
+  #     "lengths": [len(doc0.title), len(doc0.creator), len(doc1.title), ...],
+  #     "tokens":  { "東京": [doc, field, n, pos1, ..., posn, doc, field, n, ...], ... }
+  #   }
+  # doc and field are positions in "refs" and "fields"; n is the number of
+  # occurrences and pos are character offsets in the normalized field text.
+  # Offsets let the runtime require the bigrams of a query to be adjacent
+  # (exact substring match); lengths feed BM25 ranking.
+  class Builder
+    FORMAT = "cjk_index/2"
+
+    # Gap between the values of a multi-valued field, so that a phrase cannot
+    # match across two values.
+    VALUE_GAP = 1
+
+    attr_reader :fields, :refs, :normalizer
+
+    # normalizer must match the one the runtime script was generated with
+    # (Runtime.source(normalizer:)); the index records its tables and the
+    # runtime refuses an index built with different ones.
+    def initialize(fields:, boosts: {}, normalizer: Normalizer.default)
+      raise ArgumentError, "fields must not be empty" if fields.empty?
+
+      @normalizer = normalizer
+      @fields = fields.map(&:to_s)
+      @boosts = @fields.map { |f| (boosts[f] || boosts[f.to_sym] || 1).to_f }
+      @refs = []
+      @lengths = []
+      @postings = Hash.new { |h, k| h[k] = [] }
+    end
+
+    # doc is a Hash of field name => value. Array values are indexed as
+    # separate strings (e.g. multi-valued subject fields).
+    def add(ref, doc)
+      doc_index = @refs.length
+      @refs << ref.to_s
+      @fields.each_with_index do |field, field_index|
+        value = doc[field] || doc[field.to_sym]
+        positions = Hash.new { |h, k| h[k] = [] }
+        base = 0
+        Array(value).each do |v|
+          text = v.to_s
+          Tokenizer.tokens_with_offsets(text, unigrams: true, normalizer: @normalizer)
+                   .each { |t, offset| positions[t] << (base + offset) }
+          base += @normalizer.normalize(text).length + VALUE_GAP
+        end
+        @lengths << [base - VALUE_GAP, 0].max
+        positions.each { |token, offsets| @postings[token].push(doc_index, field_index, offsets.length, *offsets) }
+      end
+      self
+    end
+
+    def to_h
+      {
+        "format" => FORMAT,
+        "variants" => @normalizer.tables,
+        "fields" => @fields,
+        "boosts" => @boosts.map { |b| b == b.to_i ? b.to_i : b },
+        "refs" => @refs,
+        "lengths" => @lengths,
+        "tokens" => @postings.sort.to_h
+      }
+    end
+
+    def to_json(*_args)
+      JSON.generate(to_h)
+    end
+  end
+end

data/lib/cjk_index/jekyll.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "jekyll"
+require_relative "../cjk_index"
+
+module CJKIndex
+  # Jekyll plugin. Add `gem "cjk_index"` to the :jekyll_plugins group (or list
+  # "cjk_index/jekyll" under `plugins:`), then configure in _config.yml:
+  #
+  #   cjk_index:
+  #     output: assets/cjk-index          # where files are written (default)
+  #     variants: [ja]                    # variant tables: ja (default), zh (Traditional -> Simplified)
+  #     indexes:
+  #       - name: posts                   # -> assets/cjk-index/posts.json
+  #         collection: posts             # a Jekyll collection, or...
+  #         fields: [title, tags, content]
+  #         boosts: { title: 3 }
+  #       - name: items
+  #         data: metadata                # ...rows of _data/metadata.csv
+  #         ref: objectid                 # column used as the result's ref
+  #         fields: [title, creator, subject]
+  #       - name: search
+  #         preset: collectionbuilder     # reads CollectionBuilder's own config
+  #
+  # The browser script is written to <output>/cjk-index.js.
+  module Jekyll
+    DEFAULT_OUTPUT = "assets/cjk-index"
+    RUNTIME_NAME = "cjk-index.js"
+
+    # A file whose content is produced at build time (no source file, no Liquid).
+    class GeneratedFile < ::Jekyll::StaticFile
+      def initialize(site, dir, name, content)
+        super(site, site.source, dir, name)
+        @generated_content = content
+      end
+
+      def write(dest)
+        path = destination(dest)
+        return false if File.exist?(path) && File.read(path, encoding: "UTF-8") == @generated_content
+
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, @generated_content)
+        true
+      end
+    end
+
+    class Generator < ::Jekyll::Generator
+      safe true
+      priority :lowest
+
+      def generate(site)
+        config = site.config["cjk_index"]
+        return unless config.is_a?(Hash)
+
+        output = config.fetch("output", DEFAULT_OUTPUT).sub(%r{\A/+}, "").sub(%r{/+\z}, "")
+        normalizer = Normalizer.new(tables: Array(config.fetch("variants", Normalizer::DEFAULT_TABLES)))
+        site.static_files << GeneratedFile.new(site, output, RUNTIME_NAME, Runtime.source(normalizer: normalizer))
+        Array(config["indexes"]).each do |spec|
+          builder = build(site, spec, normalizer)
+          name = "#{spec.fetch('name')}.json"
+          site.static_files << GeneratedFile.new(site, output, name, builder.to_json)
+          ::Jekyll.logger.info "cjk_index:", "#{output}/#{name} (#{builder.refs.length} documents)"
+        end
+      end
+
+      private
+
+      def build(site, spec, normalizer)
+        return Presets::CollectionBuilder.build(site, spec, normalizer) if spec["preset"] == "collectionbuilder"
+        raise ::Jekyll::Errors::FatalException, "cjk_index: unknown preset #{spec['preset']}" if spec["preset"]
+
+        builder = Builder.new(fields: Array(spec.fetch("fields")), boosts: spec.fetch("boosts", {}),
+                              normalizer: normalizer)
+        if spec["collection"]
+          docs = site.collections.fetch(spec["collection"]) do
+            raise ::Jekyll::Errors::FatalException, "cjk_index: no collection #{spec['collection']}"
+          end.docs
+          docs.each { |doc| builder.add(doc.url, document_fields(doc, builder.fields)) }
+        elsif spec["data"]
+          ref = spec.fetch("ref", "id")
+          Array(site.data[spec["data"]]).each do |row|
+            builder.add(row[ref], row) unless row[ref].to_s.empty?
+          end
+        else
+          raise ::Jekyll::Errors::FatalException, "cjk_index: index #{spec['name']} needs collection or data"
+        end
+        builder
+      end
+
+      def document_fields(doc, fields)
+        fields.to_h { |f| [f, f == "content" ? doc.content : doc.data[f]] }
+      end
+    end
+
+    module Presets
+      # Mirrors assets/js/lunr-store.js of CollectionBuilder (collectionbuilder-csv):
+      # items with an objectid, child objects only when the theme asks for them,
+      # fields marked index=true in _data/config-search.csv, and the same "id"
+      # values, so results can be looked up in the existing search store.
+      module CollectionBuilder
+        module_function
+
+        def build(site, spec, normalizer)
+          rows = Array(site.data[site.config["metadata"]])
+          children = site.data.dig("theme", "search-child-objects") == true
+          fields = Array(site.data["config-search"])
+                   .select { |f| f["index"].to_s == "true" }
+                   .map { |f| f["field"] }
+          boosts = spec.fetch("boosts") { fields.first ? { fields.first => 3 } : {} }
+          builder = Builder.new(fields: fields, boosts: boosts, normalizer: normalizer)
+          rows.each do |row|
+            next if row["objectid"].to_s.empty?
+            next if !row["parentid"].to_s.empty? && !children
+
+            builder.add(ref_for(row), row.to_h { |k, v| [k, v.is_a?(String) ? v.split.join(" ") : v] })
+          end
+          builder
+        end
+
+        def ref_for(row)
+          parent = row["parentid"].to_s
+          parent.empty? ? "#{row['objectid']}.html" : "#{parent}.html##{row['objectid']}"
+        end
+      end
+    end
+  end
+end

data/lib/cjk_index/normalizer.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module CJKIndex
+  # Folds text so that spelling variants a reader treats as "the same" compare
+  # equal: NFKC (full-width -> half-width etc.), lowercase, katakana ->
+  # hiragana, character variants from the selected tables, and the iteration
+  # mark 々 expanded to the preceding character.
+  #
+  # The same rules run in the browser (see Runtime), so an index built here
+  # and a query typed there are folded identically.
+  #
+  # Variant tables (data/*.tsv):
+  #   ja  Japanese old forms -> modern forms (圖 -> 図). Default.
+  #   zh  Traditional -> Simplified Chinese, from OpenCC. Opt in: it also
+  #       merges characters that are distinct in Japanese (後 and 后).
+  # With several tables, characters linked by any table fold to one form,
+  # preferring the target of the table listed first: with [ja, zh],
+  # 圖, 図 and 图 all become 図.
+  class Normalizer
+    DATA_DIR = File.expand_path("../../data", __dir__)
+    TABLES = {
+      "ja" => "ja-kyujitai.tsv",
+      "zh" => "zh-hant-hans.tsv"
+    }.freeze
+    DEFAULT_TABLES = %w[ja].freeze
+
+    # Katakana ァ (U+30A1) .. ヶ (U+30F6) map to hiragana by subtracting 0x60.
+    KATAKANA_FIRST = 0x30A1
+    KATAKANA_LAST = 0x30F6
+    KANA_OFFSET = 0x60
+    ITERATION_MARK = "々"
+
+    def self.default
+      @default ||= new
+    end
+
+    def self.read_table(name)
+      file = TABLES.fetch(name) { raise ArgumentError, "unknown variant table #{name.inspect} (#{TABLES.keys.join(', ')})" }
+      File.readlines(File.join(DATA_DIR, file), chomp: true, encoding: "UTF-8")
+          .reject { |line| line.empty? || line.start_with?("#") }
+          .map { |line| line.split("\t", 2) }
+    end
+
+    attr_reader :tables, :variants
+
+    def initialize(tables: DEFAULT_TABLES)
+      @tables = tables.map(&:to_s).uniq.freeze
+      @variants = build_variants.freeze
+    end
+
+    def normalize(text)
+      return "" if text.nil?
+
+      prev = nil
+      text.to_s.unicode_normalize(:nfkc).downcase.each_char.map do |ch|
+        code = ch.ord
+        ch = (code - KANA_OFFSET).chr(Encoding::UTF_8) if code.between?(KATAKANA_FIRST, KATAKANA_LAST)
+        ch = @variants.fetch(ch, ch)
+        ch = prev if ch == ITERATION_MARK && prev
+        prev = ch
+      end.join
+    end
+
+    private
+
+    # Union-find over all pairs; each class folds to the target preferred by
+    # the earliest table (lowest code point among equals, for determinism).
+    def build_variants
+      parent = {}
+      find = lambda do |x|
+        parent[x] ||= x
+        parent[x] = find.call(parent[x]) unless parent[x] == x
+        parent[x]
+      end
+      rank = {}
+      @tables.each_with_index do |name, priority|
+        self.class.read_table(name).each do |from, to|
+          parent[find.call(from)] = find.call(to)
+          rank[to] = [rank.fetch(to, [priority, to.ord]), [priority, to.ord]].min
+        end
+      end
+      classes = parent.keys.group_by { |x| find.call(x) }
+      classes.each_value.with_object({}) do |members, map|
+        canonical = members.min_by { |m| rank.fetch(m, [Float::INFINITY, m.ord]) }
+        members.each { |m| map[m] = canonical unless m == canonical }
+      end
+    end
+  end
+end

data/lib/cjk_index/runtime.js

@@ -0,0 +1,318 @@
+/*!
+ * cjk_index runtime — queries an index built by the cjk_index Ruby gem.
+ * Generated file: the variant table and character classes below are written
+ * by CJKIndex::Runtime so that they match the Ruby side exactly.
+ * License: MIT
+ */
+(function (global) {
+  'use strict';
+
+  var VARIANTS = /*@VARIANTS@*/{};
+  var TABLES = /*@TABLES@*/[];
+  var CJK = new RegExp(/*@CJK@*/'', 'u');
+  var SEPARATOR = new RegExp(/*@SEPARATOR@*/'', 'u');
+  var FORMAT = 'cjk_index/2';
+
+  // BM25 parameters (the usual defaults, as in Lucene and Pagefind).
+  var K1 = 1.2;
+  var B = 0.75;
+  // Score multiplier for matches found by prefix or typo expansion.
+  var EXPANDED = 0.6;
+
+  // Folds one character; prev is the previous folded character (for 々).
+  function foldChar(ch, prev) {
+    var c = ch.codePointAt(0);
+    if (c >= 0x30a1 && c <= 0x30f6) ch = String.fromCodePoint(c - 0x60);
+    if (VARIANTS[ch]) ch = VARIANTS[ch];
+    if (ch === '々' && prev) ch = prev;
+    return ch;
+  }
+
+  function normalize(text) {
+    if (text === null || text === undefined) return '';
+    var out = '';
+    var prev = null;
+    for (var ch of String(text).normalize('NFKC').toLowerCase()) {
+      prev = foldChar(ch, prev);
+      out += prev;
+    }
+    return out;
+  }
+
+  // Same loop as CJKIndex::Tokenizer.phrases in Ruby: split at separators
+  // into phrases of [token, offset]; CJK runs become bigrams (plus unigrams
+  // when asked), other runs stay words. Offsets count code points.
+  function phrases(text, unigrams) {
+    var result = [];
+    var phrase = [];
+    var run = [];
+    var runCjk = null;
+    var runStart = 0;
+    var flush = function () {
+      if (!run.length) return;
+      if (runCjk) {
+        if (run.length === 1 || unigrams) {
+          for (var i = 0; i < run.length; i++) phrase.push([run[i], runStart + i]);
+        }
+        for (var j = 0; j < run.length - 1; j++) phrase.push([run[j] + run[j + 1], runStart + j]);
+      } else {
+        phrase.push([run.join(''), runStart]);
+      }
+      run = [];
+    };
+    var offset = 0;
+    for (var ch of normalize(text)) {
+      if (SEPARATOR.test(ch)) {
+        flush();
+        if (phrase.length) result.push(phrase);
+        phrase = [];
+        runCjk = null;
+      } else {
+        var cjk = CJK.test(ch);
+        if (runCjk !== null && cjk !== runCjk) flush();
+        if (!run.length) runStart = offset;
+        runCjk = cjk;
+        run.push(ch);
+      }
+      offset++;
+    }
+    flush();
+    if (phrase.length) result.push(phrase);
+    return result;
+  }
+
+  function tokenize(text, unigrams) {
+    var out = [];
+    phrases(text, unigrams).forEach(function (p) { p.forEach(function (t) { out.push(t[0]); }); });
+    return out;
+  }
+
+  // Substring test on normalized text, ignoring whitespace. Useful for
+  // filters that do not need the index (e.g. a browse page).
+  function includes(haystack, needle) {
+    var strip = function (s) { return normalize(s).replace(/\s+/g, ''); };
+    return strip(haystack).indexOf(strip(needle)) !== -1;
+  }
+
+  function isLatin(token) { return !CJK.test(token); }
+
+  // true when a and b differ by exactly one insertion, deletion or substitution.
+  function oneEdit(a, b) {
+    if (a === b || Math.abs(a.length - b.length) > 1) return false;
+    var i = 0;
+    while (i < a.length && i < b.length && a[i] === b[i]) i++;
+    if (a.length === b.length) return a.slice(i + 1) === b.slice(i + 1);
+    if (a.length > b.length) return a.slice(i + 1) === b.slice(i);
+    return a.slice(i) === b.slice(i + 1);
+  }
+
+  function Index(data, options) {
+    if (!data || data.format !== FORMAT) {
+      throw new Error('cjk_index: unsupported index format ' + (data && data.format));
+    }
+    if (JSON.stringify(data.variants) !== JSON.stringify(TABLES)) {
+      throw new Error('cjk_index: index built with variant tables ' + JSON.stringify(data.variants) +
+        ' but this script folds with ' + JSON.stringify(TABLES) + '; rebuild both together');
+    }
+    options = options || {};
+    this.prefix = options.prefix !== false;          // last Latin word matches as a prefix
+    this.typos = options.typos !== false;            // Latin words of 5+ letters allow one edit
+    this.fields = data.fields;
+    this.boosts = data.boosts;
+    this.refs = data.refs;
+    this.lengths = data.lengths;
+    this.tokens = data.tokens;
+    var nf = this.fields.length;
+    var avg = new Array(nf).fill(0);
+    for (var i = 0; i < this.lengths.length; i++) avg[i % nf] += this.lengths[i];
+    this.avgLength = avg.map(function (s) { return s / Math.max(1, data.refs.length) || 1; });
+    this.latinTokens = Object.keys(this.tokens).filter(isLatin).sort();
+  }
+
+  // Map of "doc,field" key -> Set of positions, merged over the alternatives.
+  Index.prototype._positions = function (tokens) {
+    var nf = this.fields.length;
+    var map = new Map();
+    for (var t = 0; t < tokens.length; t++) {
+      var p = this.tokens[tokens[t]];
+      if (!p) continue;
+      for (var i = 0; i < p.length;) {
+        var key = p[i] * nf + p[i + 1];
+        var n = p[i + 2];
+        var set = map.get(key);
+        if (!set) { set = new Set(); map.set(key, set); }
+        for (var k = 0; k < n; k++) set.add(p[i + 3 + k]);
+        i += 3 + n;
+      }
+    }
+    return map;
+  };
+
+  // Alternatives for one query token: itself, and for Latin words the index
+  // words it is a prefix of (last word only) or one edit away from.
+  Index.prototype._alternatives = function (token, isLast) {
+    if (!isLatin(token)) return { tokens: [token], expanded: false };
+    var out = this.tokens[token] ? [token] : [];
+    var keys = this.latinTokens;
+    if (this.prefix && isLast) {
+      var lo = 0, hi = keys.length;
+      while (lo < hi) { var mid = (lo + hi) >> 1; if (keys[mid] < token) lo = mid + 1; else hi = mid; }
+      for (var i = lo; i < keys.length && keys[i].indexOf(token) === 0; i++) if (keys[i] !== token) out.push(keys[i]);
+    }
+    if (this.typos && !this.tokens[token] && Array.from(token).length >= 5) {
+      keys.forEach(function (k) { if (oneEdit(token, k)) out.push(k); });
+    }
+    return { tokens: out, expanded: out.length > 0 && out[0] !== token };
+  };
+
+  // Every phrase of the query must occur in the document (in any field),
+  // with its tokens adjacent and in order. Returns [{ref, score, fields}]
+  // with the best match first; fields lists the names of the fields that matched.
+  Index.prototype.search = function (query) {
+    var self = this;
+    var nf = this.fields.length;
+    var n = this.refs.length;
+    var qp = phrases(query, false);
+    if (!qp.length) return [];
+    var scores = new Map();
+    var matchedFields = new Map();
+    var docsSoFar = null;
+
+    for (var q = 0; q < qp.length; q++) {
+      var phrase = qp[q];
+      var expanded = false;
+      var parts = phrase.map(function (tok, i) {
+        var alt = self._alternatives(tok[0], q === qp.length - 1 && i === phrase.length - 1);
+        if (alt.expanded) expanded = true;
+        return { offset: tok[1], positions: self._positions(alt.tokens) };
+      });
+      if (parts.some(function (p) { return p.positions.size === 0; })) return [];
+      parts.sort(function (a, b) { return a.positions.size - b.positions.size; });
+      var rare = parts[0];
+      var hits = new Map();                       // "doc,field" key -> phrase occurrences
+      rare.positions.forEach(function (set, key) {
+        var tf = 0;
+        set.forEach(function (pos) {
+          var start = pos - rare.offset;
+          for (var i = 1; i < parts.length; i++) {
+            var other = parts[i].positions.get(key);
+            if (!other || !other.has(start + parts[i].offset)) return;
+          }
+          tf++;
+        });
+        if (tf) hits.set(key, tf);
+      });
+      var docs = new Set();
+      hits.forEach(function (tf, key) { docs.add(Math.floor(key / nf)); });
+      if (!docs.size) return [];
+      if (docsSoFar) {
+        docs.forEach(function (d) { if (!docsSoFar.has(d)) docs.delete(d); });
+        if (!docs.size) return [];
+      }
+      docsSoFar = docs;
+      var idf = Math.log(1 + (n - docs.size + 0.5) / (docs.size + 0.5));
+      var weight = expanded ? EXPANDED : 1;
+      hits.forEach(function (tf, key) {
+        var doc = Math.floor(key / nf), field = key % nf;
+        var norm = 1 - B + B * self.lengths[key] / self.avgLength[field];
+        var s = weight * self.boosts[field] * idf * (tf * (K1 + 1)) / (tf + K1 * norm);
+        scores.set(doc, (scores.get(doc) || 0) + s);
+        var f = matchedFields.get(doc);
+        if (!f) { f = new Set(); matchedFields.set(doc, f); }
+        f.add(self.fields[field]);
+      });
+    }
+
+    var results = [];
+    docsSoFar.forEach(function (doc) {
+      results.push({ ref: self.refs[doc], score: scores.get(doc), fields: Array.from(matchedFields.get(doc)), doc: doc });
+    });
+    results.sort(function (a, b) { return b.score - a.score || a.doc - b.doc; });
+    return results.map(function (r) { return { ref: r.ref, score: r.score, fields: r.fields }; });
+  };
+
+  function escapeHtml(s) {
+    return s.replace(/[&<>"']/g, function (c) {
+      return { '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' }[c];
+    });
+  }
+
+  // [start, end) ranges, in code points of the original text, where a phrase
+  // of the query occurs after normalization.
+  function matchRanges(text, query) {
+    var chars = Array.from(String(text === null || text === undefined ? '' : text));
+    var norm = [];
+    var origin = [];
+    var prev = null;
+    chars.forEach(function (ch, i) {
+      for (var c of ch.normalize('NFKC').toLowerCase()) {
+        prev = foldChar(c, prev);
+        norm.push(prev);
+        origin.push(i);
+      }
+    });
+    var normText = norm.join('');
+    var ranges = [];
+    var needles = [];
+    var current = '';
+    for (var qc of normalize(query)) {
+      if (SEPARATOR.test(qc)) { if (current) needles.push(current); current = ''; } else current += qc;
+    }
+    if (current) needles.push(current);
+    needles.forEach(function (needle) {
+      var len = Array.from(needle).length;
+      var from = 0;
+      var at;
+      while ((at = normText.indexOf(needle, from)) !== -1) {
+        var cp = Array.from(normText.slice(0, at)).length;
+        ranges.push([origin[cp], origin[cp + len - 1] + 1]);
+        from = at + needle.length;
+      }
+    });
+    ranges.sort(function (a, b) { return a[0] - b[0]; });
+    var merged = [];
+    ranges.forEach(function (r) {
+      var last = merged[merged.length - 1];
+      if (last && r[0] <= last[1]) last[1] = Math.max(last[1], r[1]); else merged.push(r.slice());
+    });
+    return { chars: chars, ranges: merged };
+  }
+
+  // HTML-escaped text with every match of the query wrapped in <mark>.
+  // 「東京帝國大學」 is marked for the query 「帝国大学」.
+  function highlight(text, query) {
+    var m = matchRanges(text, query);
+    var out = '';
+    var at = 0;
+    m.ranges.forEach(function (r) {
+      out += escapeHtml(m.chars.slice(at, r[0]).join('')) + '<mark>' + escapeHtml(m.chars.slice(r[0], r[1]).join('')) + '</mark>';
+      at = r[1];
+    });
+    return out + escapeHtml(m.chars.slice(at).join(''));
+  }
+
+  // A highlighted window of about `length` characters around the first match.
+  function excerpt(text, query, length) {
+    length = length || 80;
+    var m = matchRanges(text, query);
+    if (!m.ranges.length) return m.chars.length > length ? escapeHtml(m.chars.slice(0, length).join('')) + '…' : escapeHtml(m.chars.join(''));
+    var start = Math.max(0, m.ranges[0][0] - Math.floor(length / 3));
+    var end = Math.min(m.chars.length, start + length);
+    var part = m.chars.slice(start, end).join('');
+    return (start > 0 ? '…' : '') + highlight(part, query) + (end < m.chars.length ? '…' : '');
+  }
+
+  function load(url, options) {
+    return fetch(url).then(function (res) {
+      if (!res.ok) throw new Error('cjk_index: ' + url + ' returned ' + res.status);
+      return res.json();
+    }).then(function (data) { return new Index(data, options); });
+  }
+
+  var api = {
+    tables: TABLES, normalize: normalize, tokenize: tokenize, phrases: phrases, includes: includes,
+    highlight: highlight, excerpt: excerpt, Index: Index, load: load
+  };
+  if (typeof module !== 'undefined' && module.exports) module.exports = api;
+  else global.CJKIndex = api;
+})(typeof window !== 'undefined' ? window : globalThis);

data/lib/cjk_index/runtime.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "json"
+
+module CJKIndex
+  # Writes the browser script. The normalization table and character classes
+  # come from Normalizer and Tokenizer, so Ruby (indexing) and JavaScript
+  # (querying) cannot drift apart.
+  module Runtime
+    TEMPLATE_PATH = File.expand_path("runtime.js", __dir__)
+
+    module_function
+
+    def source(normalizer: Normalizer.default)
+      File.read(TEMPLATE_PATH, encoding: "UTF-8")
+          # Block form: a replacement string would treat its backslashes as
+          # back-references and drop half of the regexp escapes.
+          .sub("/*@VARIANTS@*/{}") { JSON.generate(normalizer.variants) }
+          .sub("/*@TABLES@*/[]") { JSON.generate(normalizer.tables) }
+          .sub("/*@CJK@*/''") { JSON.generate(Tokenizer.char_class(Tokenizer::CJK_RANGES)) }
+          .sub("/*@SEPARATOR@*/''") do
+            JSON.generate(Tokenizer.char_class(Tokenizer::SEPARATOR_RANGES, Tokenizer::SEPARATOR_CHARS))
+          end
+    end
+  end
+end