cjk_index 0.2.0 → 1.1.0

data/lib/cjk_index/postings.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+module CJKIndex
+  # The compact encoding of a posting list in format 4, shared by Builder
+  # (writing) and Searcher (reading); runtime.js decodes it the same way.
+  #
+  # A list is a sequence of entries, ordered by document and field. Each
+  # entry is written as numbers:
+  #
+  #   key gap, n, first position, gap to the next position, ...
+  #
+  # where key = document * number of fields + field, and the key gap is the
+  # difference to the key of the previous entry minus 1 (the first entry
+  # counts from -1). n is the number of positions. Each number is written in
+  # printable ASCII: the last digit in base 60 from the first 60 characters
+  # of ALPHABET, the digits before it in base 30 from the other 30, most
+  # significant first. 0..59 take one character, up to 1,799 two, up to
+  # 53,999 three. ALPHABET has neither " nor \, so a list is a JSON string
+  # without escapes.
+  module Postings
+    ALPHABET = ((0x23..0x7d).map(&:chr) - ["\\"]).join.freeze
+    LAST = 60
+    LEAD = 30
+    # The characters that end a number are the bytes below LEAD_BYTE
+    # (FINAL_BYTES in the notation of String#count).
+    LEAD_BYTE = ALPHABET.getbyte(LAST)
+    FINAL_BYTES = "#-[\\]-_"
+    # byte => digit; 0...60 ends a number, 60...90 leads.
+    DIGIT = Array.new(256, -1).tap { |t| ALPHABET.each_byte.with_index { |b, i| t[b] = i } }.freeze
+
+    def self.number_slow(value)
+      s = ALPHABET[value % LAST]
+      value /= LAST
+      while value.positive?
+        s = ALPHABET[LAST + (value % LEAD)] + s
+        value /= LEAD
+      end
+      s
+    end
+
+    # Every number of up to three characters, written once.
+    NUMBERS = Array.new(LAST * LEAD * LEAD) { |v| number_slow(v).freeze }.freeze
+
+    module_function
+
+    def number(value) = NUMBERS[value] || number_slow(value)
+
+    # Appends one entry to out. offsets are ascending positions.
+    def append(out, key_gap, offsets)
+      out << number(key_gap) << number(offsets.length)
+      last = 0
+      offsets.each do |pos|
+        out << number(pos - last)
+        last = pos
+      end
+      out
+    end
+
+    # All numbers of a list.
+    def numbers(str)
+      out = []
+      acc = 0
+      str.each_byte do |b|
+        d = DIGIT[b]
+        if d >= LAST then acc = (acc * LEAD) + d - LAST
+        else
+          out << ((acc * LAST) + d)
+          acc = 0
+        end
+      end
+      out
+    end
+
+    # Yields key, start, body and stop for each entry: its key, and the byte
+    # offsets where the entry starts, where its key gap ends and where it
+    # ends. Loops over bytes rather than numbers, so that the changes below
+    # can copy the rest of a list as it is.
+    def each_entry(str)
+      return enum_for(:each_entry, str) unless block_given?
+
+      key = -1
+      i = 0
+      len = str.bytesize
+      while i < len
+        start = i
+        acc = 0
+        while (d = DIGIT[str.getbyte(i)]) >= LAST
+          acc = (acc * LEAD) + d - LAST
+          i += 1
+        end
+        key += (acc * LAST) + d + 1
+        i += 1
+        body = i
+        acc = 0
+        while (d = DIGIT[str.getbyte(i)]) >= LAST
+          acc = (acc * LEAD) + d - LAST
+          i += 1
+        end
+        n = (acc * LAST) + d
+        i += 1
+        # Skip n numbers: each ends with a byte below LEAD_BYTE. Many at a
+        # time are counted by String#count, which runs in C.
+        while n > 8
+          t = str.byteslice(i, n).count(FINAL_BYTES)
+          i += n
+          n -= t
+        end
+        while n.positive?
+          n -= 1 if str.getbyte(i) < LEAD_BYTE
+          i += 1
+        end
+        yield key, start, body, i
+      end
+    end
+
+    # Key of the last entry, -1 for an empty list.
+    def last_key(str)
+      key = -1
+      each_entry(str) { |k| key = k }
+      key
+    end
+
+    # Each key of a list, in order.
+    def each_key(str)
+      return enum_for(:each_key, str) unless block_given?
+
+      each_entry(str) { |key| yield key }
+    end
+
+    # The list without the entries whose keys are in range (consecutive).
+    def delete(str, range)
+      before = -1
+      from = nil
+      each_entry(str) do |key, start, body|
+        if range.cover?(key)
+          from ||= start
+        elsif from
+          return str.byteslice(0, from) + number(key - before - 1) + str.byteslice(body..)
+        elsif key > range.end
+          return str
+        else
+          before = key
+        end
+      end
+      from ? str.byteslice(0, from) : str
+    end
+
+    # The list with an entry for key (not yet in it) and offsets inserted in order.
+    def insert(str, key, offsets)
+      before = -1
+      each_entry(str) do |k, start, body|
+        if k > key
+          entry = append(+"", key - before - 1, offsets)
+          return str.byteslice(0, start) + entry + number(k - key - 1) + str.byteslice(body..)
+        end
+        before = k
+      end
+      append(+str, key - before - 1, offsets)
+    end
+
+    # The list with each key's document replaced by numbering[document], which
+    # keeps the order (removed documents have no entries).
+    def renumber(str, fields, numbering)
+      out = +""
+      old_before = -1
+      new_before = -1
+      copied = 0 # str up to here is in out or to be copied as it is
+      each_entry(str) do |key, start, body|
+        new_key = (numbering[key / fields] * fields) + (key % fields)
+        if new_key - new_before != key - old_before
+          out << str.byteslice(copied, start - copied) << number(new_key - new_before - 1)
+          copied = body
+        end
+        old_before = key
+        new_before = new_key
+      end
+      copied.zero? ? str : out << str.byteslice(copied..)
+    end
+
+    # The flat list of format 3, [doc, field, n, pos, ..., doc, ...].
+    def decode(str, fields)
+      nums = numbers(str)
+      out = []
+      key = -1
+      i = 0
+      while i < nums.length
+        key += nums[i] + 1
+        n = nums[i + 1]
+        out << (key / fields) << (key % fields) << n
+        pos = 0
+        j = i + 2
+        stop = j + n
+        while j < stop
+          pos += nums[j]
+          out << pos
+          j += 1
+        end
+        i = stop
+      end
+      out
+    end
+
+    # A flat list of format 2 or 3 in this encoding.
+    def encode(flat, fields)
+      out = +""
+      last = -1
+      i = 0
+      while i < flat.length
+        key = (flat[i] * fields) + flat[i + 1]
+        n = flat[i + 2]
+        append(out, key - last - 1, flat[i + 3, n])
+        last = key
+        i += 3 + n
+      end
+      out
+    end
+  end
+end

data/lib/cjk_index/rake_task.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "rake"
+require "rake/tasklib"
+require "fileutils"
+require_relative "cli"
+
+module CJKIndex
+  # A Rake task that builds an index (and the browser script) from a CSV,
+  # JSON or JSON Lines file, usable from any Ruby build (Rails, Sinatra, a
+  # plain Rakefile). It does what `cjk_index build` does.
+  #
+  #   require "cjk_index/rake_task"
+  #
+  #   CJKIndex::RakeTask.new(:search_index) do |t|
+  #     t.source  = "data/items.csv"
+  #     t.output  = "public/search/items.json"
+  #     t.runtime = "public/search/cjk-index.js"   # optional
+  #     t.ref     = "id"
+  #     t.fields  = %w[title creator]
+  #     t.boosts  = { "title" => 3 }
+  #     t.filters = { "type" => "value", "date" => "year" }
+  #   end
+  #
+  #   rake search_index
+  #
+  # records can be set instead of source: an Array of Hashes, or an object
+  # that responds to call and returns one (e.g. a lambda reading the database).
+  class RakeTask < Rake::TaskLib
+    attr_accessor :name, :source, :records, :output, :runtime, :ref, :fields, :boosts, :filters, :variants, :format,
+                  :description
+
+    def initialize(name = :cjk_index)
+      super()
+      @name = name
+      @boosts = {}
+      @filters = {}
+      @variants = Normalizer::DEFAULT_TABLES
+      @description = "Build the CJK search index"
+      yield self if block_given?
+      define
+    end
+
+    # Builds the index now; the task calls this.
+    def build
+      raise CLI::Error, "#{name}: set output" unless output
+
+      data = records.respond_to?(:call) ? records.call : records
+      data ||= source ? CLI.read_records(source.to_s, format || CLI.format_of(source)) : raise(CLI::Error, "#{name}: set source or records")
+      builder = CLI.build_index(records: data, ref: ref, fields: fields, boosts: boosts, filters: filters, variants: variants)
+      write(output, builder.to_json)
+      write(runtime, Runtime.source(normalizer: builder.normalizer)) if runtime
+      builder
+    end
+
+    private
+
+    def define
+      desc description
+      task(name) do
+        builder = build
+        puts "#{output}: #{builder.size} records"
+      end
+    end
+
+    def write(path, content)
+      FileUtils.mkdir_p(File.dirname(path.to_s))
+      File.write(path.to_s, content)
+    end
+  end
+end

data/lib/cjk_index/runtime.js

@@ -9,9 +9,49 @@
 
   var VARIANTS = /*@VARIANTS@*/{};
   var TABLES = /*@TABLES@*/[];
+  var DIGEST = /*@DIGEST@*/'';
   var CJK = new RegExp(/*@CJK@*/'', 'u');
   var SEPARATOR = new RegExp(/*@SEPARATOR@*/'', 'u');
-  var FORMATS = ['cjk_index/2', 'cjk_index/3'];
+  var FORMATS = ['cjk_index/2', 'cjk_index/3', 'cjk_index/4'];
+  var COMPACT = 'cjk_index/4';
+  var hasOwn = Object.prototype.hasOwnProperty;
+
+  // Posting lists of format 4 (CJKIndex::Postings in Ruby): numbers written
+  // with 90 printable characters, '#' to '}' without '\'. The first 60 are the
+  // last digit of a number (base 60), the other 30 lead it (base 30).
+  var DIGIT = [];
+  for (var code = 0x23, digit = 0; code <= 0x7d; code++) if (code !== 0x5c) DIGIT[code] = digit++;
+
+  // A posting list of format 4 as the flat list of format 3:
+  // [doc, field, n, pos, ..., doc, field, n, ...].
+  function decodePostings(s, nf) {
+    var out = [];
+    var key = -1, state = 0, left = 0, pos = 0, acc = 0;
+    for (var i = 0; i < s.length; i++) {
+      var d = DIGIT[s.charCodeAt(i)];
+      if (d >= 60) { acc = acc * 30 + d - 60; continue; }
+      var v = acc * 60 + d;
+      acc = 0;
+      if (state === 0) { key += v + 1; out.push(Math.floor(key / nf), key % nf); state = 1; }
+      else if (state === 1) { out.push(v); left = v; pos = 0; state = 2; }
+      else { pos += v; out.push(pos); if (--left === 0) state = 0; }
+    }
+    return out;
+  }
+
+  // Compares strings by code point, as Ruby compares UTF-8 strings (the
+  // order of the shards). JavaScript's < compares UTF-16 code units, which
+  // puts characters beyond U+FFFF before U+E000-U+FFFF.
+  function compareCodePoints(a, b) {
+    var i = 0, j = 0;
+    while (i < a.length && j < b.length) {
+      var x = a.codePointAt(i), y = b.codePointAt(j);
+      if (x !== y) return x < y ? -1 : 1;
+      i += x > 0xffff ? 2 : 1;
+      j += y > 0xffff ? 2 : 1;
+    }
+    return (i < a.length ? 1 : 0) - (j < b.length ? 1 : 0);
+  }
 
   // BM25 parameters (the usual defaults, as in Lucene and Pagefind).
   var K1 = 1.2;
@@ -119,6 +159,10 @@
       throw new Error('cjk_index: index built with variant tables ' + JSON.stringify(data.variants) +
         ' but this script folds with ' + JSON.stringify(TABLES) + '; rebuild both together');
     }
+    if (data.variant_digest !== undefined && data.variant_digest !== DIGEST) {
+      throw new Error('cjk_index: index built with other contents of the variant tables ' +
+        JSON.stringify(TABLES) + ' (an older cjk_index?); rebuild it');
+    }
     options = options || {};
     this.prefix = options.prefix !== false;          // last Latin word matches as a prefix
     this.typos = options.typos !== false;            // Latin words of 5+ letters allow one edit
@@ -126,15 +170,30 @@
     this.boosts = data.boosts;
     this.refs = data.refs;
     this.lengths = data.lengths;
-    this.tokens = data.tokens;
+    this.tokens = data.tokens || null;
+    this.encoded = data.format === COMPACT;
+    this.decoded = new Map();
+    if (data.shards) {
+      // Split index: the manifest lists the first token and file of each shard.
+      this.loadShard = options.loadShard;
+      if (!this.loadShard) throw new Error('cjk_index: a split index needs options.loadShard (CJKIndex.load sets it)');
+      this.shardFirst = data.shards.map(function (s) { return s[0]; });
+      this.shardFile = data.shards.map(function (s) { return s[1]; });
+      this.shards = new Map();     // file -> tokens, once loaded
+      this.pending = new Map();    // file -> Promise while loading
+    }
     var nf = this.fields.length;
     var avg = new Array(nf).fill(0);
     for (var i = 0; i < this.lengths.length; i++) avg[i % nf] += this.lengths[i];
     this.avgLength = avg.map(function (s) { return s / Math.max(1, data.refs.length) || 1; });
-    this.latinTokens = Object.keys(this.tokens).filter(isLatin).sort();
+    // A split index lists its Latin tokens in a file, fetched by prepare
+    // only for typo expansion.
+    this.latinFile = this.tokens ? null : data.latin;
+    if (this.tokens) this._setLatin(Object.keys(this.tokens).filter(isLatin));
     this.slotOf = new Map();
     for (var d = 0; d < this.refs.length; d++) if (!this.slotOf.has(this.refs[d])) this.slotOf.set(this.refs[d], d);
-    // Filter values by document: keys (value filters) or a year, with the
+    // Filter values by document: a year, or the ids of values (value
+    // filters), with the folded key of each id, the ids of each key, the
     // form to show for each key and its rank of first appearance.
     this.filters = {};
     var filters = data.filters || {};
@@ -142,19 +201,22 @@
       var f = filters[name];
       var docs = f.docs;
       var display = new Map();
+      var keys = null;
+      var ids = new Map();
       if (f.kind !== 'year') {
-        var keys = f.values.map(filterKey);
+        keys = f.values.map(filterKey);
         f.values.forEach(function (v, i) { if (!display.has(keys[i])) display.set(keys[i], v); });
-        docs = f.docs.map(function (ids) { return ids.map(function (id) { return keys[id]; }); });
+        keys.forEach(function (key, id) { if (!ids.has(key)) ids.set(key, []); ids.get(key).push(id); });
       }
       var rank = new Map();
       docs.forEach(function (value) {
-        (f.kind === 'year' ? (value === null ? [] : [value]) : value).forEach(function (key) {
+        (f.kind === 'year' ? (value === null ? [] : [value]) : value).forEach(function (v) {
+          var key = f.kind === 'year' ? v : keys[v];
           if (!rank.has(key)) rank.set(key, rank.size);
         });
       });
       if (f.kind === 'year') rank.forEach(function (r, year) { display.set(year, year); });
-      this.filters[name] = { kind: f.kind, docs: docs, display: display, rank: rank };
+      this.filters[name] = { kind: f.kind, docs: docs, keys: keys, ids: ids, display: display, rank: rank };
     }, this);
   }
 
@@ -175,8 +237,12 @@
             (max === null || max === undefined || year <= max);
         }]);
       } else {
-        var keys = new Set((Array.isArray(c) ? c : [c]).map(filterKey));
-        out.push([f, function (values) { return values.some(function (k) { return keys.has(k); }); }]);
+        // Compares value ids, not folded strings.
+        var ids = new Set();
+        (Array.isArray(c) ? c : [c]).forEach(function (v) {
+          (f.ids.get(filterKey(v)) || []).forEach(function (id) { ids.add(id); });
+        });
+        out.push([f, function (list) { return list.some(function (id) { return ids.has(id); }); }]);
       }
     });
     return out;
@@ -206,7 +272,8 @@
     var counts = new Map();
     docs.forEach(function (doc) {
       var v = f.docs[doc];
-      (f.kind === 'year' ? (v === null ? [] : [v]) : v).forEach(function (key) {
+      (f.kind === 'year' ? (v === null ? [] : [v]) : v).forEach(function (x) {
+        var key = f.kind === 'year' ? x : f.keys[x];
         counts.set(key, (counts.get(key) || 0) + 1);
       });
     });
@@ -215,12 +282,109 @@
     return out.map(function (e) { return { value: f.display.get(e[0]), count: e[1] }; });
   };
 
+  Index.prototype._setLatin = function (tokens) {
+    this.latinTokens = tokens.slice().sort();
+    this.latinSet = new Set(this.latinTokens);
+  };
+
+  // Number of the shard that holds token (split index), or -1.
+  Index.prototype._shardIndex = function (token) {
+    var lo = 0, hi = this.shardFirst.length;
+    while (lo < hi) {
+      var mid = (lo + hi) >> 1;
+      if (compareCodePoints(this.shardFirst[mid], token) <= 0) lo = mid + 1; else hi = mid;
+    }
+    return lo - 1;
+  };
+
+  function notLoaded(what) {
+    return new Error('cjk_index: ' + what + ' not loaded; use searchAsync(query) or await prepare(query)');
+  }
+
+  // Tokens of shard i, or undefined when it is not loaded: then its file is
+  // added to missing (prepare), or an error is thrown (search).
+  Index.prototype._shard = function (i, missing) {
+    var tokens = this.shards.get(this.shardFile[i]);
+    if (tokens) return tokens;
+    if (!missing) throw notLoaded('the shard ' + this.shardFile[i] + ' is');
+    missing.files.add(this.shardFile[i]);
+    return undefined;
+  };
+
+  // The object that holds token's posting list if it is in the index; null
+  // when it cannot be; undefined when its shard is not loaded (see _shard).
+  Index.prototype._table = function (token, missing) {
+    if (this.tokens) return this.tokens;
+    var i = this._shardIndex(token);
+    return i < 0 ? null : this._shard(i, missing);
+  };
+
+  // true or false, or undefined when not known yet (prepare).
+  Index.prototype._has = function (token, missing) {
+    var table = this._table(token, missing);
+    return table === undefined ? undefined : !!table && hasOwn.call(table, token);
+  };
+
+  // The posting list of a token as [doc, field, n, pos, ...], or null.
+  Index.prototype._postings = function (token) {
+    var table = this._table(token);
+    if (!table || !hasOwn.call(table, token)) return null;
+    if (!this.encoded) return table[token];
+    var p = this.decoded.get(token);
+    if (!p) { p = decodePostings(table[token], this.fields.length); this.decoded.set(token, p); }
+    return p;
+  };
+
+  Index.prototype._load = function (file, done) {
+    var self = this;
+    var p = this.pending.get(file);
+    if (!p) {
+      p = Promise.resolve(this.loadShard(file)).then(done);
+      p.catch(function () { self.pending.delete(file); });
+      this.pending.set(file, p);
+    }
+    return p;
+  };
+
+  // Fetches what a search for query needs from a split index, so that
+  // search(query) can then run: the shards of its tokens, of the words a
+  // last Latin word is a prefix of, and, for a Latin word of 5+ letters not
+  // in the index, the list of Latin tokens (once) and the shards of the
+  // words one edit away. Resolves to the index; at once for one file.
+  Index.prototype.prepare = function (query) {
+    var self = this;
+    if (!this.shardFirst) return Promise.resolve(this);
+    var missing = { files: new Set(), latin: false };
+    var qp = phrases(query, false);
+    qp.forEach(function (phrase, q) {
+      phrase.forEach(function (tok, i) {
+        self._alternatives(tok[0], q === qp.length - 1 && i === phrase.length - 1, missing).tokens.forEach(function (t) {
+          self._table(t, missing);
+        });
+      });
+    });
+    var loads = Array.from(missing.files).map(function (file) {
+      return self._load(file, function (tokens) { self.shards.set(file, tokens); });
+    });
+    if (missing.latin) loads.push(this._load(this.latinFile, function (tokens) { self._setLatin(tokens); }));
+    if (!loads.length) return Promise.resolve(this);
+    // What was loaded may show more to load (typo candidates).
+    return Promise.all(loads).then(function () { return self.prepare(query); });
+  };
+
+  // search, after fetching the shards it needs. Works for every index, so
+  // a page can use it whether or not the index is split.
+  Index.prototype.searchAsync = function (query, options) {
+    var self = this;
+    return this.prepare(query).then(function () { return self.search(query, options); });
+  };
+
   // Map of "doc,field" key -> Set of positions, merged over the alternatives.
   Index.prototype._positions = function (tokens) {
     var nf = this.fields.length;
     var map = new Map();
     for (var t = 0; t < tokens.length; t++) {
-      var p = this.tokens[tokens[t]];
+      var p = this._postings(tokens[t]);
       if (!p) continue;
       for (var i = 0; i < p.length;) {
         var key = p[i] * nf + p[i + 1];
@@ -236,17 +400,36 @@
 
   // Alternatives for one query token: itself, and for Latin words the index
   // words it is a prefix of (last word only) or one edit away from.
-  Index.prototype._alternatives = function (token, isLast) {
+  // With missing (prepare), what is not loaded yet is noted there and left out.
+  Index.prototype._alternatives = function (token, isLast, missing) {
     if (!isLatin(token)) return { tokens: [token], expanded: false };
-    var out = this.tokens[token] ? [token] : [];
-    var keys = this.latinTokens;
+    var has = this.tokens ? this.latinSet.has(token) : this._has(token, missing);
+    var out = has ? [token] : [];
     if (this.prefix && isLast) {
-      var lo = 0, hi = keys.length;
-      while (lo < hi) { var mid = (lo + hi) >> 1; if (keys[mid] < token) lo = mid + 1; else hi = mid; }
-      for (var i = lo; i < keys.length && keys[i].indexOf(token) === 0; i++) if (keys[i] !== token) out.push(keys[i]);
+      if (this.tokens) {
+        var keys = this.latinTokens;
+        var lo = 0, hi = keys.length;
+        while (lo < hi) { var mid = (lo + hi) >> 1; if (keys[mid] < token) lo = mid + 1; else hi = mid; }
+        for (var i = lo; i < keys.length && keys[i].indexOf(token) === 0; i++) if (keys[i] !== token) out.push(keys[i]);
+      } else {
+        // The words starting with token are in its shard and the next ones
+        // that start with it (the tokens are sorted).
+        var first = Math.max(0, this._shardIndex(token));
+        for (var s = first; s < this.shardFirst.length && (s === first || this.shardFirst[s].indexOf(token) === 0); s++) {
+          var shard = this._shard(s, missing);
+          if (!shard) continue;
+          for (var k in shard) if (hasOwn.call(shard, k) && k !== token && k.indexOf(token) === 0) out.push(k);
+        }
+      }
     }
-    if (this.typos && !this.tokens[token] && Array.from(token).length >= 5) {
-      keys.forEach(function (k) { if (oneEdit(token, k)) out.push(k); });
+    if (this.typos && has === false && Array.from(token).length >= 5) {
+      if (this.latinTokens) {
+        this.latinTokens.forEach(function (k) { if (oneEdit(token, k)) out.push(k); });
+      } else if (missing) {
+        missing.latin = true;
+      } else {
+        throw notLoaded('the Latin tokens are');
+      }
     }
     return { tokens: out, expanded: out.length > 0 && out[0] !== token };
   };
@@ -335,20 +518,38 @@
     });
   }
 
+  // Characters that NFKC may compose with the one before (combining marks,
+  // Hangul vowel and final jamo, half-width sound marks): kept in one cluster
+  // so that the cluster normalizes as it does inside the whole text.
+  var JOINS_PREVIOUS = /^[\p{M}\u1160-\u11ff\ud7b0-\ud7ff\u314f-\u3163\u3187-\u318e\uff9e\uff9f]$/u;
+
   // [start, end) ranges, in code points of the original text, where a phrase
   // of the query occurs after normalization.
   function matchRanges(text, query) {
     var chars = Array.from(String(text === null || text === undefined ? '' : text));
     var norm = [];
-    var origin = [];
+    var start = [];
+    var end = [];
     var prev = null;
-    chars.forEach(function (ch, i) {
-      for (var c of ch.normalize('NFKC').toLowerCase()) {
+    // Each folded code point remembers which original characters it came
+    // from. A character may fold to several (ko: one Private Use Area
+    // character -> a jamo sequence), and a cluster may compose into fewer.
+    function emit(str, from, to) {
+      for (var c of str.normalize('NFKC').toLowerCase()) {
         prev = foldChar(c, prev);
-        norm.push(prev);
-        origin.push(i);
+        for (var p of prev) { norm.push(p); start.push(from); end.push(to); }
       }
-    });
+    }
+    for (var i = 0, j; i < chars.length; i = j) {
+      for (j = i + 1; j < chars.length && JOINS_PREVIOUS.test(chars[j]); j++);
+      var cluster = chars.slice(i, j);
+      var whole = cluster.join('').normalize('NFKC');
+      if (cluster.map(function (c) { return c.normalize('NFKC'); }).join('') === whole) {
+        cluster.forEach(function (c, k) { emit(c, i + k, i + k + 1); });
+      } else {
+        emit(cluster.join(''), i, j);
+      }
+    }
     var normText = norm.join('');
     var ranges = [];
     var needles = [];
@@ -363,7 +564,7 @@
       var at;
       while ((at = normText.indexOf(needle, from)) !== -1) {
         var cp = Array.from(normText.slice(0, at)).length;
-        ranges.push([origin[cp], origin[cp + len - 1] + 1]);
+        ranges.push([start[cp], end[cp + len - 1]]);
         from = at + needle.length;
       }
     });
@@ -400,11 +601,24 @@
     return (start > 0 ? '…' : '') + highlight(part, query) + (end < m.chars.length ? '…' : '');
   }
 
-  function load(url, options) {
+  function fetchJson(url) {
     return fetch(url).then(function (res) {
       if (!res.ok) throw new Error('cjk_index: ' + url + ' returned ' + res.status);
       return res.json();
-    }).then(function (data) { return new Index(data, options); });
+    });
+  }
+
+  // Loads an index file, or the manifest of a split index: its shards are
+  // then fetched from the same directory as queries need them (searchAsync).
+  function load(url, options) {
+    options = Object.assign({}, options);
+    return fetchJson(url).then(function (data) {
+      if (data && data.shards && !options.loadShard) {
+        var base = new URL(url, typeof location !== 'undefined' ? location.href : undefined);
+        options.loadShard = function (file) { return fetchJson(new URL(file, base).href); };
+      }
+      return new Index(data, options);
+    });
   }
 
   var api = {

data/lib/cjk_index/runtime.rb

@@ -17,6 +17,7 @@ module CJKIndex
           # back-references and drop half of the regexp escapes.
           .sub("/*@VARIANTS@*/{}") { JSON.generate(normalizer.variants) }
           .sub("/*@TABLES@*/[]") { JSON.generate(normalizer.tables) }
+          .sub("/*@DIGEST@*/''") { JSON.generate(normalizer.digest) }
           .sub("/*@CJK@*/''") { JSON.generate(Tokenizer.char_class(Tokenizer::CJK_RANGES)) }
           .sub("/*@SEPARATOR@*/''") do
             JSON.generate(Tokenizer.char_class(Tokenizer::SEPARATOR_RANGES, Tokenizer::SEPARATOR_CHARS))