cjk_index 0.2.0 → 1.0.0

data/lib/cjk_index/rake_task.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "rake"
+require "rake/tasklib"
+require "fileutils"
+require_relative "cli"
+
+module CJKIndex
+  # A Rake task that builds an index (and the browser script) from a CSV,
+  # JSON or JSON Lines file, usable from any Ruby build (Rails, Sinatra, a
+  # plain Rakefile). It does what `cjk_index build` does.
+  #
+  #   require "cjk_index/rake_task"
+  #
+  #   CJKIndex::RakeTask.new(:search_index) do |t|
+  #     t.source  = "data/items.csv"
+  #     t.output  = "public/search/items.json"
+  #     t.runtime = "public/search/cjk-index.js"   # optional
+  #     t.ref     = "id"
+  #     t.fields  = %w[title creator]
+  #     t.boosts  = { "title" => 3 }
+  #     t.filters = { "type" => "value", "date" => "year" }
+  #   end
+  #
+  #   rake search_index
+  #
+  # records can be set instead of source: an Array of Hashes, or an object
+  # that responds to call and returns one (e.g. a lambda reading the database).
+  class RakeTask < Rake::TaskLib
+    attr_accessor :name, :source, :records, :output, :runtime, :ref, :fields, :boosts, :filters, :variants, :format,
+                  :description
+
+    def initialize(name = :cjk_index)
+      super()
+      @name = name
+      @boosts = {}
+      @filters = {}
+      @variants = Normalizer::DEFAULT_TABLES
+      @description = "Build the CJK search index"
+      yield self if block_given?
+      define
+    end
+
+    # Builds the index now; the task calls this.
+    def build
+      raise CLI::Error, "#{name}: set output" unless output
+
+      data = records.respond_to?(:call) ? records.call : records
+      data ||= source ? CLI.read_records(source.to_s, format || CLI.format_of(source)) : raise(CLI::Error, "#{name}: set source or records")
+      builder = CLI.build_index(records: data, ref: ref, fields: fields, boosts: boosts, filters: filters, variants: variants)
+      write(output, builder.to_json)
+      write(runtime, Runtime.source(normalizer: builder.normalizer)) if runtime
+      builder
+    end
+
+    private
+
+    def define
+      desc description
+      task(name) do
+        builder = build
+        puts "#{output}: #{builder.size} records"
+      end
+    end
+
+    def write(path, content)
+      FileUtils.mkdir_p(File.dirname(path.to_s))
+      File.write(path.to_s, content)
+    end
+  end
+end

data/lib/cjk_index/runtime.js

@@ -9,9 +9,49 @@
 
   var VARIANTS = /*@VARIANTS@*/{};
   var TABLES = /*@TABLES@*/[];
+  var DIGEST = /*@DIGEST@*/'';
   var CJK = new RegExp(/*@CJK@*/'', 'u');
   var SEPARATOR = new RegExp(/*@SEPARATOR@*/'', 'u');
-  var FORMATS = ['cjk_index/2', 'cjk_index/3'];
+  var FORMATS = ['cjk_index/2', 'cjk_index/3', 'cjk_index/4'];
+  var COMPACT = 'cjk_index/4';
+  var hasOwn = Object.prototype.hasOwnProperty;
+
+  // Posting lists of format 4 (CJKIndex::Postings in Ruby): numbers written
+  // with 90 printable characters, '#' to '}' without '\'. The first 60 are the
+  // last digit of a number (base 60), the other 30 lead it (base 30).
+  var DIGIT = [];
+  for (var code = 0x23, digit = 0; code <= 0x7d; code++) if (code !== 0x5c) DIGIT[code] = digit++;
+
+  // A posting list of format 4 as the flat list of format 3:
+  // [doc, field, n, pos, ..., doc, field, n, ...].
+  function decodePostings(s, nf) {
+    var out = [];
+    var key = -1, state = 0, left = 0, pos = 0, acc = 0;
+    for (var i = 0; i < s.length; i++) {
+      var d = DIGIT[s.charCodeAt(i)];
+      if (d >= 60) { acc = acc * 30 + d - 60; continue; }
+      var v = acc * 60 + d;
+      acc = 0;
+      if (state === 0) { key += v + 1; out.push(Math.floor(key / nf), key % nf); state = 1; }
+      else if (state === 1) { out.push(v); left = v; pos = 0; state = 2; }
+      else { pos += v; out.push(pos); if (--left === 0) state = 0; }
+    }
+    return out;
+  }
+
+  // Compares strings by code point, as Ruby compares UTF-8 strings (the
+  // order of the shards). JavaScript's < compares UTF-16 code units, which
+  // puts characters beyond U+FFFF before U+E000-U+FFFF.
+  function compareCodePoints(a, b) {
+    var i = 0, j = 0;
+    while (i < a.length && j < b.length) {
+      var x = a.codePointAt(i), y = b.codePointAt(j);
+      if (x !== y) return x < y ? -1 : 1;
+      i += x > 0xffff ? 2 : 1;
+      j += y > 0xffff ? 2 : 1;
+    }
+    return (i < a.length ? 1 : 0) - (j < b.length ? 1 : 0);
+  }
 
   // BM25 parameters (the usual defaults, as in Lucene and Pagefind).
   var K1 = 1.2;
@@ -119,6 +159,10 @@
       throw new Error('cjk_index: index built with variant tables ' + JSON.stringify(data.variants) +
         ' but this script folds with ' + JSON.stringify(TABLES) + '; rebuild both together');
     }
+    if (data.variant_digest !== undefined && data.variant_digest !== DIGEST) {
+      throw new Error('cjk_index: index built with other contents of the variant tables ' +
+        JSON.stringify(TABLES) + ' (an older cjk_index?); rebuild it');
+    }
     options = options || {};
     this.prefix = options.prefix !== false;          // last Latin word matches as a prefix
     this.typos = options.typos !== false;            // Latin words of 5+ letters allow one edit
@@ -126,15 +170,30 @@
     this.boosts = data.boosts;
     this.refs = data.refs;
     this.lengths = data.lengths;
-    this.tokens = data.tokens;
+    this.tokens = data.tokens || null;
+    this.encoded = data.format === COMPACT;
+    this.decoded = new Map();
+    if (data.shards) {
+      // Split index: the manifest lists the first token and file of each shard.
+      this.loadShard = options.loadShard;
+      if (!this.loadShard) throw new Error('cjk_index: a split index needs options.loadShard (CJKIndex.load sets it)');
+      this.shardFirst = data.shards.map(function (s) { return s[0]; });
+      this.shardFile = data.shards.map(function (s) { return s[1]; });
+      this.shards = new Map();     // file -> tokens, once loaded
+      this.pending = new Map();    // file -> Promise while loading
+    }
     var nf = this.fields.length;
     var avg = new Array(nf).fill(0);
     for (var i = 0; i < this.lengths.length; i++) avg[i % nf] += this.lengths[i];
     this.avgLength = avg.map(function (s) { return s / Math.max(1, data.refs.length) || 1; });
-    this.latinTokens = Object.keys(this.tokens).filter(isLatin).sort();
+    // A split index lists its Latin tokens in a file, fetched by prepare
+    // only for typo expansion.
+    this.latinFile = this.tokens ? null : data.latin;
+    if (this.tokens) this._setLatin(Object.keys(this.tokens).filter(isLatin));
     this.slotOf = new Map();
     for (var d = 0; d < this.refs.length; d++) if (!this.slotOf.has(this.refs[d])) this.slotOf.set(this.refs[d], d);
-    // Filter values by document: keys (value filters) or a year, with the
+    // Filter values by document: a year, or the ids of values (value
+    // filters), with the folded key of each id, the ids of each key, the
     // form to show for each key and its rank of first appearance.
     this.filters = {};
     var filters = data.filters || {};
@@ -142,19 +201,22 @@
       var f = filters[name];
       var docs = f.docs;
       var display = new Map();
+      var keys = null;
+      var ids = new Map();
       if (f.kind !== 'year') {
-        var keys = f.values.map(filterKey);
+        keys = f.values.map(filterKey);
         f.values.forEach(function (v, i) { if (!display.has(keys[i])) display.set(keys[i], v); });
-        docs = f.docs.map(function (ids) { return ids.map(function (id) { return keys[id]; }); });
+        keys.forEach(function (key, id) { if (!ids.has(key)) ids.set(key, []); ids.get(key).push(id); });
       }
       var rank = new Map();
       docs.forEach(function (value) {
-        (f.kind === 'year' ? (value === null ? [] : [value]) : value).forEach(function (key) {
+        (f.kind === 'year' ? (value === null ? [] : [value]) : value).forEach(function (v) {
+          var key = f.kind === 'year' ? v : keys[v];
           if (!rank.has(key)) rank.set(key, rank.size);
         });
       });
       if (f.kind === 'year') rank.forEach(function (r, year) { display.set(year, year); });
-      this.filters[name] = { kind: f.kind, docs: docs, display: display, rank: rank };
+      this.filters[name] = { kind: f.kind, docs: docs, keys: keys, ids: ids, display: display, rank: rank };
     }, this);
   }
 
@@ -175,8 +237,12 @@
             (max === null || max === undefined || year <= max);
         }]);
       } else {
-        var keys = new Set((Array.isArray(c) ? c : [c]).map(filterKey));
-        out.push([f, function (values) { return values.some(function (k) { return keys.has(k); }); }]);
+        // Compares value ids, not folded strings.
+        var ids = new Set();
+        (Array.isArray(c) ? c : [c]).forEach(function (v) {
+          (f.ids.get(filterKey(v)) || []).forEach(function (id) { ids.add(id); });
+        });
+        out.push([f, function (list) { return list.some(function (id) { return ids.has(id); }); }]);
       }
     });
     return out;
@@ -206,7 +272,8 @@
     var counts = new Map();
     docs.forEach(function (doc) {
       var v = f.docs[doc];
-      (f.kind === 'year' ? (v === null ? [] : [v]) : v).forEach(function (key) {
+      (f.kind === 'year' ? (v === null ? [] : [v]) : v).forEach(function (x) {
+        var key = f.kind === 'year' ? x : f.keys[x];
         counts.set(key, (counts.get(key) || 0) + 1);
       });
     });
@@ -215,12 +282,109 @@
     return out.map(function (e) { return { value: f.display.get(e[0]), count: e[1] }; });
   };
 
+  Index.prototype._setLatin = function (tokens) {
+    this.latinTokens = tokens.slice().sort();
+    this.latinSet = new Set(this.latinTokens);
+  };
+
+  // Number of the shard that holds token (split index), or -1.
+  Index.prototype._shardIndex = function (token) {
+    var lo = 0, hi = this.shardFirst.length;
+    while (lo < hi) {
+      var mid = (lo + hi) >> 1;
+      if (compareCodePoints(this.shardFirst[mid], token) <= 0) lo = mid + 1; else hi = mid;
+    }
+    return lo - 1;
+  };
+
+  function notLoaded(what) {
+    return new Error('cjk_index: ' + what + ' not loaded; use searchAsync(query) or await prepare(query)');
+  }
+
+  // Tokens of shard i, or undefined when it is not loaded: then its file is
+  // added to missing (prepare), or an error is thrown (search).
+  Index.prototype._shard = function (i, missing) {
+    var tokens = this.shards.get(this.shardFile[i]);
+    if (tokens) return tokens;
+    if (!missing) throw notLoaded('the shard ' + this.shardFile[i] + ' is');
+    missing.files.add(this.shardFile[i]);
+    return undefined;
+  };
+
+  // The object that holds token's posting list if it is in the index; null
+  // when it cannot be; undefined when its shard is not loaded (see _shard).
+  Index.prototype._table = function (token, missing) {
+    if (this.tokens) return this.tokens;
+    var i = this._shardIndex(token);
+    return i < 0 ? null : this._shard(i, missing);
+  };
+
+  // true or false, or undefined when not known yet (prepare).
+  Index.prototype._has = function (token, missing) {
+    var table = this._table(token, missing);
+    return table === undefined ? undefined : !!table && hasOwn.call(table, token);
+  };
+
+  // The posting list of a token as [doc, field, n, pos, ...], or null.
+  Index.prototype._postings = function (token) {
+    var table = this._table(token);
+    if (!table || !hasOwn.call(table, token)) return null;
+    if (!this.encoded) return table[token];
+    var p = this.decoded.get(token);
+    if (!p) { p = decodePostings(table[token], this.fields.length); this.decoded.set(token, p); }
+    return p;
+  };
+
+  Index.prototype._load = function (file, done) {
+    var self = this;
+    var p = this.pending.get(file);
+    if (!p) {
+      p = Promise.resolve(this.loadShard(file)).then(done);
+      p.catch(function () { self.pending.delete(file); });
+      this.pending.set(file, p);
+    }
+    return p;
+  };
+
+  // Fetches what a search for query needs from a split index, so that
+  // search(query) can then run: the shards of its tokens, of the words a
+  // last Latin word is a prefix of, and, for a Latin word of 5+ letters not
+  // in the index, the list of Latin tokens (once) and the shards of the
+  // words one edit away. Resolves to the index; at once for one file.
+  Index.prototype.prepare = function (query) {
+    var self = this;
+    if (!this.shardFirst) return Promise.resolve(this);
+    var missing = { files: new Set(), latin: false };
+    var qp = phrases(query, false);
+    qp.forEach(function (phrase, q) {
+      phrase.forEach(function (tok, i) {
+        self._alternatives(tok[0], q === qp.length - 1 && i === phrase.length - 1, missing).tokens.forEach(function (t) {
+          self._table(t, missing);
+        });
+      });
+    });
+    var loads = Array.from(missing.files).map(function (file) {
+      return self._load(file, function (tokens) { self.shards.set(file, tokens); });
+    });
+    if (missing.latin) loads.push(this._load(this.latinFile, function (tokens) { self._setLatin(tokens); }));
+    if (!loads.length) return Promise.resolve(this);
+    // What was loaded may show more to load (typo candidates).
+    return Promise.all(loads).then(function () { return self.prepare(query); });
+  };
+
+  // search, after fetching the shards it needs. Works for every index, so
+  // a page can use it whether or not the index is split.
+  Index.prototype.searchAsync = function (query, options) {
+    var self = this;
+    return this.prepare(query).then(function () { return self.search(query, options); });
+  };
+
   // Map of "doc,field" key -> Set of positions, merged over the alternatives.
   Index.prototype._positions = function (tokens) {
     var nf = this.fields.length;
     var map = new Map();
     for (var t = 0; t < tokens.length; t++) {
-      var p = this.tokens[tokens[t]];
+      var p = this._postings(tokens[t]);
       if (!p) continue;
       for (var i = 0; i < p.length;) {
         var key = p[i] * nf + p[i + 1];
@@ -236,17 +400,36 @@
 
   // Alternatives for one query token: itself, and for Latin words the index
   // words it is a prefix of (last word only) or one edit away from.
-  Index.prototype._alternatives = function (token, isLast) {
+  // With missing (prepare), what is not loaded yet is noted there and left out.
+  Index.prototype._alternatives = function (token, isLast, missing) {
     if (!isLatin(token)) return { tokens: [token], expanded: false };
-    var out = this.tokens[token] ? [token] : [];
-    var keys = this.latinTokens;
+    var has = this.tokens ? this.latinSet.has(token) : this._has(token, missing);
+    var out = has ? [token] : [];
     if (this.prefix && isLast) {
-      var lo = 0, hi = keys.length;
-      while (lo < hi) { var mid = (lo + hi) >> 1; if (keys[mid] < token) lo = mid + 1; else hi = mid; }
-      for (var i = lo; i < keys.length && keys[i].indexOf(token) === 0; i++) if (keys[i] !== token) out.push(keys[i]);
+      if (this.tokens) {
+        var keys = this.latinTokens;
+        var lo = 0, hi = keys.length;
+        while (lo < hi) { var mid = (lo + hi) >> 1; if (keys[mid] < token) lo = mid + 1; else hi = mid; }
+        for (var i = lo; i < keys.length && keys[i].indexOf(token) === 0; i++) if (keys[i] !== token) out.push(keys[i]);
+      } else {
+        // The words starting with token are in its shard and the next ones
+        // that start with it (the tokens are sorted).
+        var first = Math.max(0, this._shardIndex(token));
+        for (var s = first; s < this.shardFirst.length && (s === first || this.shardFirst[s].indexOf(token) === 0); s++) {
+          var shard = this._shard(s, missing);
+          if (!shard) continue;
+          for (var k in shard) if (hasOwn.call(shard, k) && k !== token && k.indexOf(token) === 0) out.push(k);
+        }
+      }
     }
-    if (this.typos && !this.tokens[token] && Array.from(token).length >= 5) {
-      keys.forEach(function (k) { if (oneEdit(token, k)) out.push(k); });
+    if (this.typos && has === false && Array.from(token).length >= 5) {
+      if (this.latinTokens) {
+        this.latinTokens.forEach(function (k) { if (oneEdit(token, k)) out.push(k); });
+      } else if (missing) {
+        missing.latin = true;
+      } else {
+        throw notLoaded('the Latin tokens are');
+      }
     }
     return { tokens: out, expanded: out.length > 0 && out[0] !== token };
   };
@@ -335,20 +518,38 @@
     });
   }
 
+  // Characters that NFKC may compose with the one before (combining marks,
+  // Hangul vowel and final jamo, half-width sound marks): kept in one cluster
+  // so that the cluster normalizes as it does inside the whole text.
+  var JOINS_PREVIOUS = /^[\p{M}\u1160-\u11ff\ud7b0-\ud7ff\u314f-\u3163\u3187-\u318e\uff9e\uff9f]$/u;
+
   // [start, end) ranges, in code points of the original text, where a phrase
   // of the query occurs after normalization.
   function matchRanges(text, query) {
     var chars = Array.from(String(text === null || text === undefined ? '' : text));
     var norm = [];
-    var origin = [];
+    var start = [];
+    var end = [];
     var prev = null;
-    chars.forEach(function (ch, i) {
-      for (var c of ch.normalize('NFKC').toLowerCase()) {
+    // Each folded code point remembers which original characters it came
+    // from. A character may fold to several (ko: one Private Use Area
+    // character -> a jamo sequence), and a cluster may compose into fewer.
+    function emit(str, from, to) {
+      for (var c of str.normalize('NFKC').toLowerCase()) {
         prev = foldChar(c, prev);
-        norm.push(prev);
-        origin.push(i);
+        for (var p of prev) { norm.push(p); start.push(from); end.push(to); }
       }
-    });
+    }
+    for (var i = 0, j; i < chars.length; i = j) {
+      for (j = i + 1; j < chars.length && JOINS_PREVIOUS.test(chars[j]); j++);
+      var cluster = chars.slice(i, j);
+      var whole = cluster.join('').normalize('NFKC');
+      if (cluster.map(function (c) { return c.normalize('NFKC'); }).join('') === whole) {
+        cluster.forEach(function (c, k) { emit(c, i + k, i + k + 1); });
+      } else {
+        emit(cluster.join(''), i, j);
+      }
+    }
     var normText = norm.join('');
     var ranges = [];
     var needles = [];
@@ -363,7 +564,7 @@
       var at;
       while ((at = normText.indexOf(needle, from)) !== -1) {
         var cp = Array.from(normText.slice(0, at)).length;
-        ranges.push([origin[cp], origin[cp + len - 1] + 1]);
+        ranges.push([start[cp], end[cp + len - 1]]);
         from = at + needle.length;
       }
     });
@@ -400,11 +601,24 @@
     return (start > 0 ? '…' : '') + highlight(part, query) + (end < m.chars.length ? '…' : '');
   }
 
-  function load(url, options) {
+  function fetchJson(url) {
     return fetch(url).then(function (res) {
       if (!res.ok) throw new Error('cjk_index: ' + url + ' returned ' + res.status);
       return res.json();
-    }).then(function (data) { return new Index(data, options); });
+    });
+  }
+
+  // Loads an index file, or the manifest of a split index: its shards are
+  // then fetched from the same directory as queries need them (searchAsync).
+  function load(url, options) {
+    options = Object.assign({}, options);
+    return fetchJson(url).then(function (data) {
+      if (data && data.shards && !options.loadShard) {
+        var base = new URL(url, typeof location !== 'undefined' ? location.href : undefined);
+        options.loadShard = function (file) { return fetchJson(new URL(file, base).href); };
+      }
+      return new Index(data, options);
+    });
   }
 
   var api = {

data/lib/cjk_index/runtime.rb

@@ -17,6 +17,7 @@ module CJKIndex
           # back-references and drop half of the regexp escapes.
           .sub("/*@VARIANTS@*/{}") { JSON.generate(normalizer.variants) }
           .sub("/*@TABLES@*/[]") { JSON.generate(normalizer.tables) }
+          .sub("/*@DIGEST@*/''") { JSON.generate(normalizer.digest) }
           .sub("/*@CJK@*/''") { JSON.generate(Tokenizer.char_class(Tokenizer::CJK_RANGES)) }
           .sub("/*@SEPARATOR@*/''") do
             JSON.generate(Tokenizer.char_class(Tokenizer::SEPARATOR_RANGES, Tokenizer::SEPARATOR_CHARS))

data/lib/cjk_index/searcher.rb

@@ -28,13 +28,25 @@ module CJKIndex
     # documents, also after later add, remove and update.
     # prefix: the last Latin word of a query also matches longer words.
     # typos: Latin words of 5+ letters also match words one edit away.
+    #
+    # path is an index file, or a split index (its directory or its
+    # manifest); the shards of a split index are read as queries need them.
     def self.load(path, **options)
-      new(File.read(path, encoding: "UTF-8"), **options)
+      path = File.join(path, Builder::MANIFEST) if File.directory?(path)
+      data = JSON.parse(File.read(path, encoding: "UTF-8"))
+      if data.is_a?(Hash) && data["shards"]
+        dir = File.dirname(path)
+        options = { load_shard: ->(file) { JSON.parse(File.read(File.join(dir, file), encoding: "UTF-8")) } }.merge(options)
+      end
+      new(data, **options)
     end
 
-    def initialize(index, prefix: true, typos: true, normalizer: nil)
+    # load_shard: for the manifest of a split index, a callable that returns
+    # a shard (a Hash of token => posting list) given its file name.
+    def initialize(index, prefix: true, typos: true, normalizer: nil, load_shard: nil)
       @prefix = prefix
       @typos = typos
+      @load_shard = load_shard
       if index.is_a?(Builder)
         @builder = index
         data = index.live_data
@@ -46,9 +58,10 @@ module CJKIndex
       end
 
       @normalizer = normalizer || Normalizer.new(tables: data["variants"])
-      if @normalizer.tables != data["variants"]
-        raise ArgumentError, "cjk_index: index built with variant tables #{data['variants'].inspect} " \
-                             "but this normalizer folds with #{@normalizer.tables.inspect}; rebuild both together"
+      @normalizer.check!(data)
+
+      if data["shards"] && !@load_shard
+        raise ArgumentError, "cjk_index: a split index needs load_shard (or Searcher.load(directory))"
       end
 
       setup(data)
@@ -149,7 +162,7 @@ module CJKIndex
       docs.each do |doc|
         value = filter[:docs][doc]
         if filter[:kind] == "year" then counts[value] += 1 unless value.nil?
-        else value.each { |key| counts[key] += 1 }
+        else value.each { |id| counts[filter[:keys][id]] += 1 }
         end
       end
       counts.sort_by { |key, count| [-count, filter[:rank][key]] }
@@ -169,39 +182,54 @@ module CJKIndex
       @refs = data["refs"]
       @lengths = data["lengths"]
       @tokens = data["tokens"]
+      @encoded = data["format"] == Builder::FORMAT
+      @decoded = {}
+      if data["shards"]
+        @shard_first = data["shards"].map(&:first)
+        @shard_file = data["shards"].map(&:last)
+        @shards = {}
+      end
       @count = data["count"] || @refs.length
       nf = @fields.length
       avg = Array.new(nf, 0.0)
       @lengths.each_with_index { |len, i| avg[i % nf] += len if len }
       @avg_length = avg.map { |s| (v = s / [1, @count].max).zero? ? 1 : v }
-      @latin_tokens = @tokens.keys.select { |t| latin?(t) }.sort
+      # A split index lists its Latin tokens in a file, read only for typos.
+      @latin_file = data["latin"] unless @tokens
+      @latin_tokens = @latin_file ? nil : @tokens.keys.select { |t| latin?(t) }.sort
+      @latin_set = @latin_tokens&.to_set
       @slot_of = {}
       @refs.each_with_index { |ref, doc| @slot_of[ref] ||= doc if ref }
       @filters = (data["filters"] || {}).to_h { |name, f| [name, setup_filter(f)] }
     end
 
-    # Filter values by document: keys (value filters) or a year, with the
+    # Filter values by document: a year, or the ids of values (value
+    # filters), with the folded key of each id, the ids of each key, the
     # form to show for each key and its rank of first appearance.
     def setup_filter(f)
-      docs = if f["kind"] == "year" then f["docs"]
-             elsif f["values"]
-               keys = f["values"].map { |v| Builder.filter_key(v, @normalizer) }
-               f["docs"].map { |ids| ids.map { |id| keys[id] } }
-             else
-               f["docs"].map { |pairs| pairs&.map(&:first) }
-             end
-      display = {}
-      if f["values"] then f["values"].each { |v| display[Builder.filter_key(v, @normalizer)] ||= v }
-      elsif f["kind"] == "value" then f["docs"].each { |pairs| pairs&.each { |key, v| display[key] ||= v } }
+      if f["kind"] == "year"
+        rank = {}
+        f["docs"].each_with_index { |year, doc| rank[year] ||= rank.size if @refs[doc] && !year.nil? }
+        return { kind: "year", docs: f["docs"], display: rank.keys.to_h { |year| [year, year] }, rank: rank }
       end
-      rank = {}
-      docs.each_with_index do |value, doc|
-        next unless @refs[doc]
 
-        Array(value).each { |key| rank[key] ||= rank.size }
+      values = f["values"]
+      docs = f["docs"]
+      unless values
+        id_of = {}
+        values = []
+        docs = docs.map { |pairs| (pairs || []).map { |key, v| id_of[key] ||= (values << v).size - 1 } }
       end
-      display = rank.keys.to_h { |year| [year, year] } if f["kind"] == "year"
-      { kind: f["kind"], docs: docs, display: display, rank: rank }
+      keys = values.map { |v| Builder.filter_key(v, @normalizer) }
+      ids = Hash.new { |h, key| h[key] = [] }
+      keys.each_with_index { |key, id| ids[key] << id }
+      display = {}
+      values.each_with_index { |v, id| display[keys[id]] ||= v }
+      rank = {}
+      docs.each_with_index do |list, doc|
+        list.each { |id| rank[keys[id]] ||= rank.size } if @refs[doc]
+      end
+      { kind: f["kind"], docs: docs, keys: keys, ids: ids, display: display, rank: rank }
     end
 
     def refresh
@@ -222,8 +250,9 @@ module CJKIndex
                   end
           [f, ->(year) { !year.nil? && range.cover?(year) }]
         else
-          keys = Array(condition).map { |v| Builder.filter_key(v, @normalizer) }.to_set
-          [f, ->(values) { values.any? { |key| keys.include?(key) } }]
+          # Compares value ids, not folded strings.
+          ids = Array(condition).flat_map { |v| f[:ids].fetch(Builder.filter_key(v, @normalizer), []) }.to_set
+          [f, ->(list) { list.any? { |id| ids.include?(id) } }]
         end
       end
     end
@@ -237,7 +266,7 @@ module CJKIndex
       nf = @fields.length
       map = {}
       tokens.each do |token|
-        postings = @tokens.fetch(token, nil) or next
+        postings = postings(token) or next
         i = 0
         while i < postings.length
           count = postings[i + 2]
@@ -248,21 +277,56 @@ module CJKIndex
       map
     end
 
+    # Number of the shard that holds token (split index), or -1.
+    def shard_index(token) = (@shard_first.bsearch_index { |first| first > token } || @shard_first.length) - 1
+
+    def shard(i) = @shards[@shard_file[i]] ||= @load_shard.call(@shard_file[i])
+
+    # The Hash that holds token's posting list if it is in the index, or nil.
+    def table(token)
+      return @tokens unless @shard_first
+
+      i = shard_index(token)
+      i.negative? ? nil : shard(i)
+    end
+
+    def token?(token) = !!table(token)&.key?(token)
+
+    # The posting list of a token as [doc, field, n, pos, ...], or nil.
+    def postings(token)
+      list = table(token)&.[](token)
+      return list unless list && @encoded
+
+      @decoded[token] ||= Postings.decode(list, @fields.length)
+    end
+
+    # Index words that start with token, token itself left out.
+    def with_prefix(token)
+      unless @shard_first
+        from = @latin_tokens.bsearch_index { |k| k >= token } || @latin_tokens.length
+        return @latin_tokens[from..].take_while { |k| k.start_with?(token) }.reject { |k| k == token }
+      end
+
+      # In token's shard and the next ones that start with it (tokens are sorted).
+      first = [shard_index(token), 0].max
+      last = first
+      last += 1 while last + 1 < @shard_first.length && @shard_first[last + 1].start_with?(token)
+      (first..last).flat_map { |i| shard(i).keys.select { |k| k != token && k.start_with?(token) } }
+    end
+
+    def latin_tokens
+      @latin_tokens ||= @load_shard.call(@latin_file).sort
+    end
+
     # Alternatives for one query token: itself, and for Latin words the index
     # words it is a prefix of (last word only) or one edit away from.
     def alternatives(token, last)
       return [[token], false] unless latin?(token)
 
-      out = @tokens.key?(token) ? [token] : []
-      if @prefix && last
-        from = @latin_tokens.bsearch_index { |k| k >= token } || @latin_tokens.length
-        @latin_tokens[from..].each do |k|
-          break unless k.start_with?(token)
-
-          out << k unless k == token
-        end
-      end
-      out.concat(@latin_tokens.select { |k| one_edit?(token, k) }) if @typos && !@tokens.key?(token) && token.length >= 5
+      has = @shard_first ? token?(token) : @latin_set.include?(token)
+      out = has ? [token] : []
+      out.concat(with_prefix(token)) if @prefix && last
+      out.concat(latin_tokens.select { |k| one_edit?(token, k) }) if @typos && !has && token.length >= 5
       [out, !out.empty? && out.first != token]
     end