cjk_index 0.2.0 → 1.0.0

data/exe/cjk_index

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "cjk_index/cli"
+
+exit CJKIndex::CLI.run(ARGV)

data/lib/cjk_index/bridgetown.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "bridgetown-core"
+require_relative "../cjk_index"
+
+module CJKIndex
+  # Bridgetown plugin. Add `gem "cjk_index"` to the Gemfile, then in
+  # config/initializers.rb:
+  #
+  #   Bridgetown.configure do |config|
+  #     init :cjk_index,
+  #          output: "cjk-index",               # folder inside the output (default)
+  #          variants: ["ja"],                  # variant tables: ja (default), zh
+  #          indexes: [
+  #            { name: "posts",                 # -> output/cjk-index/posts.json
+  #              collection: "posts",           # a collection (or a list of them), or...
+  #              fields: ["title", "tags", "content"],
+  #              boosts: { title: 3 },
+  #              filters: { category: "value", date: "year" } },
+  #            { name: "items", data: "metadata", ref: "id", fields: ["title"] } # ...rows of src/_data/metadata.csv
+  #          ]
+  #   end
+  #
+  # After each build the index files and the browser script (cjk-index.js)
+  # are written to the output folder. A resource is indexed by its URL; the
+  # field "content" is the source text of the resource (Markdown, before
+  # rendering), any other field is read from its front matter.
+  module Bridgetown
+    DEFAULT_OUTPUT = "cjk-index"
+    RUNTIME_NAME = "cjk-index.js"
+
+    module_function
+
+    # Writes the browser script and every index of the config (a Hash with
+    # symbol or string keys) into site's destination folder.
+    def write(site, config)
+      config = stringify(config)
+      output = config.fetch("output", DEFAULT_OUTPUT).to_s.gsub(%r{\A/+|/+\z}, "")
+      normalizer = Normalizer.new(tables: Array(config.fetch("variants", Normalizer::DEFAULT_TABLES)))
+      put(site, output, RUNTIME_NAME, Runtime.source(normalizer: normalizer))
+      Array(config["indexes"]).each do |spec|
+        builder = build(site, stringify(spec), normalizer)
+        put(site, output, "#{spec['name'] || spec[:name]}.json", builder.to_json)
+        ::Bridgetown.logger.info "cjk_index:", "#{output}/#{spec['name'] || spec[:name]}.json (#{builder.size} documents)"
+      end
+    end
+
+    def build(site, spec, normalizer)
+      builder = Builder.new(fields: Array(spec.fetch("fields")), boosts: spec.fetch("boosts", {}),
+                            normalizer: normalizer, filters: spec.fetch("filters", {}))
+      if spec["collection"]
+        Array(spec["collection"]).each do |name|
+          collection = site.collections[name] or raise ArgumentError, "cjk_index: no collection #{name}"
+          collection.resources.each do |resource|
+            next unless resource.write?
+
+            builder.add(resource.relative_url, resource_fields(resource, builder.fields | builder.filters.keys))
+          end
+        end
+      elsif spec["data"]
+        ref = spec.fetch("ref", "id")
+        Array(site.data[spec["data"]]).each do |row|
+          row = row.to_h.transform_keys(&:to_s)
+          builder.add(row[ref], row) unless row[ref].to_s.empty?
+        end
+      else
+        raise ArgumentError, "cjk_index: index #{spec['name']} needs collection or data"
+      end
+      builder
+    end
+
+    def resource_fields(resource, fields)
+      fields.to_h { |f| [f, f == "content" ? resource.untransformed_content : plain(resource.data[f])] }
+    end
+
+    # A Time or Date as "2026-10-01": Builder applies Array() to values, which
+    # would turn a Time into its parts [sec, min, hour, ...].
+    def plain(value)
+      value.respond_to?(:strftime) ? value.strftime("%Y-%m-%d") : value
+    end
+
+    def stringify(hash) = hash.to_h { |k, v| [k.to_s, v] }
+
+    # Skips the write when the file already has this content.
+    def put(site, dir, name, content)
+      path = site.in_dest_dir(dir, name)
+      return if File.exist?(path) && File.read(path, encoding: "UTF-8") == content
+
+      FileUtils.mkdir_p(File.dirname(path))
+      File.write(path, content)
+    end
+  end
+end
+
+Bridgetown.initializer :cjk_index do |_config, **options|
+  Bridgetown::Hooks.register_one :site, :post_write, reloadable: false do |site|
+    CJKIndex::Bridgetown.write(site, options)
+  end
+end

data/lib/cjk_index/builder.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require "json"
+require "digest"
+require "fileutils"
 
 module CJKIndex
   # Builds a prebuilt inverted index that the browser runtime loads as JSON.
@@ -8,37 +10,67 @@ module CJKIndex
   #   builder = CJKIndex::Builder.new(fields: %w[title creator], boosts: { "title" => 3 },
   #                                   filters: { "type" => "value", "date" => "year" })
   #   builder.add("item1.html", "title" => "東京帝國大學", "creator" => "…", "type" => "地図", "date" => "1900")
-  #   File.write("search-index.json", builder.to_json)
+  #   builder.write("search-index.json")     # one file (the same as File.write(path, builder.to_json))
+  #   builder.write_split("search-index")    # or a manifest and shards, loaded as a query needs them
   #
-  # Format (version 3):
+  # Format (version 4), one file:
   #   {
-  #     "format":  "cjk_index/3",
+  #     "format":  "cjk_index/4",
   #     "variants": ["ja"],
   #     "fields":  ["title", "creator"],
   #     "boosts":  [3, 1],
   #     "refs":    ["item1.html", ...],
   #     "lengths": [len(doc0.title), len(doc0.creator), len(doc1.title), ...],
-  #     "tokens":  { "東京": [doc, field, n, pos1, ..., posn, doc, field, n, ...], ... },
+  #     "tokens":  { "東京": "#$#'+…", ... },
   #     "filters": {
   #       "type": { "kind": "value", "values": ["地図", ...], "docs": [[0], [], ...] },
   #       "date": { "kind": "year", "docs": [1900, null, ...] }
   #     }
   #   }
-  # doc and field are positions in "refs" and "fields"; n is the number of
-  # occurrences and pos are character offsets in the normalized field text.
+  # Each token maps to its posting list: for each document and field where it
+  # occurs, the number of occurrences and their character offsets in the
+  # normalized field text, written compactly as a string (see Postings).
   # Offsets let the runtime require the bigrams of a query to be adjacent
   # (exact substring match); lengths feed BM25 ranking.
   # "filters" is present only when filters were configured. A value filter
   # stores each distinct value once (by its folded form, as first written)
   # and lists the values of each document by number; a year filter stores one
-  # year or null per document. Version 2 is the same without "filters".
+  # year or null per document.
+  #
+  # Split (write_split): a directory with the manifest index.json, which is
+  # the same without "tokens" and adds
+  #     "latin":  "9c1d…07.json",                 file of the Latin tokens
+  #     "shards": [["一", "3f2a…e1.json"], ...]   first token and file of each shard
+  # the shard files, each a JSON object of tokens => posting lists, and the
+  # file of Latin tokens: a JSON array of every token without CJK characters,
+  # sorted, that lets the runtime expand Latin prefixes and typos. The
+  # tokens are sorted by code point and cut into shards of about shard_size
+  # bytes, so a token is in the last shard whose first token is not greater.
+  # A shard file is named after a digest of its content, so a browser may
+  # cache it: a changed shard gets a new name. The runtime loads the manifest
+  # (refs, lengths, filters: what a blank query and facets need), and fetches
+  # only the shards of the tokens of a query, and the Latin tokens when the
+  # query has a Latin word (on a technical blog they are most of the
+  # manifest's size otherwise).
+  #
+  # Older formats, still read by Builder.load, Searcher and the runtime:
+  # version 3 writes each posting list as a JSON array
+  # [doc, field, n, pos1, ..., posn, doc, field, n, ...] (doc and field are
+  # positions in "refs" and "fields"); version 2 is version 3 without "filters".
   #
   # Documents can be removed and replaced (remove, update), also in an index
-  # read back with Builder.load. to_h is then the same as for an index built
-  # from scratch with the remaining documents in the same order.
+  # read back with Builder.load or Builder.read. to_h is then the same as for
+  # an index built from scratch with the remaining documents in the same order.
   class Builder
-    FORMAT = "cjk_index/3"
-    FORMATS = ["cjk_index/2", FORMAT].freeze
+    FORMAT = "cjk_index/4"
+    FORMATS = ["cjk_index/2", "cjk_index/3", FORMAT].freeze
+
+    # Name of the manifest in a split index.
+    MANIFEST = "index.json"
+    # Default size of a shard in bytes (of JSON, before compression).
+    SHARD_SIZE = 32_000
+    # Names of shard files: a digest of the content.
+    SHARD_NAME = /\A\h{16}\.json\z/
 
     # Gap between the values of a multi-valued field, so that a phrase cannot
     # match across two values.
@@ -63,13 +95,19 @@ module CJKIndex
       @fields = fields.map(&:to_s)
       @boosts = @fields.map { |f| (boosts[f] || boosts[f.to_sym] || 1).to_f }
       @filters = Builder.filter_specs(filters)
-      # Indexed by slot; a removed document leaves nil until to_h renumbers.
+      # Indexed by slot; a removed document leaves nil until the slots are
+      # renumbered (compact).
       @refs = []
       @lengths = []
       @filter_values = []
-      @doc_tokens = []
-      @doc_tokens_complete = true
-      @postings = Hash.new { |h, k| h[k] = [] }
+      # token => posting list in the encoding of Postings, with slots for
+      # documents (the same as the format while there are no removed slots).
+      @postings = {}
+      # token => key of its last entry, filled as entries are appended.
+      @last_keys = {}
+      # Tokens of each slot, kept only once a document has been removed or
+      # replaced (nil until then): a build that only adds does not need them.
+      @doc_tokens = nil
       @slots = Hash.new { |h, k| h[k] = [] }
       @count = 0
       @version = 0
@@ -89,20 +127,43 @@ module CJKIndex
     end
 
     # Reads an index written by to_json (String) or to_h back, to update it.
+    # Reads formats 2, 3 and 4; a split index is read with read.
     # A value filter keeps the form of each value that the index stored.
     def self.load(index)
       data = index.is_a?(String) ? JSON.parse(index) : index
       unless data.is_a?(Hash) && FORMATS.include?(data["format"])
         raise ArgumentError, "cjk_index: unsupported index format #{data.is_a?(Hash) ? data['format'].inspect : data.class}"
       end
+      raise ArgumentError, "cjk_index: a split index is read with Builder.read(directory)" unless data["tokens"]
 
       filters = (data["filters"] || {}).transform_values { |f| f.slice("kind", "separator") }
+      normalizer = Normalizer.new(tables: data["variants"])
+      normalizer.check!(data)
       builder = new(fields: data["fields"], boosts: data["fields"].zip(data["boosts"]).to_h,
-                    normalizer: Normalizer.new(tables: data["variants"]), filters: filters)
+                    normalizer: normalizer, filters: filters)
       builder.send(:restore, data)
       builder
     end
 
+    # Reads an index from a file (written by write or to_json) or from the
+    # directory of a split index (written by write_split), to update it.
+    def self.read(path)
+      load(Builder.read_data(path))
+    end
+
+    # The index at path as one Hash with "tokens", for a file or a split
+    # index (its directory or its manifest).
+    def self.read_data(path) # :nodoc:
+      path = File.join(path, MANIFEST) if File.directory?(path)
+      data = JSON.parse(File.read(path, encoding: "UTF-8"))
+      return data unless data.is_a?(Hash) && data["shards"]
+
+      dir = File.dirname(path)
+      tokens = {}
+      data["shards"].each { |_, file| tokens.merge!(JSON.parse(File.read(File.join(dir, file), encoding: "UTF-8"))) }
+      data.reject { |k, _| %w[latin shards].include?(k) }.merge("tokens" => tokens)
+    end
+
     # Folded form of a value filter's value, used to compare values.
     def self.filter_key(value, normalizer)
       normalizer.normalize(value.to_s).gsub(WHITESPACE, " ").delete_prefix(" ").delete_suffix(" ")
@@ -111,6 +172,7 @@ module CJKIndex
     # The first four-digit number in a value (after folding full-width digits).
     def self.year(value, normalizer)
       return value.to_i if value.is_a?(Numeric)
+      return value.year if value.respond_to?(:year) # Time, Date (YAML front matter)
 
       normalizer.normalize(value.to_s)[/(?<!\d)\d{4}(?!\d)/]&.to_i
     end
@@ -155,17 +217,10 @@ module CJKIndex
     end
 
     def to_h
-      live = holes? ? @refs.each_index.reject { |slot| @refs[slot].nil? } : @refs.each_index.to_a
-      h = {
-        "format" => FORMAT,
-        "variants" => @normalizer.tables,
-        "fields" => @fields,
-        "boosts" => @boosts.map { |b| b == b.to_i ? b.to_i : b },
-        "refs" => live.map { |slot| @refs[slot] },
-        "lengths" => live.flat_map { |slot| @lengths[slot * @fields.length, @fields.length] },
-        "tokens" => holes? ? renumbered_postings(live) : @postings.sort.to_h
-      }
-      h["filters"] = @filters.to_h { |name, spec| [name, filter_to_h(name, spec, live)] } unless @filters.empty?
+      compact
+      h = header
+      h["tokens"] = @postings.keys.sort!.to_h { |token| [token, @postings[token].dup] }
+      h["filters"] = filters_h unless @filters.empty?
       h
     end
 
@@ -173,10 +228,91 @@ module CJKIndex
       JSON.generate(to_h)
     end
 
+    # Writes the index to one file, the same as File.write(path, to_json),
+    # without first making the whole JSON in memory. The file is written to a
+    # temporary file and renamed, so a reader never sees it partly written.
+    def write(path)
+      compact
+      atomic_write(path) do |io|
+        io << JSON.generate(header).delete_suffix("}") << ',"tokens":{'
+        @postings.keys.sort!.each_with_index do |token, i|
+          io << "," if i.positive?
+          io << JSON.generate(token) << ':"' << @postings[token] << '"'
+        end
+        io << "}"
+        io << ',"filters":' << JSON.generate(filters_h) unless @filters.empty?
+        io << "}"
+      end
+      path
+    end
+
+    # Writes a split index to the directory dir: the manifest index.json and
+    # shard files of about shard_size bytes (see the format above). Shard
+    # files are written first and the manifest last (renamed into place), and
+    # shard files that the new manifest does not list are then removed.
+    # Returns the path of the manifest.
+    def write_split(dir, shard_size: SHARD_SIZE)
+      FileUtils.mkdir_p(dir)
+      names = []
+      manifest = nil
+      each_split_file(shard_size: shard_size) do |name, content|
+        if name == MANIFEST
+          manifest = content
+        else
+          path = File.join(dir, name)
+          atomic_write(path) { |io| io << content } unless File.exist?(path) && File.size(path) == content.bytesize
+          names << name
+        end
+      end
+      atomic_write(File.join(dir, MANIFEST)) { |io| io << manifest }
+      Dir.children(dir).each do |name|
+        File.delete(File.join(dir, name)) if name.match?(SHARD_NAME) && !names.include?(name)
+      end
+      File.join(dir, MANIFEST)
+    end
+
+    # Yields [name, content] for each file of a split index: the shards, the
+    # Latin tokens, then the manifest (MANIFEST). For writing them somewhere other than a
+    # directory on disk (the Jekyll plugin uses it).
+    def each_split_file(shard_size: SHARD_SIZE)
+      return enum_for(:each_split_file, shard_size: shard_size) unless block_given?
+
+      compact
+      shards = []
+      latin = []
+      part = []
+      bytes = 0
+      first = nil
+      flush = lambda do
+        content = "{#{part.join(',')}}"
+        name = "#{Digest::SHA256.hexdigest(content)[0, 16]}.json"
+        yield name, content
+        shards << [first, name]
+        part = []
+        bytes = 0
+      end
+      @postings.keys.sort!.each do |token|
+        latin << token unless token.match?(Tokenizer::CJK)
+        pair = "#{JSON.generate(token)}:\"#{@postings[token]}\""
+        flush.call if bytes.positive? && bytes + pair.bytesize > shard_size
+        first = token if part.empty?
+        part << pair
+        bytes += pair.bytesize + 1
+      end
+      flush.call unless part.empty?
+      content = JSON.generate(latin)
+      latin_name = "#{Digest::SHA256.hexdigest(content)[0, 16]}.json"
+      yield latin_name, content
+      manifest = header.merge("latin" => latin_name, "shards" => shards)
+      manifest["filters"] = filters_h unless @filters.empty?
+      yield MANIFEST, JSON.generate(manifest)
+    end
+
     # For Searcher: the current documents by slot, with nil for removed ones.
     def live_data # :nodoc:
       {
-        "variants" => @normalizer.tables, "fields" => @fields, "boosts" => @boosts,
+        "format" => FORMAT, "variants" => @normalizer.tables, "variant_digest" => @normalizer.digest,
+        "fields" => @fields, "boosts" => @boosts,
         "refs" => @refs, "lengths" => @lengths, "tokens" => @postings, "count" => @count,
         "filters" => @filters.to_h do |name, spec|
           [name, { "kind" => spec["kind"], "docs" => @filter_values.map { |v| v && v[name] } }]
@@ -186,46 +322,71 @@ module CJKIndex
 
     private
 
+    def header
+      {
+        "format" => FORMAT,
+        "variants" => @normalizer.tables,
+        "variant_digest" => @normalizer.digest,
+        "fields" => @fields,
+        "boosts" => @boosts.map { |b| b == b.to_i ? b.to_i : b },
+        "refs" => @refs.dup,
+        "lengths" => @lengths.dup
+      }
+    end
+
+    def filters_h = @filters.to_h { |name, spec| [name, filter_to_h(name, spec)] }
+
+    def atomic_write(path)
+      tmp = "#{path}.#{Process.pid}.tmp"
+      File.open(tmp, "wb") { |io| yield io }
+      File.rename(tmp, path)
+    ensure
+      File.delete(tmp) if tmp && File.exist?(tmp)
+    end
+
     def index(slot, doc)
+      nf = @fields.length
+      # A document replaced in place goes before the later slots, so that
+      # every list stays in the order a fresh build writes.
       in_place = slot < @refs.length - 1
-      tokens = []
+      tokens = @doc_tokens ? [] : nil
       @fields.each_with_index do |field, field_index|
         positions = Hash.new { |h, k| h[k] = [] }
         base = 0
         Array(value_of(doc, field)).each do |v|
-          text = v.to_s
-          Tokenizer.tokens_with_offsets(text, unigrams: true, normalizer: @normalizer)
-                   .each { |t, offset| positions[t] << (base + offset) }
-          base += @normalizer.normalize(text).length + VALUE_GAP
+          text = @normalizer.normalize(v.to_s)
+          Tokenizer.each_token(text, unigrams: true) { |t, offset| positions[t] << (base + offset) }
+          base += text.length + VALUE_GAP
         end
-        @lengths[(slot * @fields.length) + field_index] = [base - VALUE_GAP, 0].max
+        @lengths[(slot * nf) + field_index] = [base - VALUE_GAP, 0].max
+        key = (slot * nf) + field_index
         positions.each do |token, offsets|
-          postings = @postings[token]
-          # A document replaced in place goes before the later slots, so that
-          # every list stays in the order a fresh build writes.
-          at = in_place ? insertion_point(postings, slot, field_index) : postings.length
-          postings.insert(at, slot, field_index, offsets.length, *offsets)
+          if in_place
+            @postings[token] = Postings.insert(@postings[token] || +"", key, offsets)
+            @last_keys.delete(token)
+          else
+            list = (@postings[token] ||= +"")
+            last = @last_keys.fetch(token) { Postings.last_key(list) }
+            Postings.append(list, key - last - 1, offsets)
+            @last_keys[token] = key
+          end
         end
-        tokens.concat(positions.keys)
+        tokens&.concat(positions.keys)
       end
-      @doc_tokens[slot] = tokens.uniq
+      @doc_tokens[slot] = tokens.uniq if tokens
       @filter_values[slot] = @filters.to_h { |name, spec| [name, filter_value(spec, value_of(doc, name))] }
       @version += 1
     end
 
     def unindex(slot)
+      nf = @fields.length
+      first = slot * nf
       doc_tokens(slot).each do |token|
-        postings = @postings[token]
-        kept = []
-        i = 0
-        while i < postings.length
-          n = 3 + postings[i + 2]
-          kept.concat(postings[i, n]) unless postings[i] == slot
-          i += n
-        end
+        kept = Postings.delete(@postings[token], first..(first + nf - 1))
         if kept.empty? then @postings.delete(token)
         else @postings[token] = kept
         end
+        @last_keys.delete(token)
       end
       @doc_tokens[slot] = nil
       @version += 1
@@ -235,26 +396,28 @@ module CJKIndex
 
     # Value filter: [[key, value as written], ...]; year filter: Integer or nil.
     def filter_value(spec, raw)
+      # Array() would split a Time into its parts
+      raw = raw.is_a?(Array) ? raw : [raw].compact
       if spec["kind"] == "year"
-        Array(raw).each do |v|
+        raw.each do |v|
           year = Builder.year(v, @normalizer)
           return year if year
         end
         return nil
       end
 
-      Array(raw).flat_map { |v| spec["separator"] ? v.to_s.split(spec["separator"]) : [v.to_s] }
+      raw.flat_map { |v| spec["separator"] ? v.to_s.split(spec["separator"]) : [v.to_s] }
                 .map { |v| v.gsub(WHITESPACE, " ").strip }.reject(&:empty?)
                 .map { |v| [Builder.filter_key(v, @normalizer), v] }.uniq(&:first)
     end
 
-    def filter_to_h(name, spec, live)
-      return { "kind" => "year", "docs" => live.map { |slot| @filter_values[slot][name] } } if spec["kind"] == "year"
+    def filter_to_h(name, spec)
+      return { "kind" => "year", "docs" => @filter_values.map { |v| v[name] } } if spec["kind"] == "year"
 
       ids = {}
       values = []
-      docs = live.map do |slot|
-        @filter_values[slot][name].map do |key, value|
+      docs = @filter_values.map do |v|
+        v[name].map do |key, value|
           ids.fetch(key) do
             values << value
             ids[key] = values.length - 1
@@ -266,45 +429,37 @@ module CJKIndex
 
     def holes? = @count < @refs.length
 
-    # Where an entry for slot and field goes in a list ordered by slot, then field.
-    def insertion_point(postings, slot, field_index)
-      i = 0
-      while i < postings.length && (postings[i] < slot || (postings[i] == slot && postings[i + 1] < field_index))
-        i += 3 + postings[i + 2]
-      end
-      i
-    end
+    # Renumbers the slots to positions among the live documents, dropping the
+    # removed ones. Renumbering keeps the order, so sorted lists stay sorted.
+    def compact
+      return unless holes?
 
-    # Postings with slots renumbered to positions among the live documents.
-    # Removed slots appear in no list any more, and renumbering keeps the
-    # order, so sorted lists stay sorted.
-    def renumbered_postings(live)
+      nf = @fields.length
+      live = @refs.each_index.reject { |slot| @refs[slot].nil? }
       number = []
       live.each_with_index { |slot, doc| number[slot] = doc }
-      @postings.sort.to_h do |token, postings|
-        postings = postings.dup
-        i = 0
-        while i < postings.length
-          postings[i] = number[postings[i]]
-          i += 3 + postings[i + 2]
-        end
-        [token, postings]
-      end
+      @postings.each_key { |token| @postings[token] = Postings.renumber(@postings[token], nf, number) }
+      @last_keys.clear
+      @refs = live.map { |slot| @refs[slot] }
+      @lengths = live.flat_map { |slot| @lengths[slot * nf, nf] }
+      @filter_values = live.map { |slot| @filter_values[slot] }
+      @doc_tokens = live.map { |slot| @doc_tokens[slot] } if @doc_tokens
+      @slots.each_value { |slots| slots.map! { |slot| number[slot] } }
+      @version += 1
     end
 
-    # Tokens of a slot. An index read with load does not record them, so the
-    # first change scans the postings once.
+    # Tokens of a slot. They are not recorded until a document is first
+    # removed or replaced; that first change scans the postings once.
     def doc_tokens(slot)
-      unless @doc_tokens_complete
-        @postings.each do |token, postings|
-          i = 0
-          while i < postings.length
-            (@doc_tokens[postings[i]] ||= []) << token
-            i += 3 + postings[i + 2]
+      unless @doc_tokens
+        nf = @fields.length
+        @doc_tokens = []
+        @postings.each do |token, list|
+          Postings.each_key(list) do |key|
+            tokens = (@doc_tokens[key / nf] ||= [])
+            tokens << token unless tokens.last.equal?(token)
           end
         end
-        @doc_tokens.each { |tokens| tokens&.uniq! }
-        @doc_tokens_complete = true
       end
       @doc_tokens[slot] || []
     end
@@ -318,7 +473,12 @@ module CJKIndex
         raise ArgumentError, "cjk_index: index has #{@lengths.length} lengths for #{@count} documents"
       end
 
-      data["tokens"].each { |token, postings| @postings[token] = postings.dup }
+      nf = @fields.length
+      if data["format"] == FORMAT
+        data["tokens"].each { |token, list| @postings[token] = +list }
+      else
+        data["tokens"].each { |token, flat| @postings[token] = Postings.encode(flat, nf) }
+      end
       filters = data["filters"] || {}
       @filter_values = Array.new(@count) do |slot|
         filters.to_h do |name, f|
@@ -328,8 +488,6 @@ module CJKIndex
           [name, value.map { |id| [Builder.filter_key(f["values"][id], @normalizer), f["values"][id]] }]
         end
       end
-      @doc_tokens = []
-      @doc_tokens_complete = false
     end
   end
 end