cjk_index 0.2.0 → 1.0.0

data/lib/cjk_index/cli.rb

@@ -0,0 +1,240 @@
+# frozen_string_literal: true
+
+require "json"
+require "optparse"
+require_relative "../cjk_index"
+
+module CJKIndex
+  # The command-line tool (exe/cjk_index) and the logic that the Rake task
+  # shares with it. Needs only Ruby's standard library; "csv" is loaded the
+  # first time a CSV file is read (see Records.csv).
+  #
+  #   cjk_index build items.csv --ref objectid --fields title,creator --boost title=3 -o index.json
+  #   cjk_index search index.json 鳥瞰図 --filter type=地図
+  #   cjk_index runtime -o cjk-index.js
+  module CLI
+    # A mistake in the arguments or the input; run prints it and returns 2.
+    class Error < StandardError; end
+
+    USAGE = <<~TEXT
+      Usage: cjk_index COMMAND [options]
+
+      Commands:
+        build     read records (CSV, JSON, JSON Lines) and write an index
+        search    query an index file
+        runtime   write the browser script
+
+      Run "cjk_index COMMAND --help" for the options of a command.
+      Options: -h, --help   -v, --version
+    TEXT
+
+    FORMATS = %w[csv json jsonl].freeze
+
+    module_function
+
+    # Runs the tool and returns the exit status: 0 done, 1 failed (input could
+    # not be read), 2 wrong usage. Output goes to the given streams.
+    def run(argv, stdin: $stdin, stdout: $stdout, stderr: $stderr)
+      argv = argv.dup
+      command = argv.shift
+      case command
+      when nil
+        stderr.puts USAGE
+        return 2
+      when "-h", "--help", "help" then stdout.puts USAGE
+      when "-v", "--version", "version" then stdout.puts VERSION
+      when "build" then build_command(argv, stdin, stdout)
+      when "search" then search_command(argv, stdout)
+      when "runtime" then runtime_command(argv, stdout)
+      else raise Error, "unknown command #{command.inspect}\n\n#{USAGE}"
+      end
+      0
+    rescue Error, OptionParser::ParseError, ArgumentError => e
+      stderr.puts "cjk_index: #{e.message.sub(/\Acjk_index: /, '')}"
+      2
+    rescue SystemCallError, JSON::ParserError, IOError => e
+      stderr.puts "cjk_index: #{e.message}"
+      1
+    end
+
+    # ---- build -------------------------------------------------------------
+
+    def build_command(argv, stdin, stdout)
+      opts = { fields: nil, boosts: {}, filters: {}, variants: Normalizer::DEFAULT_TABLES }
+      output = runtime = nil
+      parser = OptionParser.new do |o|
+        o.banner = "Usage: cjk_index build [FILE|-] [options]\n" \
+                   "Records come from FILE or standard input: CSV, a JSON array, or JSON Lines."
+        o.on("--format FORMAT", FORMATS, "csv, json or jsonl (default: from the file name, or sniffed on stdin)") { |v| opts[:format] = v }
+        o.on("--ref COLUMN", "field that identifies a record (default: first field)") { |v| opts[:ref] = v }
+        o.on("--fields LIST", Array, "fields to index (default: all but the ref)") { |v| opts[:fields] = v }
+        o.on("--boost FIELD=N", "weight of a field, repeatable (default 1)") { |v| name, n = split_pair(v, "--boost"); opts[:boosts][name] = Float(n) }
+        o.on("--filter FIELD[=KIND[:SEP]]", "filter on a field: KIND value (default) or year; SEP splits several values in one cell") do |v|
+          name, spec = parse_filter(v)
+          opts[:filters][name] = spec
+        end
+        o.on("--variants LIST", Array, "variant tables: ja (default), zh-safe, zh, ko") { |v| opts[:variants] = v }
+        o.on("-o", "--output PATH", "index file (default: standard output)") { |v| output = v }
+        o.on("--runtime PATH", "also write the browser script for these variants") { |v| runtime = v }
+      end
+      parser.parse!(argv)
+      raise Error, "too many arguments: #{argv.drop(1).join(' ')}" if argv.length > 1
+
+      source = argv.first
+      source = nil if source == "-"
+      format = opts.delete(:format) || format_of(source)
+      builder = build_index(**opts, records: read_records(source, format, stdin))
+      write_output(output, stdout, builder.to_json)
+      write_output(runtime, nil, Runtime.source(normalizer: builder.normalizer)) if runtime
+    end
+
+    # Builds an index from records (an Array of Hashes with string keys).
+    # Shared by the command and CJKIndex::RakeTask.
+    def build_index(records:, ref: nil, fields: nil, boosts: {}, filters: {}, variants: Normalizer::DEFAULT_TABLES)
+      raise Error, "no records" if records.empty?
+
+      keys = records.first.keys.map(&:to_s)
+      ref = (ref || keys.first).to_s
+      raise Error, "no field #{ref.inspect} in the records (fields: #{keys.join(', ')})" unless keys.include?(ref)
+
+      fields = Array(fields || (keys - [ref])).map(&:to_s)
+      raise Error, "no fields to index" if fields.empty?
+
+      missing = (fields + filters.keys.map(&:to_s) + boosts.keys.map(&:to_s)) - keys
+      raise Error, "no such field in the records: #{missing.uniq.join(', ')} (fields: #{keys.join(', ')})" unless missing.empty?
+
+      builder = Builder.new(fields: fields, boosts: boosts, filters: filters,
+                            normalizer: Normalizer.new(tables: Array(variants).map(&:to_s)))
+      records.each do |record|
+        value = record[ref] || record[ref.to_sym]
+        builder.add(value, record) unless value.to_s.empty?
+      end
+      builder
+    end
+
+    # ---- reading records ---------------------------------------------------
+
+    def format_of(path)
+      case File.extname(path.to_s).downcase
+      when ".csv" then "csv"
+      when ".json" then "json"
+      when ".jsonl", ".ndjson" then "jsonl"
+      end
+    end
+
+    def read_records(path, format, stdin = $stdin)
+      text = path ? File.read(path, encoding: "UTF-8") : stdin.read.to_s.dup.force_encoding("UTF-8")
+      text = text.delete_prefix("")
+      format ||= text.lstrip.start_with?("[") ? "json" : "jsonl"
+      case format
+      when "csv" then csv(text)
+      when "json"
+        data = JSON.parse(text)
+        raise Error, "the JSON must be an array of objects" unless data.is_a?(Array) && data.all?(Hash)
+
+        data
+      else
+        text.each_line.reject { |l| l.strip.empty? }.map.with_index(1) do |line, n|
+          JSON.parse(line).tap { |r| raise Error, "line #{n} is not a JSON object" unless r.is_a?(Hash) }
+        end
+      end
+    end
+
+    # CSV is a bundled gem, not a default one, since Ruby 3.4: under Bundler
+    # it can only be loaded when the Gemfile lists it. The gem does not depend
+    # on it, so it is loaded here and a missing one is reported.
+    def csv(text)
+      begin
+        require "csv"
+      rescue LoadError
+        raise Error, "reading CSV needs the csv gem (a bundled gem since Ruby 3.4); " \
+                     "run `gem install csv`, add `gem \"csv\"` to your Gemfile, or give JSON / JSON Lines instead"
+      end
+      CSV.parse(text, headers: true).map(&:to_h)
+    rescue CSV::MalformedCSVError => e
+      raise Error, "invalid CSV: #{e.message}"
+    end
+
+    # ---- search ------------------------------------------------------------
+
+    def search_command(argv, stdout)
+      filter = {}
+      limit = 10
+      json = false
+      parser = OptionParser.new do |o|
+        o.banner = "Usage: cjk_index search INDEX.json QUERY [options]"
+        o.on("--filter FIELD=VALUE", "keep matching records; repeat for several values or fields; a year filter takes 1900 or 1890..1910") do |v|
+          name, value = split_pair(v, "--filter")
+          (filter[name] ||= []) << value
+        end
+        o.on("--limit N", Integer, "results to show (default 10, 0 for all)") { |v| limit = v }
+        o.on("--json", "print JSON: [{ref, score, fields}]") { json = true }
+      end
+      parser.parse!(argv)
+      path, *words = argv
+      raise Error, "usage: cjk_index search INDEX.json QUERY" if path.nil? || (words.empty? && filter.empty?)
+
+      searcher = Searcher.load(path)
+      filter = filter.transform_values { |values| values.map { |v| filter_value(v) } }
+      filter = filter.transform_values { |values| values.length == 1 ? values.first : values }
+      results = searcher.search(words.join(" "), filter: filter.empty? ? nil : filter)
+      shown = limit.positive? ? results.first(limit) : results
+      if json
+        stdout.puts JSON.generate(shown)
+      else
+        shown.each { |r| stdout.puts format("%s\t%.3f\t%s", r[:ref], r[:score], r[:fields].join(",")) }
+        stdout.puts "(#{results.length} result#{'s' unless results.length == 1})" if shown.length < results.length
+      end
+    rescue Errno::ENOENT
+      raise Error, "no such file: #{path}"
+    end
+
+    # ---- runtime -----------------------------------------------------------
+
+    def runtime_command(argv, stdout)
+      variants = Normalizer::DEFAULT_TABLES
+      output = nil
+      OptionParser.new do |o|
+        o.banner = "Usage: cjk_index runtime [options]"
+        o.on("--variants LIST", Array, "variant tables: ja (default), zh-safe, zh, ko; must match the index") { |v| variants = v }
+        o.on("-o", "--output PATH", "script file (default: standard output)") { |v| output = v }
+      end.parse!(argv)
+      raise Error, "too many arguments: #{argv.join(' ')}" unless argv.empty?
+
+      write_output(output, stdout, Runtime.source(normalizer: Normalizer.new(tables: variants)))
+    end
+
+    # ---- helpers -----------------------------------------------------------
+
+    def split_pair(text, option)
+      name, value = text.split("=", 2)
+      raise Error, "#{option} takes NAME=VALUE, got #{text.inspect}" if name.to_s.empty? || value.nil?
+
+      [name, value]
+    end
+
+    # "type", "date=year", "subject=value:;"
+    def parse_filter(text)
+      name, spec = text.split("=", 2)
+      kind, separator = (spec || "value").split(":", 2)
+      raise Error, "--filter #{text.inspect}: kind must be value or year" unless Builder::FILTER_KINDS.include?(kind)
+
+      [name, separator ? { "kind" => kind, "separator" => separator } : kind]
+    end
+
+    def filter_value(text)
+      return text unless (m = text.match(/\A(\d*)\.\.(\d*)\z/)) && !(m[1].empty? && m[2].empty?)
+
+      { "min" => (m[1].empty? ? nil : m[1].to_i), "max" => (m[2].empty? ? nil : m[2].to_i) }.compact
+    end
+
+    def write_output(path, stdout, content)
+      if path
+        File.write(path, content)
+      else
+        stdout.write(content)
+        stdout.write("\n") unless content.end_with?("\n")
+      end
+    end
+  end
+end

data/lib/cjk_index/index_file.rb

@@ -27,6 +27,10 @@ module CJKIndex
   #
   # Locks use flock, which is reliable on a local disk; do not share the file
   # over NFS.
+  #
+  # The index is always one file (format 4, as Builder#write). A split index
+  # (Builder#write_split) is not supported here: it is meant for static
+  # sites, written once per build.
   class IndexFile
     # Raised by update and write when another writer holds the lock too long.
     class LockTimeout < StandardError; end

data/lib/cjk_index/jekyll.rb

@@ -25,8 +25,15 @@ module CJKIndex
   #           date: year
   #       - name: search
   #         preset: collectionbuilder     # reads CollectionBuilder's own config
+  #       - name: texts
+  #         collection: texts
+  #         fields: [title, content]
+  #         split: true                   # -> assets/cjk-index/texts/index.json and shards
+  #         shard_size: 32000             # bytes per shard (optional)
   #
-  # The browser script is written to <output>/cjk-index.js.
+  # The browser script is written to <output>/cjk-index.js. A split index is
+  # loaded with CJKIndex.load('<output>/<name>/index.json') and searched with
+  # searchAsync, which fetches only the shards a query needs (see Builder).
   module Jekyll
     DEFAULT_OUTPUT = "assets/cjk-index"
     RUNTIME_NAME = "cjk-index.js"
@@ -61,9 +68,18 @@ module CJKIndex
         site.static_files << GeneratedFile.new(site, output, RUNTIME_NAME, Runtime.source(normalizer: normalizer))
         Array(config["indexes"]).each do |spec|
           builder = build(site, spec, normalizer)
-          name = "#{spec.fetch('name')}.json"
-          site.static_files << GeneratedFile.new(site, output, name, builder.to_json)
-          ::Jekyll.logger.info "cjk_index:", "#{output}/#{name} (#{builder.refs.length} documents)"
+          if spec["split"]
+            dir = "#{output}/#{spec.fetch('name')}"
+            files = builder.each_split_file(shard_size: Integer(spec.fetch("shard_size", Builder::SHARD_SIZE)))
+                           .map { |name, content| GeneratedFile.new(site, dir, name, content) }
+            site.static_files.concat(files)
+            ::Jekyll.logger.info "cjk_index:", "#{dir}/#{Builder::MANIFEST} (#{builder.refs.length} documents, " \
+                                               "#{files.length - 2} shards)"
+          else
+            name = "#{spec.fetch('name')}.json"
+            site.static_files << GeneratedFile.new(site, output, name, builder.to_json)
+            ::Jekyll.logger.info "cjk_index:", "#{output}/#{name} (#{builder.refs.length} documents)"
+          end
         end
       end
 
@@ -79,7 +95,7 @@ module CJKIndex
           docs = site.collections.fetch(spec["collection"]) do
             raise ::Jekyll::Errors::FatalException, "cjk_index: no collection #{spec['collection']}"
           end.docs
-          docs.each { |doc| builder.add(doc.url, document_fields(doc, builder.fields)) }
+          docs.each { |doc| builder.add(doc.url, document_fields(doc, builder.fields | builder.filters.keys)) }
         elsif spec["data"]
           ref = spec.fetch("ref", "id")
           Array(site.data[spec["data"]]).each do |row|

data/lib/cjk_index/normalizer.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "digest"
+require "json"
+
 module CJKIndex
   # Folds text so that spelling variants a reader treats as "the same" compare
   # equal: NFKC (full-width -> half-width etc.), lowercase, katakana ->
@@ -9,18 +12,29 @@ module CJKIndex
   # The same rules run in the browser (see Runtime), so an index built here
   # and a query typed there are folded identically.
   #
-  # Variant tables (data/*.tsv):
-  #   ja  Japanese old forms -> modern forms (圖 -> 図). Default.
-  #   zh  Traditional -> Simplified Chinese, from OpenCC. Opt in: it also
-  #       merges characters that are distinct in Japanese (後 and 后).
+  # Variant tables (data/*.tsv, regenerated by script/variant_tables.rb):
+  #   ja       Japanese old forms and glyph variants -> one form (圖 -> 図),
+  #            from the Joyo and Jinmeiyo kanji lists and Unihan. Default.
+  #   zh       Traditional -> Simplified Chinese, from OpenCC. Opt in: it also
+  #            merges characters that are distinct in Japanese (後 and 后).
+  #   zh-safe  The pairs of zh that are one-to-one both ways (圖 -> 图 but
+  #            not 後 -> 后 or 發/髮 -> 发). Opt in.
+  #   ko       Old Hangul in the Hanyang Private Use Area -> standard jamo
+  #            sequences (one character -> several). Opt in.
   # With several tables, characters linked by any table fold to one form,
   # preferring the target of the table listed first: with [ja, zh],
   # 圖, 図 and 图 all become 図.
+  #
+  # Table format: UTF-8 lines "variant<TAB>folded", optionally followed by
+  # another tab and a comment column; lines starting with "#" are comments.
+  # "folded" may be several characters.
   class Normalizer
     DATA_DIR = File.expand_path("../../data", __dir__)
     TABLES = {
       "ja" => "ja-kyujitai.tsv",
-      "zh" => "zh-hant-hans.tsv"
+      "zh" => "zh-hant-hans.tsv",
+      "zh-safe" => "zh-hant-hans-safe.tsv",
+      "ko" => "ko-hanyang-pua.tsv"
     }.freeze
     DEFAULT_TABLES = %w[ja].freeze
 
@@ -38,31 +52,65 @@ module CJKIndex
       file = TABLES.fetch(name) { raise ArgumentError, "unknown variant table #{name.inspect} (#{TABLES.keys.join(', ')})" }
       File.readlines(File.join(DATA_DIR, file), chomp: true, encoding: "UTF-8")
           .reject { |line| line.empty? || line.start_with?("#") }
-          .map { |line| line.split("\t", 2) }
+          .map { |line| line.split("\t")[0, 2] }
     end
 
-    attr_reader :tables, :variants
+    attr_reader :tables, :variants, :digest
+
+    KATAKANA = "#{KATAKANA_FIRST.chr(Encoding::UTF_8)}-#{KATAKANA_LAST.chr(Encoding::UTF_8)}"
+    HIRAGANA = "#{(KATAKANA_FIRST - KANA_OFFSET).chr(Encoding::UTF_8)}-#{(KATAKANA_LAST - KANA_OFFSET).chr(Encoding::UTF_8)}"
 
     def initialize(tables: DEFAULT_TABLES)
       @tables = tables.map(&:to_s).uniq.freeze
       @variants = build_variants.freeze
+      keys = @variants.keys.select { |k| k.length == 1 }
+      @variant_chars = keys.empty? ? nil : Regexp.new("[#{keys.map { |k| format('\\u{%x}', k.ord) }.join}]")
+      # What the tables fold, not their names: an index built with an older
+      # "ja" table must not be read with a newer one.
+      @digest = Digest::SHA256.hexdigest(JSON.generate(@variants.sort)).slice(0, 16).freeze
+    end
+
+    # Raises unless data (an index) was built with the same tables as this
+    # normalizer. An index without a digest predates 1.0: it still loads, but
+    # it folded with older tables and should be rebuilt.
+    def check!(data)
+      if tables != data["variants"]
+        raise ArgumentError, "cjk_index: index built with variant tables #{data['variants'].inspect} " \
+                             "but this normalizer folds with #{tables.inspect}; rebuild both together"
+      end
+      return if data["variant_digest"].nil? || data["variant_digest"] == digest
+
+      raise ArgumentError, "cjk_index: index built with other contents of the variant tables " \
+                           "#{tables.inspect} (an older cjk_index?); rebuild it"
     end
 
+    # Character by character: katakana to hiragana, then the variant tables,
+    # then 々 to the (folded) character before it. Done as passes over the
+    # whole string, which gives the same result as one loop over the
+    # characters (the loop in runtime.js) and is several times faster in Ruby.
     def normalize(text)
       return "" if text.nil?
 
+      s = text.to_s.unicode_normalize(:nfkc).downcase
+      s = s.tr(KATAKANA, HIRAGANA)
+      # 々 repeats the folded character before it, which may be several
+      # characters (ko): fold one character at a time, as runtime.js does
+      return iterate(s) if s.include?(ITERATION_MARK)
+
+      @variant_chars ? s.gsub(@variant_chars, @variants) : s
+    end
+
+    private
+
+    def iterate(s)
       prev = nil
-      text.to_s.unicode_normalize(:nfkc).downcase.each_char.map do |ch|
-        code = ch.ord
-        ch = (code - KANA_OFFSET).chr(Encoding::UTF_8) if code.between?(KATAKANA_FIRST, KATAKANA_LAST)
+      s.each_char.map do |ch|
         ch = @variants.fetch(ch, ch)
         ch = prev if ch == ITERATION_MARK && prev
         prev = ch
       end.join
     end
 
-    private
-
     # Union-find over all pairs; each class folds to the target preferred by
     # the earliest table (lowest code point among equals, for determinism).
     def build_variants

data/lib/cjk_index/postings.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+module CJKIndex
+  # The compact encoding of a posting list in format 4, shared by Builder
+  # (writing) and Searcher (reading); runtime.js decodes it the same way.
+  #
+  # A list is a sequence of entries, ordered by document and field. Each
+  # entry is written as numbers:
+  #
+  #   key gap, n, first position, gap to the next position, ...
+  #
+  # where key = document * number of fields + field, and the key gap is the
+  # difference to the key of the previous entry minus 1 (the first entry
+  # counts from -1). n is the number of positions. Each number is written in
+  # printable ASCII: the last digit in base 60 from the first 60 characters
+  # of ALPHABET, the digits before it in base 30 from the other 30, most
+  # significant first. 0..59 take one character, up to 1,799 two, up to
+  # 53,999 three. ALPHABET has neither " nor \, so a list is a JSON string
+  # without escapes.
+  module Postings
+    ALPHABET = ((0x23..0x7d).map(&:chr) - ["\\"]).join.freeze
+    LAST = 60
+    LEAD = 30
+    # The characters that end a number are the bytes below LEAD_BYTE
+    # (FINAL_BYTES in the notation of String#count).
+    LEAD_BYTE = ALPHABET.getbyte(LAST)
+    FINAL_BYTES = "#-[\\]-_"
+    # byte => digit; 0...60 ends a number, 60...90 leads.
+    DIGIT = Array.new(256, -1).tap { |t| ALPHABET.each_byte.with_index { |b, i| t[b] = i } }.freeze
+
+    def self.number_slow(value)
+      s = ALPHABET[value % LAST]
+      value /= LAST
+      while value.positive?
+        s = ALPHABET[LAST + (value % LEAD)] + s
+        value /= LEAD
+      end
+      s
+    end
+
+    # Every number of up to three characters, written once.
+    NUMBERS = Array.new(LAST * LEAD * LEAD) { |v| number_slow(v).freeze }.freeze
+
+    module_function
+
+    def number(value) = NUMBERS[value] || number_slow(value)
+
+    # Appends one entry to out. offsets are ascending positions.
+    def append(out, key_gap, offsets)
+      out << number(key_gap) << number(offsets.length)
+      last = 0
+      offsets.each do |pos|
+        out << number(pos - last)
+        last = pos
+      end
+      out
+    end
+
+    # All numbers of a list.
+    def numbers(str)
+      out = []
+      acc = 0
+      str.each_byte do |b|
+        d = DIGIT[b]
+        if d >= LAST then acc = (acc * LEAD) + d - LAST
+        else
+          out << ((acc * LAST) + d)
+          acc = 0
+        end
+      end
+      out
+    end
+
+    # Yields key, start, body and stop for each entry: its key, and the byte
+    # offsets where the entry starts, where its key gap ends and where it
+    # ends. Loops over bytes rather than numbers, so that the changes below
+    # can copy the rest of a list as it is.
+    def each_entry(str)
+      return enum_for(:each_entry, str) unless block_given?
+
+      key = -1
+      i = 0
+      len = str.bytesize
+      while i < len
+        start = i
+        acc = 0
+        while (d = DIGIT[str.getbyte(i)]) >= LAST
+          acc = (acc * LEAD) + d - LAST
+          i += 1
+        end
+        key += (acc * LAST) + d + 1
+        i += 1
+        body = i
+        acc = 0
+        while (d = DIGIT[str.getbyte(i)]) >= LAST
+          acc = (acc * LEAD) + d - LAST
+          i += 1
+        end
+        n = (acc * LAST) + d
+        i += 1
+        # Skip n numbers: each ends with a byte below LEAD_BYTE. Many at a
+        # time are counted by String#count, which runs in C.
+        while n > 8
+          t = str.byteslice(i, n).count(FINAL_BYTES)
+          i += n
+          n -= t
+        end
+        while n.positive?
+          n -= 1 if str.getbyte(i) < LEAD_BYTE
+          i += 1
+        end
+        yield key, start, body, i
+      end
+    end
+
+    # Key of the last entry, -1 for an empty list.
+    def last_key(str)
+      key = -1
+      each_entry(str) { |k| key = k }
+      key
+    end
+
+    # Each key of a list, in order.
+    def each_key(str)
+      return enum_for(:each_key, str) unless block_given?
+
+      each_entry(str) { |key| yield key }
+    end
+
+    # The list without the entries whose keys are in range (consecutive).
+    def delete(str, range)
+      before = -1
+      from = nil
+      each_entry(str) do |key, start, body|
+        if range.cover?(key)
+          from ||= start
+        elsif from
+          return str.byteslice(0, from) + number(key - before - 1) + str.byteslice(body..)
+        elsif key > range.end
+          return str
+        else
+          before = key
+        end
+      end
+      from ? str.byteslice(0, from) : str
+    end
+
+    # The list with an entry for key (not yet in it) and offsets inserted in order.
+    def insert(str, key, offsets)
+      before = -1
+      each_entry(str) do |k, start, body|
+        if k > key
+          entry = append(+"", key - before - 1, offsets)
+          return str.byteslice(0, start) + entry + number(k - key - 1) + str.byteslice(body..)
+        end
+        before = k
+      end
+      append(+str, key - before - 1, offsets)
+    end
+
+    # The list with each key's document replaced by numbering[document], which
+    # keeps the order (removed documents have no entries).
+    def renumber(str, fields, numbering)
+      out = +""
+      old_before = -1
+      new_before = -1
+      copied = 0 # str up to here is in out or to be copied as it is
+      each_entry(str) do |key, start, body|
+        new_key = (numbering[key / fields] * fields) + (key % fields)
+        if new_key - new_before != key - old_before
+          out << str.byteslice(copied, start - copied) << number(new_key - new_before - 1)
+          copied = body
+        end
+        old_before = key
+        new_before = new_key
+      end
+      copied.zero? ? str : out << str.byteslice(copied..)
+    end
+
+    # The flat list of format 3, [doc, field, n, pos, ..., doc, ...].
+    def decode(str, fields)
+      nums = numbers(str)
+      out = []
+      key = -1
+      i = 0
+      while i < nums.length
+        key += nums[i] + 1
+        n = nums[i + 1]
+        out << (key / fields) << (key % fields) << n
+        pos = 0
+        j = i + 2
+        stop = j + n
+        while j < stop
+          pos += nums[j]
+          out << pos
+          j += 1
+        end
+        i = stop
+      end
+      out
+    end
+
+    # A flat list of format 2 or 3 in this encoding.
+    def encode(flat, fields)
+      out = +""
+      last = -1
+      i = 0
+      while i < flat.length
+        key = (flat[i] * fields) + flat[i + 1]
+        n = flat[i + 2]
+        append(out, key - last - 1, flat[i + 3, n])
+        last = key
+        i += 3 + n
+      end
+      out
+    end
+  end
+end