cjk_index 0.2.0 → 1.1.0

data/lib/cjk_index/searcher.rb

@@ -28,13 +28,25 @@ module CJKIndex
     # documents, also after later add, remove and update.
     # prefix: the last Latin word of a query also matches longer words.
     # typos: Latin words of 5+ letters also match words one edit away.
+    #
+    # path is an index file, or a split index (its directory or its
+    # manifest); the shards of a split index are read as queries need them.
     def self.load(path, **options)
-      new(File.read(path, encoding: "UTF-8"), **options)
+      path = File.join(path, Builder::MANIFEST) if File.directory?(path)
+      data = JSON.parse(File.read(path, encoding: "UTF-8"))
+      if data.is_a?(Hash) && data["shards"]
+        dir = File.dirname(path)
+        options = { load_shard: ->(file) { JSON.parse(File.read(File.join(dir, file), encoding: "UTF-8")) } }.merge(options)
+      end
+      new(data, **options)
     end
 
-    def initialize(index, prefix: true, typos: true, normalizer: nil)
+    # load_shard: for the manifest of a split index, a callable that returns
+    # a shard (a Hash of token => posting list) given its file name.
+    def initialize(index, prefix: true, typos: true, normalizer: nil, load_shard: nil)
       @prefix = prefix
       @typos = typos
+      @load_shard = load_shard
       if index.is_a?(Builder)
         @builder = index
         data = index.live_data
@@ -45,10 +57,11 @@ module CJKIndex
         end
       end
 
-      @normalizer = normalizer || Normalizer.new(tables: data["variants"])
-      if @normalizer.tables != data["variants"]
-        raise ArgumentError, "cjk_index: index built with variant tables #{data['variants'].inspect} " \
-                             "but this normalizer folds with #{@normalizer.tables.inspect}; rebuild both together"
+      @normalizer = normalizer || @builder&.normalizer || Normalizer.new(tables: data["variants"])
+      @normalizer.check!(data)
+
+      if data["shards"] && !@load_shard
+        raise ArgumentError, "cjk_index: a split index needs load_shard (or Searcher.load(directory))"
       end
 
       setup(data)
@@ -149,7 +162,7 @@ module CJKIndex
       docs.each do |doc|
         value = filter[:docs][doc]
         if filter[:kind] == "year" then counts[value] += 1 unless value.nil?
-        else value.each { |key| counts[key] += 1 }
+        else value.each { |id| counts[filter[:keys][id]] += 1 }
         end
       end
       counts.sort_by { |key, count| [-count, filter[:rank][key]] }
@@ -169,39 +182,54 @@ module CJKIndex
       @refs = data["refs"]
       @lengths = data["lengths"]
       @tokens = data["tokens"]
+      @encoded = data["format"] == Builder::FORMAT
+      @decoded = {}
+      if data["shards"]
+        @shard_first = data["shards"].map(&:first)
+        @shard_file = data["shards"].map(&:last)
+        @shards = {}
+      end
       @count = data["count"] || @refs.length
       nf = @fields.length
       avg = Array.new(nf, 0.0)
       @lengths.each_with_index { |len, i| avg[i % nf] += len if len }
       @avg_length = avg.map { |s| (v = s / [1, @count].max).zero? ? 1 : v }
-      @latin_tokens = @tokens.keys.select { |t| latin?(t) }.sort
+      # A split index lists its Latin tokens in a file, read only for typos.
+      @latin_file = data["latin"] unless @tokens
+      @latin_tokens = @latin_file ? nil : @tokens.keys.select { |t| latin?(t) }.sort
+      @latin_set = @latin_tokens&.to_set
       @slot_of = {}
       @refs.each_with_index { |ref, doc| @slot_of[ref] ||= doc if ref }
       @filters = (data["filters"] || {}).to_h { |name, f| [name, setup_filter(f)] }
     end
 
-    # Filter values by document: keys (value filters) or a year, with the
+    # Filter values by document: a year, or the ids of values (value
+    # filters), with the folded key of each id, the ids of each key, the
     # form to show for each key and its rank of first appearance.
     def setup_filter(f)
-      docs = if f["kind"] == "year" then f["docs"]
-             elsif f["values"]
-               keys = f["values"].map { |v| Builder.filter_key(v, @normalizer) }
-               f["docs"].map { |ids| ids.map { |id| keys[id] } }
-             else
-               f["docs"].map { |pairs| pairs&.map(&:first) }
-             end
-      display = {}
-      if f["values"] then f["values"].each { |v| display[Builder.filter_key(v, @normalizer)] ||= v }
-      elsif f["kind"] == "value" then f["docs"].each { |pairs| pairs&.each { |key, v| display[key] ||= v } }
+      if f["kind"] == "year"
+        rank = {}
+        f["docs"].each_with_index { |year, doc| rank[year] ||= rank.size if @refs[doc] && !year.nil? }
+        return { kind: "year", docs: f["docs"], display: rank.keys.to_h { |year| [year, year] }, rank: rank }
       end
-      rank = {}
-      docs.each_with_index do |value, doc|
-        next unless @refs[doc]
 
-        Array(value).each { |key| rank[key] ||= rank.size }
+      values = f["values"]
+      docs = f["docs"]
+      unless values
+        id_of = {}
+        values = []
+        docs = docs.map { |pairs| (pairs || []).map { |key, v| id_of[key] ||= (values << v).size - 1 } }
       end
-      display = rank.keys.to_h { |year| [year, year] } if f["kind"] == "year"
-      { kind: f["kind"], docs: docs, display: display, rank: rank }
+      keys = values.map { |v| Builder.filter_key(v, @normalizer) }
+      ids = Hash.new { |h, key| h[key] = [] }
+      keys.each_with_index { |key, id| ids[key] << id }
+      display = {}
+      values.each_with_index { |v, id| display[keys[id]] ||= v }
+      rank = {}
+      docs.each_with_index do |list, doc|
+        list.each { |id| rank[keys[id]] ||= rank.size } if @refs[doc]
+      end
+      { kind: f["kind"], docs: docs, keys: keys, ids: ids, display: display, rank: rank }
     end
 
     def refresh
@@ -222,8 +250,9 @@ module CJKIndex
                   end
           [f, ->(year) { !year.nil? && range.cover?(year) }]
         else
-          keys = Array(condition).map { |v| Builder.filter_key(v, @normalizer) }.to_set
-          [f, ->(values) { values.any? { |key| keys.include?(key) } }]
+          # Compares value ids, not folded strings.
+          ids = Array(condition).flat_map { |v| f[:ids].fetch(Builder.filter_key(v, @normalizer), []) }.to_set
+          [f, ->(list) { list.any? { |id| ids.include?(id) } }]
         end
       end
     end
@@ -237,7 +266,7 @@ module CJKIndex
       nf = @fields.length
       map = {}
       tokens.each do |token|
-        postings = @tokens.fetch(token, nil) or next
+        postings = postings(token) or next
         i = 0
         while i < postings.length
           count = postings[i + 2]
@@ -248,21 +277,56 @@ module CJKIndex
       map
     end
 
+    # Number of the shard that holds token (split index), or -1.
+    def shard_index(token) = (@shard_first.bsearch_index { |first| first > token } || @shard_first.length) - 1
+
+    def shard(i) = @shards[@shard_file[i]] ||= @load_shard.call(@shard_file[i])
+
+    # The Hash that holds token's posting list if it is in the index, or nil.
+    def table(token)
+      return @tokens unless @shard_first
+
+      i = shard_index(token)
+      i.negative? ? nil : shard(i)
+    end
+
+    def token?(token) = !!table(token)&.key?(token)
+
+    # The posting list of a token as [doc, field, n, pos, ...], or nil.
+    def postings(token)
+      list = table(token)&.[](token)
+      return list unless list && @encoded
+
+      @decoded[token] ||= Postings.decode(list, @fields.length)
+    end
+
+    # Index words that start with token, token itself left out.
+    def with_prefix(token)
+      unless @shard_first
+        from = @latin_tokens.bsearch_index { |k| k >= token } || @latin_tokens.length
+        return @latin_tokens[from..].take_while { |k| k.start_with?(token) }.reject { |k| k == token }
+      end
+
+      # In token's shard and the next ones that start with it (tokens are sorted).
+      first = [shard_index(token), 0].max
+      last = first
+      last += 1 while last + 1 < @shard_first.length && @shard_first[last + 1].start_with?(token)
+      (first..last).flat_map { |i| shard(i).keys.select { |k| k != token && k.start_with?(token) } }
+    end
+
+    def latin_tokens
+      @latin_tokens ||= @load_shard.call(@latin_file).sort
+    end
+
     # Alternatives for one query token: itself, and for Latin words the index
     # words it is a prefix of (last word only) or one edit away from.
     def alternatives(token, last)
       return [[token], false] unless latin?(token)
 
-      out = @tokens.key?(token) ? [token] : []
-      if @prefix && last
-        from = @latin_tokens.bsearch_index { |k| k >= token } || @latin_tokens.length
-        @latin_tokens[from..].each do |k|
-          break unless k.start_with?(token)
-
-          out << k unless k == token
-        end
-      end
-      out.concat(@latin_tokens.select { |k| one_edit?(token, k) }) if @typos && !@tokens.key?(token) && token.length >= 5
+      has = @shard_first ? token?(token) : @latin_set.include?(token)
+      out = has ? [token] : []
+      out.concat(with_prefix(token)) if @prefix && last
+      out.concat(latin_tokens.select { |k| one_edit?(token, k) }) if @typos && !has && token.length >= 5
       [out, !out.empty? && out.first != token]
     end
 

data/lib/cjk_index/tokenizer.rb

@@ -68,6 +68,36 @@ module CJKIndex
       phrases(text, unigrams: unigrams, normalizer: normalizer).flatten(1)
     end
 
+    # A run of separators, of CJK characters, or of other characters. A
+    # separator is never part of a run (・ is in the katakana block).
+    CJK_WORD = /[#{CJK.source[1..-2]}&&[^#{SEPARATOR.source[1..-2]}]]/
+    RUN = /(#{SEPARATOR.source}+)|(#{CJK_WORD.source}+)|((?:(?!#{SEPARATOR.source})(?!#{CJK.source}).)+)/m
+
+    # Yields each token of already normalized text with its offset: the
+    # tokens of tokens_with_offsets, without building the phrases. Used to
+    # build the index, where it is several times faster.
+    def each_token(normalized, unigrams: false)
+      offset = 0
+      normalized.scan(RUN) do |_, cjk, other|
+        run = Regexp.last_match(0)
+        if cjk
+          chars = run.chars
+          if chars.length == 1 || unigrams
+            chars.each_with_index { |c, i| yield c, offset + i }
+          end
+          i = 0
+          last = chars.length - 1
+          while i < last
+            yield chars[i] + chars[i + 1], offset + i
+            i += 1
+          end
+        elsif other
+          yield run, offset
+        end
+        offset += run.length
+      end
+    end
+
     # Splits at separators (spaces, punctuation) into phrases, each a list of
     # [token, offset]. A query matches when every phrase occurs somewhere in
     # the document, with the tokens of each phrase at the same relative offsets.

data/lib/cjk_index/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CJKIndex
-  VERSION = "0.2.0"
+  VERSION = "1.1.0"
 end

data/lib/cjk_index.rb

@@ -3,6 +3,7 @@
 require_relative "cjk_index/version"
 require_relative "cjk_index/normalizer"
 require_relative "cjk_index/tokenizer"
+require_relative "cjk_index/postings"
 require_relative "cjk_index/builder"
 require_relative "cjk_index/runtime"
 require_relative "cjk_index/searcher"

metadata

@@ -1,33 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: cjk_index
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Satoru Nakamura
-bindir: bin
+bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies: []
 description: Builds a bigram search index in Ruby (with kana, old/new kanji and NFKC
   normalization), and queries it in Ruby or with a small browser runtime that returns
-  the same results. Includes a Jekyll plugin and a CollectionBuilder preset.
-executables: []
+  the same results. Includes a command-line tool, a Rake task, a Jekyll plugin (with
+  a CollectionBuilder preset) and a Bridgetown plugin.
+executables:
+- cjk_index
 extensions: []
 extra_rdoc_files: []
 files:
 - CHANGELOG.md
 - LICENSE
 - LICENSE-OpenCC
+- LICENSE-Unicode
 - NOTICE
 - README.md
 - data/ja-kyujitai.tsv
+- data/ko-hanyang-pua.tsv
+- data/zh-hant-hans-safe.tsv
 - data/zh-hant-hans.tsv
+- exe/cjk_index
 - lib/cjk_index.rb
+- lib/cjk_index/bridgetown.rb
 - lib/cjk_index/builder.rb
+- lib/cjk_index/cli.rb
 - lib/cjk_index/index_file.rb
 - lib/cjk_index/jekyll.rb
 - lib/cjk_index/normalizer.rb
+- lib/cjk_index/postings.rb
+- lib/cjk_index/rake_task.rb
 - lib/cjk_index/runtime.js
 - lib/cjk_index/runtime.rb
 - lib/cjk_index/searcher.rb
@@ -37,6 +47,8 @@ homepage: https://github.com/nakamura196/cjk_index
 licenses:
 - MIT
 - Apache-2.0
+- Unicode-3.0
+- Unlicense
 metadata:
   source_code_uri: https://github.com/nakamura196/cjk_index
   documentation_uri: https://github.com/nakamura196/cjk_index#readme