cjk_index 0.2.0 → 1.1.0

data/lib/cjk_index/cli.rb

@@ -0,0 +1,240 @@
+# frozen_string_literal: true
+
+require "json"
+require "optparse"
+require_relative "../cjk_index"
+
+module CJKIndex
+  # The command-line tool (exe/cjk_index) and the logic that the Rake task
+  # shares with it. Needs only Ruby's standard library; "csv" is loaded the
+  # first time a CSV file is read (see Records.csv).
+  #
+  #   cjk_index build items.csv --ref objectid --fields title,creator --boost title=3 -o index.json
+  #   cjk_index search index.json 鳥瞰図 --filter type=地図
+  #   cjk_index runtime -o cjk-index.js
+  module CLI
+    # A mistake in the arguments or the input; run prints it and returns 2.
+    class Error < StandardError; end
+
+    USAGE = <<~TEXT
+      Usage: cjk_index COMMAND [options]
+
+      Commands:
+        build     read records (CSV, JSON, JSON Lines) and write an index
+        search    query an index file
+        runtime   write the browser script
+
+      Run "cjk_index COMMAND --help" for the options of a command.
+      Options: -h, --help   -v, --version
+    TEXT
+
+    FORMATS = %w[csv json jsonl].freeze
+
+    module_function
+
+    # Runs the tool and returns the exit status: 0 done, 1 failed (input could
+    # not be read), 2 wrong usage. Output goes to the given streams.
+    def run(argv, stdin: $stdin, stdout: $stdout, stderr: $stderr)
+      argv = argv.dup
+      command = argv.shift
+      case command
+      when nil
+        stderr.puts USAGE
+        return 2
+      when "-h", "--help", "help" then stdout.puts USAGE
+      when "-v", "--version", "version" then stdout.puts VERSION
+      when "build" then build_command(argv, stdin, stdout)
+      when "search" then search_command(argv, stdout)
+      when "runtime" then runtime_command(argv, stdout)
+      else raise Error, "unknown command #{command.inspect}\n\n#{USAGE}"
+      end
+      0
+    rescue Error, OptionParser::ParseError, ArgumentError => e
+      stderr.puts "cjk_index: #{e.message.sub(/\Acjk_index: /, '')}"
+      2
+    rescue SystemCallError, JSON::ParserError, IOError => e
+      stderr.puts "cjk_index: #{e.message}"
+      1
+    end
+
+    # ---- build -------------------------------------------------------------
+
+    def build_command(argv, stdin, stdout)
+      opts = { fields: nil, boosts: {}, filters: {}, variants: Normalizer::DEFAULT_TABLES }
+      output = runtime = nil
+      parser = OptionParser.new do |o|
+        o.banner = "Usage: cjk_index build [FILE|-] [options]\n" \
+                   "Records come from FILE or standard input: CSV, a JSON array, or JSON Lines."
+        o.on("--format FORMAT", FORMATS, "csv, json or jsonl (default: from the file name, or sniffed on stdin)") { |v| opts[:format] = v }
+        o.on("--ref COLUMN", "field that identifies a record (default: first field)") { |v| opts[:ref] = v }
+        o.on("--fields LIST", Array, "fields to index (default: all but the ref)") { |v| opts[:fields] = v }
+        o.on("--boost FIELD=N", "weight of a field, repeatable (default 1)") { |v| name, n = split_pair(v, "--boost"); opts[:boosts][name] = Float(n) }
+        o.on("--filter FIELD[=KIND[:SEP]]", "filter on a field: KIND value (default) or year; SEP splits several values in one cell") do |v|
+          name, spec = parse_filter(v)
+          opts[:filters][name] = spec
+        end
+        o.on("--variants LIST", Array, "variant tables: ja (default), zh-safe, zh, ko, or table files") { |v| opts[:variants] = v }
+        o.on("-o", "--output PATH", "index file (default: standard output)") { |v| output = v }
+        o.on("--runtime PATH", "also write the browser script for these variants") { |v| runtime = v }
+      end
+      parser.parse!(argv)
+      raise Error, "too many arguments: #{argv.drop(1).join(' ')}" if argv.length > 1
+
+      source = argv.first
+      source = nil if source == "-"
+      format = opts.delete(:format) || format_of(source)
+      builder = build_index(**opts, records: read_records(source, format, stdin))
+      write_output(output, stdout, builder.to_json)
+      write_output(runtime, nil, Runtime.source(normalizer: builder.normalizer)) if runtime
+    end
+
+    # Builds an index from records (an Array of Hashes with string keys).
+    # Shared by the command and CJKIndex::RakeTask.
+    def build_index(records:, ref: nil, fields: nil, boosts: {}, filters: {}, variants: Normalizer::DEFAULT_TABLES)
+      raise Error, "no records" if records.empty?
+
+      keys = records.first.keys.map(&:to_s)
+      ref = (ref || keys.first).to_s
+      raise Error, "no field #{ref.inspect} in the records (fields: #{keys.join(', ')})" unless keys.include?(ref)
+
+      fields = Array(fields || (keys - [ref])).map(&:to_s)
+      raise Error, "no fields to index" if fields.empty?
+
+      missing = (fields + filters.keys.map(&:to_s) + boosts.keys.map(&:to_s)) - keys
+      raise Error, "no such field in the records: #{missing.uniq.join(', ')} (fields: #{keys.join(', ')})" unless missing.empty?
+
+      builder = Builder.new(fields: fields, boosts: boosts, filters: filters,
+                            normalizer: Normalizer.new(tables: Array(variants).map(&:to_s)))
+      records.each do |record|
+        value = record[ref] || record[ref.to_sym]
+        builder.add(value, record) unless value.to_s.empty?
+      end
+      builder
+    end
+
+    # ---- reading records ---------------------------------------------------
+
+    def format_of(path)
+      case File.extname(path.to_s).downcase
+      when ".csv" then "csv"
+      when ".json" then "json"
+      when ".jsonl", ".ndjson" then "jsonl"
+      end
+    end
+
+    def read_records(path, format, stdin = $stdin)
+      text = path ? File.read(path, encoding: "UTF-8") : stdin.read.to_s.dup.force_encoding("UTF-8")
+      text = text.delete_prefix("")
+      format ||= text.lstrip.start_with?("[") ? "json" : "jsonl"
+      case format
+      when "csv" then csv(text)
+      when "json"
+        data = JSON.parse(text)
+        raise Error, "the JSON must be an array of objects" unless data.is_a?(Array) && data.all?(Hash)
+
+        data
+      else
+        text.each_line.reject { |l| l.strip.empty? }.map.with_index(1) do |line, n|
+          JSON.parse(line).tap { |r| raise Error, "line #{n} is not a JSON object" unless r.is_a?(Hash) }
+        end
+      end
+    end
+
+    # CSV is a bundled gem, not a default one, since Ruby 3.4: under Bundler
+    # it can only be loaded when the Gemfile lists it. The gem does not depend
+    # on it, so it is loaded here and a missing one is reported.
+    def csv(text)
+      begin
+        require "csv"
+      rescue LoadError
+        raise Error, "reading CSV needs the csv gem (a bundled gem since Ruby 3.4); " \
+                     "run `gem install csv`, add `gem \"csv\"` to your Gemfile, or give JSON / JSON Lines instead"
+      end
+      CSV.parse(text, headers: true).map(&:to_h)
+    rescue CSV::MalformedCSVError => e
+      raise Error, "invalid CSV: #{e.message}"
+    end
+
+    # ---- search ------------------------------------------------------------
+
+    def search_command(argv, stdout)
+      filter = {}
+      limit = 10
+      json = false
+      parser = OptionParser.new do |o|
+        o.banner = "Usage: cjk_index search INDEX.json QUERY [options]"
+        o.on("--filter FIELD=VALUE", "keep matching records; repeat for several values or fields; a year filter takes 1900 or 1890..1910") do |v|
+          name, value = split_pair(v, "--filter")
+          (filter[name] ||= []) << value
+        end
+        o.on("--limit N", Integer, "results to show (default 10, 0 for all)") { |v| limit = v }
+        o.on("--json", "print JSON: [{ref, score, fields}]") { json = true }
+      end
+      parser.parse!(argv)
+      path, *words = argv
+      raise Error, "usage: cjk_index search INDEX.json QUERY" if path.nil? || (words.empty? && filter.empty?)
+
+      searcher = Searcher.load(path)
+      filter = filter.transform_values { |values| values.map { |v| filter_value(v) } }
+      filter = filter.transform_values { |values| values.length == 1 ? values.first : values }
+      results = searcher.search(words.join(" "), filter: filter.empty? ? nil : filter)
+      shown = limit.positive? ? results.first(limit) : results
+      if json
+        stdout.puts JSON.generate(shown)
+      else
+        shown.each { |r| stdout.puts format("%s\t%.3f\t%s", r[:ref], r[:score], r[:fields].join(",")) }
+        stdout.puts "(#{results.length} result#{'s' unless results.length == 1})" if shown.length < results.length
+      end
+    rescue Errno::ENOENT
+      raise Error, "no such file: #{path}"
+    end
+
+    # ---- runtime -----------------------------------------------------------
+
+    def runtime_command(argv, stdout)
+      variants = Normalizer::DEFAULT_TABLES
+      output = nil
+      OptionParser.new do |o|
+        o.banner = "Usage: cjk_index runtime [options]"
+        o.on("--variants LIST", Array, "variant tables: ja (default), zh-safe, zh, ko, or table files; must match the index") { |v| variants = v }
+        o.on("-o", "--output PATH", "script file (default: standard output)") { |v| output = v }
+      end.parse!(argv)
+      raise Error, "too many arguments: #{argv.join(' ')}" unless argv.empty?
+
+      write_output(output, stdout, Runtime.source(normalizer: Normalizer.new(tables: variants)))
+    end
+
+    # ---- helpers -----------------------------------------------------------
+
+    def split_pair(text, option)
+      name, value = text.split("=", 2)
+      raise Error, "#{option} takes NAME=VALUE, got #{text.inspect}" if name.to_s.empty? || value.nil?
+
+      [name, value]
+    end
+
+    # "type", "date=year", "subject=value:;"
+    def parse_filter(text)
+      name, spec = text.split("=", 2)
+      kind, separator = (spec || "value").split(":", 2)
+      raise Error, "--filter #{text.inspect}: kind must be value or year" unless Builder::FILTER_KINDS.include?(kind)
+
+      [name, separator ? { "kind" => kind, "separator" => separator } : kind]
+    end
+
+    def filter_value(text)
+      return text unless (m = text.match(/\A(\d*)\.\.(\d*)\z/)) && !(m[1].empty? && m[2].empty?)
+
+      { "min" => (m[1].empty? ? nil : m[1].to_i), "max" => (m[2].empty? ? nil : m[2].to_i) }.compact
+    end
+
+    def write_output(path, stdout, content)
+      if path
+        File.write(path, content)
+      else
+        stdout.write(content)
+        stdout.write("\n") unless content.end_with?("\n")
+      end
+    end
+  end
+end

data/lib/cjk_index/index_file.rb

@@ -27,6 +27,10 @@ module CJKIndex
   #
   # Locks use flock, which is reliable on a local disk; do not share the file
   # over NFS.
+  #
+  # The index is always one file (format 4, as Builder#write). A split index
+  # (Builder#write_split) is not supported here: it is meant for static
+  # sites, written once per build.
   class IndexFile
     # Raised by update and write when another writer holds the lock too long.
     class LockTimeout < StandardError; end

data/lib/cjk_index/jekyll.rb

@@ -9,7 +9,7 @@ module CJKIndex
   #
   #   cjk_index:
   #     output: assets/cjk-index          # where files are written (default)
-  #     variants: [ja]                    # variant tables: ja (default), zh (Traditional -> Simplified)
+  #     variants: [ja]                    # variant tables: ja (default), zh-safe, zh, ko, or a file of your own
   #     indexes:
   #       - name: posts                   # -> assets/cjk-index/posts.json
   #         collection: posts             # a Jekyll collection, or...
@@ -25,8 +25,15 @@ module CJKIndex
   #           date: year
   #       - name: search
   #         preset: collectionbuilder     # reads CollectionBuilder's own config
+  #       - name: texts
+  #         collection: texts
+  #         fields: [title, content]
+  #         split: true                   # -> assets/cjk-index/texts/index.json and shards
+  #         shard_size: 32000             # bytes per shard (optional)
   #
-  # The browser script is written to <output>/cjk-index.js.
+  # The browser script is written to <output>/cjk-index.js. A split index is
+  # loaded with CJKIndex.load('<output>/<name>/index.json') and searched with
+  # searchAsync, which fetches only the shards a query needs (see Builder).
   module Jekyll
     DEFAULT_OUTPUT = "assets/cjk-index"
     RUNTIME_NAME = "cjk-index.js"
@@ -57,13 +64,22 @@ module CJKIndex
         return unless config.is_a?(Hash)
 
         output = config.fetch("output", DEFAULT_OUTPUT).sub(%r{\A/+}, "").sub(%r{/+\z}, "")
-        normalizer = Normalizer.new(tables: Array(config.fetch("variants", Normalizer::DEFAULT_TABLES)))
+        normalizer = Normalizer.new(tables: Array(config.fetch("variants", Normalizer::DEFAULT_TABLES)), base: site.source)
         site.static_files << GeneratedFile.new(site, output, RUNTIME_NAME, Runtime.source(normalizer: normalizer))
         Array(config["indexes"]).each do |spec|
           builder = build(site, spec, normalizer)
-          name = "#{spec.fetch('name')}.json"
-          site.static_files << GeneratedFile.new(site, output, name, builder.to_json)
-          ::Jekyll.logger.info "cjk_index:", "#{output}/#{name} (#{builder.refs.length} documents)"
+          if spec["split"]
+            dir = "#{output}/#{spec.fetch('name')}"
+            files = builder.each_split_file(shard_size: Integer(spec.fetch("shard_size", Builder::SHARD_SIZE)))
+                           .map { |name, content| GeneratedFile.new(site, dir, name, content) }
+            site.static_files.concat(files)
+            ::Jekyll.logger.info "cjk_index:", "#{dir}/#{Builder::MANIFEST} (#{builder.refs.length} documents, " \
+                                               "#{files.length - 2} shards)"
+          else
+            name = "#{spec.fetch('name')}.json"
+            site.static_files << GeneratedFile.new(site, output, name, builder.to_json)
+            ::Jekyll.logger.info "cjk_index:", "#{output}/#{name} (#{builder.refs.length} documents)"
+          end
         end
       end
 
@@ -79,7 +95,7 @@ module CJKIndex
           docs = site.collections.fetch(spec["collection"]) do
             raise ::Jekyll::Errors::FatalException, "cjk_index: no collection #{spec['collection']}"
           end.docs
-          docs.each { |doc| builder.add(doc.url, document_fields(doc, builder.fields)) }
+          docs.each { |doc| builder.add(doc.url, document_fields(doc, builder.fields | builder.filters.keys)) }
         elsif spec["data"]
           ref = spec.fetch("ref", "id")
           Array(site.data[spec["data"]]).each do |row|

data/lib/cjk_index/normalizer.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "digest"
+require "json"
+
 module CJKIndex
   # Folds text so that spelling variants a reader treats as "the same" compare
   # equal: NFKC (full-width -> half-width etc.), lowercase, katakana ->
@@ -9,18 +12,38 @@ module CJKIndex
   # The same rules run in the browser (see Runtime), so an index built here
   # and a query typed there are folded identically.
   #
-  # Variant tables (data/*.tsv):
-  #   ja  Japanese old forms -> modern forms (圖 -> 図). Default.
-  #   zh  Traditional -> Simplified Chinese, from OpenCC. Opt in: it also
-  #       merges characters that are distinct in Japanese (後 and 后).
+  # Variant tables (data/*.tsv, regenerated by script/variant_tables.rb):
+  #   ja       Japanese old forms and glyph variants -> one form (圖 -> 図),
+  #            from the Joyo and Jinmeiyo kanji lists and Unihan. Default.
+  #   zh       Traditional -> Simplified Chinese, from OpenCC. Opt in: it also
+  #            merges characters that are distinct in Japanese (後 and 后).
+  #   zh-safe  The pairs of zh that are one-to-one both ways (圖 -> 图 but
+  #            not 後 -> 后 or 發/髮 -> 发). Opt in.
+  #   ko       Old Hangul in the Hanyang Private Use Area -> standard jamo
+  #            sequences (one character -> several). Opt in.
   # With several tables, characters linked by any table fold to one form,
   # preferring the target of the table listed first: with [ja, zh],
   # 圖, 図 and 图 all become 図.
+  # Any other entry names a table file of your own, relative to base (the
+  # site's folder for Jekyll and Bridgetown, the current folder otherwise).
+  # The index records the entry as written, not the resolved path.
+  #
+  # Table format: UTF-8 lines "variant<TAB>folded", optionally followed by
+  # another tab and a comment column; lines starting with "#" are comments.
+  # "folded" may be several characters.
+  #
+  # Your own table is one group per line, tab-separated: the form to fold to,
+  # then its variants, one character each (the format of the itaiji lists
+  # that collection databases keep). Each cell is folded first (NFKC,
+  # lowercase, katakana -> hiragana), so a compatibility ideograph such as
+  # U+F900 joins the group of the character NFKC makes of it.
   class Normalizer
     DATA_DIR = File.expand_path("../../data", __dir__)
     TABLES = {
       "ja" => "ja-kyujitai.tsv",
-      "zh" => "zh-hant-hans.tsv"
+      "zh" => "zh-hant-hans.tsv",
+      "zh-safe" => "zh-hant-hans-safe.tsv",
+      "ko" => "ko-hanyang-pua.tsv"
     }.freeze
     DEFAULT_TABLES = %w[ja].freeze
 
@@ -34,35 +57,97 @@ module CJKIndex
       @default ||= new
     end
 
-    def self.read_table(name)
-      file = TABLES.fetch(name) { raise ArgumentError, "unknown variant table #{name.inspect} (#{TABLES.keys.join(', ')})" }
-      File.readlines(File.join(DATA_DIR, file), chomp: true, encoding: "UTF-8")
-          .reject { |line| line.empty? || line.start_with?("#") }
-          .map { |line| line.split("\t", 2) }
+    # Pairs [variant, folded] of a built-in table, or of a table file of your
+    # own (a name that is not built in, resolved against base).
+    def self.read_table(name, base: Dir.pwd)
+      return read_groups(name, File.expand_path(name, base)) unless TABLES.key?(name)
+
+      lines(File.join(DATA_DIR, TABLES[name])).map { |_, line| line.split("\t")[0, 2] }
+    end
+
+    def self.lines(path) # :nodoc:
+      File.readlines(path, chomp: true, encoding: "UTF-8").each_with_index
+          .reject { |line, _| line.strip.empty? || line.start_with?("#") }
+          .map { |line, i| [i + 1, line.delete_prefix("\uFEFF")] }
+    end
+
+    def self.read_groups(name, path) # :nodoc:
+      unless File.file?(path)
+        raise ArgumentError, "unknown variant table #{name.inspect}: not one of #{TABLES.keys.join(', ')}, " \
+                             "and no file at #{path}"
+      end
+
+      plain = new(tables: [])
+      lines(path).flat_map do |number, line|
+        target, *variants = line.split("\t").map(&:strip).reject(&:empty?).map { |cell| plain.normalize(cell) }
+        variants = variants.uniq.reject { |v| v == target } # NFKC may already fold a cell (㍿)
+        variants.each do |v|
+          next if v.length == 1
+
+          raise ArgumentError, "#{name}:#{number}: #{v.inspect} is not one character; " \
+                               "only the first cell (the form to fold to) may be longer"
+        end
+        variants.map { |v| [v, target] }
+      end
     end
 
-    attr_reader :tables, :variants
+    attr_reader :tables, :variants, :digest
 
-    def initialize(tables: DEFAULT_TABLES)
+    KATAKANA = "#{KATAKANA_FIRST.chr(Encoding::UTF_8)}-#{KATAKANA_LAST.chr(Encoding::UTF_8)}"
+    HIRAGANA = "#{(KATAKANA_FIRST - KANA_OFFSET).chr(Encoding::UTF_8)}-#{(KATAKANA_LAST - KANA_OFFSET).chr(Encoding::UTF_8)}"
+
+    def initialize(tables: DEFAULT_TABLES, base: Dir.pwd)
       @tables = tables.map(&:to_s).uniq.freeze
+      @base = base
       @variants = build_variants.freeze
+      keys = @variants.keys.select { |k| k.length == 1 }
+      @variant_chars = keys.empty? ? nil : Regexp.new("[#{keys.map { |k| format('\\u{%x}', k.ord) }.join}]")
+      # What the tables fold, not their names: an index built with an older
+      # "ja" table must not be read with a newer one.
+      @digest = Digest::SHA256.hexdigest(JSON.generate(@variants.sort)).slice(0, 16).freeze
+    end
+
+    # Raises unless data (an index) was built with the same tables as this
+    # normalizer. An index without a digest predates 1.0: it still loads, but
+    # it folded with older tables and should be rebuilt.
+    def check!(data)
+      if tables != data["variants"]
+        raise ArgumentError, "cjk_index: index built with variant tables #{data['variants'].inspect} " \
+                             "but this normalizer folds with #{tables.inspect}; rebuild both together"
+      end
+      return if data["variant_digest"].nil? || data["variant_digest"] == digest
+
+      raise ArgumentError, "cjk_index: index built with other contents of the variant tables " \
+                           "#{tables.inspect} (an older cjk_index?); rebuild it"
     end
 
+    # Character by character: katakana to hiragana, then the variant tables,
+    # then 々 to the (folded) character before it. Done as passes over the
+    # whole string, which gives the same result as one loop over the
+    # characters (the loop in runtime.js) and is several times faster in Ruby.
     def normalize(text)
       return "" if text.nil?
 
+      s = text.to_s.unicode_normalize(:nfkc).downcase
+      s = s.tr(KATAKANA, HIRAGANA)
+      # 々 repeats the folded character before it, which may be several
+      # characters (ko): fold one character at a time, as runtime.js does
+      return iterate(s) if s.include?(ITERATION_MARK)
+
+      @variant_chars ? s.gsub(@variant_chars, @variants) : s
+    end
+
+    private
+
+    def iterate(s)
       prev = nil
-      text.to_s.unicode_normalize(:nfkc).downcase.each_char.map do |ch|
-        code = ch.ord
-        ch = (code - KANA_OFFSET).chr(Encoding::UTF_8) if code.between?(KATAKANA_FIRST, KATAKANA_LAST)
+      s.each_char.map do |ch|
         ch = @variants.fetch(ch, ch)
         ch = prev if ch == ITERATION_MARK && prev
         prev = ch
       end.join
     end
 
-    private
-
     # Union-find over all pairs; each class folds to the target preferred by
     # the earliest table (lowest code point among equals, for determinism).
     def build_variants
@@ -74,7 +159,7 @@ module CJKIndex
       end
       rank = {}
       @tables.each_with_index do |name, priority|
-        self.class.read_table(name).each do |from, to|
+        self.class.read_table(name, base: @base).each do |from, to|
           parent[find.call(from)] = find.call(to)
           rank[to] = [rank.fetch(to, [priority, to.ord]), [priority, to.ord]].min
         end