cjk_index 0.2.0 → 1.0.0

data/lib/cjk_index/tokenizer.rb

@@ -68,6 +68,36 @@ module CJKIndex
       phrases(text, unigrams: unigrams, normalizer: normalizer).flatten(1)
     end
 
+    # A run of separators, of CJK characters, or of other characters. A
+    # separator is never part of a run (・ is in the katakana block).
+    CJK_WORD = /[#{CJK.source[1..-2]}&&[^#{SEPARATOR.source[1..-2]}]]/
+    RUN = /(#{SEPARATOR.source}+)|(#{CJK_WORD.source}+)|((?:(?!#{SEPARATOR.source})(?!#{CJK.source}).)+)/m
+
+    # Yields each token of already normalized text with its offset: the
+    # tokens of tokens_with_offsets, without building the phrases. Used to
+    # build the index, where it is several times faster.
+    def each_token(normalized, unigrams: false)
+      offset = 0
+      normalized.scan(RUN) do |_, cjk, other|
+        run = Regexp.last_match(0)
+        if cjk
+          chars = run.chars
+          if chars.length == 1 || unigrams
+            chars.each_with_index { |c, i| yield c, offset + i }
+          end
+          i = 0
+          last = chars.length - 1
+          while i < last
+            yield chars[i] + chars[i + 1], offset + i
+            i += 1
+          end
+        elsif other
+          yield run, offset
+        end
+        offset += run.length
+      end
+    end
+
     # Splits at separators (spaces, punctuation) into phrases, each a list of
     # [token, offset]. A query matches when every phrase occurs somewhere in
     # the document, with the tokens of each phrase at the same relative offsets.

data/lib/cjk_index/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CJKIndex
-  VERSION = "0.2.0"
+  VERSION = "1.0.0"
 end

data/lib/cjk_index.rb

@@ -3,6 +3,7 @@
 require_relative "cjk_index/version"
 require_relative "cjk_index/normalizer"
 require_relative "cjk_index/tokenizer"
+require_relative "cjk_index/postings"
 require_relative "cjk_index/builder"
 require_relative "cjk_index/runtime"
 require_relative "cjk_index/searcher"

metadata

@@ -1,33 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: cjk_index
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Satoru Nakamura
-bindir: bin
+bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies: []
 description: Builds a bigram search index in Ruby (with kana, old/new kanji and NFKC
   normalization), and queries it in Ruby or with a small browser runtime that returns
-  the same results. Includes a Jekyll plugin and a CollectionBuilder preset.
-executables: []
+  the same results. Includes a command-line tool, a Rake task, a Jekyll plugin (with
+  a CollectionBuilder preset) and a Bridgetown plugin.
+executables:
+- cjk_index
 extensions: []
 extra_rdoc_files: []
 files:
 - CHANGELOG.md
 - LICENSE
 - LICENSE-OpenCC
+- LICENSE-Unicode
 - NOTICE
 - README.md
 - data/ja-kyujitai.tsv
+- data/ko-hanyang-pua.tsv
+- data/zh-hant-hans-safe.tsv
 - data/zh-hant-hans.tsv
+- exe/cjk_index
 - lib/cjk_index.rb
+- lib/cjk_index/bridgetown.rb
 - lib/cjk_index/builder.rb
+- lib/cjk_index/cli.rb
 - lib/cjk_index/index_file.rb
 - lib/cjk_index/jekyll.rb
 - lib/cjk_index/normalizer.rb
+- lib/cjk_index/postings.rb
+- lib/cjk_index/rake_task.rb
 - lib/cjk_index/runtime.js
 - lib/cjk_index/runtime.rb
 - lib/cjk_index/searcher.rb
@@ -37,6 +47,8 @@ homepage: https://github.com/nakamura196/cjk_index
 licenses:
 - MIT
 - Apache-2.0
+- Unicode-3.0
+- Unlicense
 metadata:
   source_code_uri: https://github.com/nakamura196/cjk_index
   documentation_uri: https://github.com/nakamura196/cjk_index#readme