fts_lite 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.sqlite3

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fts_lite.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 nagadomi@nurs.or.jp
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,55 @@
+# FtsLite
+
+simple full text search index.
+
+## Dependency
+
+    Ruby >= 1.9.2
+    SQLite3 >= 3.7.7 (FTS4 REPLACE support)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fts_lite'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fts_lite
+
+## Usage
+
+    require 'fts_lite'
+    
+    db = FtsLite::Database.new("./db.sqlite3", :tokenizer => :bigram, :cache_size => 64000)
+    
+    docid = 1
+    text = "hoge piyo"
+    sort_value = "2012-08-01"
+    
+    db.transaction do
+      db.insert_or_replace(docid, text, sort_value)
+      db.batch_insert_or_replace([
+        {:docid => 30, :text => "hoge hoge", :sort_value => '2012-08-01'},
+        {:docid => 40, :text => "piyo piyo", :sort_value => '2012-08-02'}
+      ])
+    end
+    
+    db.search('piyo', :order => :desc, :limit => 10).each do |docid|
+      p docid
+    end
+    
+    
+    db.batch_update_sort_value([
+      {:docid => 30, :sort_value => '2012-07-01'},
+      {:docid => 40, :sort_value => '2012-07-02'}
+    ])
+    
+    db.search('piyo', :order => :desc, :limit => 10).each do |docid|
+      p docid
+    end
+

data/Rakefile

@@ -0,0 +1,10 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'test'
+  test.test_files = Dir.glob("test/**/*_test.rb")
+  test.verbose = true
+  test.warning = true
+end

data/fts_lite.gemspec

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+require File.expand_path('../lib/fts_lite/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["nagadomi"]
+  gem.email         = ["nagadomi@nurs.or.jp"]
+  gem.description   = %q{simple full text search engine}
+  gem.summary       = %q{simple full text search engine}
+  gem.homepage      = "https://github.com/nagadomi/fts_lite"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "fts_lite"
+  gem.require_paths = ["lib"]
+  gem.version       = FtsLite::VERSION
+  
+  gem.add_dependency 'bimyou_segmenter', '>= 1.2.0'
+  gem.add_dependency 'sqlite3-ruby'
+end

data/lib/fts_lite/database.rb

@@ -0,0 +1,124 @@
+require 'sqlite3'
+
+module FtsLite
+  class Database
+    DEFAULT_TOKENIZER = :bigram
+    DEFAULT_JURNAL_MODE = "MEMORY"
+    DEFAULT_TEMP_STORE = "MEMORY"
+    DEFAULT_CACHE_SIZE = 32000
+    
+    class RuntimeError < ::RuntimeError
+    end
+    
+    def initialize(path, options = {})
+      @db = SQLite3::Database.new(path)
+      @table_name = options[:table_name] || "fts_lite"
+      create_table!(options)
+      set_db_param(options)
+      @tokenizer = Tokenizer.create(options[:tokenizer] || DEFAULT_TOKENIZER)
+    end
+    def tokenize(text)
+      @tokenizer.vector(text).split(" ")
+    end
+    def close
+      @db.close
+    end
+    def transaction(&block)
+      @db.transaction do
+        block.call
+      end
+    end
+    def insert_or_replace(docid, text, sort_value = nil)
+      @db.execute("INSERT OR REPLACE INTO #{@table_name} (docid, text, sort_value) VALUES(?, ?, ?);",
+                  [docid, @tokenizer.vector(text), sort_value])
+    end
+    def update_sort_value(docid, sort_value)
+      @db.execute("UPDATE #{@table_name} SET sort_value = ? WHERE docid = ?;",
+                  [sort_value, docid])
+    end
+    def delete(docid)
+      @db.execute("DELETE FROM #{@table_name} WHERE docid = ?;", [docid])
+    end
+    def search(text, options = {})
+      limit = options[:limit]
+      order = nil
+      if (options[:order])
+        case options[:order].to_sym
+        when :desc
+          order = :desc
+        when :asc
+          order = :asc
+        end
+      end
+      sql = "SELECT docid FROM #{@table_name} WHERE text MATCH ?"
+      if (order)
+        sql += sprintf(" ORDER BY sort_value %s", order == :desc ? "DESC" : "ASC")
+      else
+        sql += sprintf(" ORDER BY docid ASC")
+      end
+      if (limit)
+        sql += sprintf(" LIMIT %d", limit)
+      end
+      sql += ";"
+      @db.execute(sql, [@tokenizer.vector(text)]).flatten
+    end
+    def count
+      @db.execute("SELECT COUNT(*) FROM #{@table_name} ;").first.first
+    end
+    def delete_all
+      @db.execute("DELETE FROM #{@table_name} ;")
+    end
+    def batch_insert(records)
+      @db.prepare("INSERT INTO #{@table_name} (docid, text, sort_value) VALUES(?, ?, ?);") do |stmt|
+        records.each do |rec|
+          stmt.execute([rec[:docid], @tokenizer.vector(rec[:text]), rec[:sort_value]])
+        end
+      end
+    end
+    def batch_insert_or_replace(records)
+      @db.prepare("INSERT OR REPLACE INTO #{@table_name} (docid, text, sort_value) VALUES(?, ?, ?);") do |stmt|
+        records.each do |rec|
+          stmt.execute([rec[:docid], @tokenizer.vector(rec[:text]), rec[:sort_value]])
+        end
+      end
+    end
+    def batch_update_sort_value(records)
+      @db.prepare("UPDATE #{@table_name} SET sort_value = ? WHERE docid = ?;") do |stmt|
+        records.each do |rec|
+          stmt.execute([rec[:sort_value], rec[:docid]])
+        end
+      end
+    end
+    def drop_table!
+      if (table_exist?)
+        @db.execute("DROP TABLE #{@table_name};")
+      end
+    end
+    
+    private
+    def create_table!(options)
+      ret = false
+      @db.transaction do 
+        tokenizer = options[:tokenizer] || DEFAULT_TOKENIZER
+        exist = table_exist?
+        if (!exist)
+          drop_table!
+          @db.execute("CREATE VIRTUAL TABLE #{@table_name} USING FTS4(text, sort_value, tokenize=simple);")
+          ret = true
+        end
+      end
+      ret
+    end
+    def table_exist?
+      @db.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?;",
+                  [@table_name]).size == 1
+    end
+    def set_db_param(options)
+      @db.transaction do 
+        @db.execute("PRAGMA journal_mode=#{options[:journal_mode] || DEFAULT_JURNAL_MODE};")
+        @db.execute("PRAGMA temp_store=#{options[:temp_store] || DEFAULT_TEMP_STORE};")
+        @db.execute("PRAGMA cache_size=#{options[:cache_size] || DEFAULT_CACHE_SIZE};")
+      end
+    end
+  end
+end

data/lib/fts_lite/tokenizer.rb

@@ -0,0 +1,85 @@
+# coding: utf-8
+require 'nkf'
+require 'bimyou_segmenter'
+
+module FtsLite
+  module Tokenizer
+    SIMPLE_DELIMITER = /[\s\.,\?!;\(\)。、．，？！「」『』（）]+/
+    
+    def self.create(name)
+      case name.to_sym
+      when :simple
+        Simple.new
+      when :bigram
+        Bigram.new
+      when :trigram
+        Trigram.new
+      when :wakachi
+        Wakachi.new
+      when :wakachi_bigram
+        WakachiBigram.new
+      else
+        raise ArgumentError
+      end
+    end
+    def self.normalize(text)
+      NKF::nkf('-wZX', text).downcase
+    end
+    class Simple
+      def vector(text)
+        split(text).join(" ")
+      end
+      def split(text)
+        Tokenizer.normalize(text).gsub(/[\.,\?!;:]/, ' ').split(SIMPLE_DELIMITER)
+      end
+    end
+    class Bigram
+      def vector(text)
+        split(text).join(" ")
+      end
+      def split(text)
+        text = Tokenizer.normalize(text)
+        text.split(SIMPLE_DELIMITER).map {|word|
+          0.upto(word.size - 2).map {|i| word[i, 2] }
+        }.flatten
+      end
+    end
+    class Trigram
+      def vector(text)
+        split(text).join(" ")
+      end
+      def split(text)
+        text = Tokenizer.normalize(text)
+        text.split(SIMPLE_DELIMITER).map {|word|
+          0.upto(word.size - 3).map {|i| word[i, 3] }
+        }.flatten
+      end
+    end
+    class Wakachi
+      def vector(text)
+        split(text).join(" ")
+      end
+      def split(text)
+        BimyouSegmenter.segment(Tokenizer.normalize(text),
+                                :white_space => false,
+                                :symbol => false)
+      end
+    end
+    class WakachiBigram
+      def vector(text)
+        split(text).join(" ")
+      end
+      def split(text)
+        words = BimyouSegmenter.segment(Tokenizer.normalize(text),
+                                        :white_space => false,
+                                        :symbol => false).map {|word|
+          if (word.size == 1)
+            word
+          else
+            0.upto(word.size - 2).map {|i| word[i, 2] }
+          end
+        }.flatten
+      end
+    end
+  end
+end

data/lib/fts_lite/version.rb

@@ -0,0 +1,3 @@
+module FtsLite
+  VERSION = "0.0.1"
+end

data/lib/fts_lite.rb

@@ -0,0 +1,6 @@
+require "fts_lite/version"
+require "fts_lite/tokenizer"
+require "fts_lite/database"
+
+module FtsLite
+end

data/test/fts_lite_test.rb

@@ -0,0 +1,131 @@
+# coding: utf-8
+require 'test_helper'
+
+class FtsLiteTest < Test::Unit::TestCase
+  DB_FILE = File.expand_path(File.join(File.dirname(__FILE__), "test.sqlite3"))
+  def setup
+    if (File.exist?(DB_FILE))
+      File.unlink(DB_FILE)
+    end
+  end
+  def teardown
+  end
+  def test_bigram
+    db = FtsLite::Database.new(DB_FILE, :tokenizer => :bigram)
+    db.transaction do 
+      db.delete_all
+      p db.tokenize("なぜナポリタンは赤いのだろうか ？")
+      db.insert_or_replace(1, "なぜナポリタンは赤いのだろうか ？", 2)
+      db.insert_or_replace(2, "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ", 1)
+      
+      assert_equal db.search("赤い").size, 1
+      assert_equal db.search("赤い")[0], 1
+      
+      assert_equal db.search("ナポリタン").size, 2
+      assert_equal db.search("ナポリタン")[0], 1
+      assert_equal db.search("ナポリタン")[1], 2
+      
+      assert_equal db.search("ナポリタン", :order => :desc).size, 2
+      assert_equal db.search("ナポリタン", :order => :desc)[0], 1
+      assert_equal db.search("ナポリタン", :order => :desc)[1], 2
+      
+      assert_equal db.search("ナポリタン", :order => :asc).size, 2
+      assert_equal db.search("ナポリタン", :order => :asc)[0], 2
+      assert_equal db.search("ナポリタン", :order => :asc)[1], 1
+      
+      db.update_sort_value(1, 1)
+      db.update_sort_value(2, 2)
+      
+      assert_equal db.search("ナポリタン", :order => :desc).size, 2
+      assert_equal db.search("ナポリタン", :order => :desc)[0], 2
+      assert_equal db.search("ナポリタン", :order => :desc)[1], 1
+      
+      assert_equal db.search("ナポリタン", :order => :asc).size, 2
+      assert_equal db.search("ナポリタン", :order => :asc)[0], 1
+      assert_equal db.search("ナポリタン", :order => :asc)[1], 2
+    end
+  end
+  def test_trigram
+    db = FtsLite::Database.new(DB_FILE, :tokenizer => :trigram)
+    db.transaction do 
+      db.delete_all
+      p db.tokenize("なぜナポリタンは赤いのだろうか ？")
+      db.insert_or_replace(1, "なぜナポリタンは赤いのだろうか ？", 2)
+      db.insert_or_replace(2, "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ", 1)
+      
+      assert_equal db.search("赤い").size, 0
+      
+      assert_equal db.search("ナポリタン").size, 2
+      assert_equal db.search("ナポリタン")[0], 1
+      assert_equal db.search("ナポリタン")[1], 2
+      
+      assert_equal db.search("ナポリタン", :order => :desc).size, 2
+      assert_equal db.search("ナポリタン", :order => :desc)[0], 1
+      assert_equal db.search("ナポリタン", :order => :desc)[1], 2
+      
+      assert_equal db.search("ナポリタン", :order => :asc).size, 2
+      assert_equal db.search("ナポリタン", :order => :asc)[0], 2
+      assert_equal db.search("ナポリタン", :order => :asc)[1], 1
+      
+      db.update_sort_value(1, 1)
+      db.update_sort_value(2, 2)
+      
+      assert_equal db.search("ナポリタン", :order => :desc).size, 2
+      assert_equal db.search("ナポリタン", :order => :desc)[0], 2
+      assert_equal db.search("ナポリタン", :order => :desc)[1], 1
+      
+      assert_equal db.search("ナポリタン", :order => :asc).size, 2
+      assert_equal db.search("ナポリタン", :order => :asc)[0], 1
+      assert_equal db.search("ナポリタン", :order => :asc)[1], 2
+    end
+  end
+  def test_wakachi_bigram
+    db = FtsLite::Database.new(DB_FILE, :tokenizer => :wakachi_bigram)
+    db.transaction do 
+      db.delete_all
+      p db.tokenize("なぜナポリタンは赤いのだろうか ？")
+      db.batch_insert([{ :docid => 1,
+                         :text => "なぜナポリタンは赤いのだろうか ？",
+                         :sort_value => 2
+                       },
+                       { :docid => 2,
+                         :text => "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ",
+                         :sort_value => 1
+                       }
+                      ])
+      assert_equal db.search("赤い").size, 1
+      assert_equal db.search("赤い")[0], 1
+      
+      assert_equal db.search("ナポリタン").size, 2
+      assert_equal db.search("ナポリタン")[0], 1
+      assert_equal db.search("ナポリタン")[1], 2
+      
+      assert_equal db.search("ナポリタン", :order => :desc).size, 2
+      assert_equal db.search("ナポリタン", :order => :desc)[0], 1
+      assert_equal db.search("ナポリタン", :order => :desc)[1], 2
+      
+      assert_equal db.search("ナポリタン", :order => :asc).size, 2
+      assert_equal db.search("ナポリタン", :order => :asc)[0], 2
+      assert_equal db.search("ナポリタン", :order => :asc)[1], 1
+      
+      db.update_sort_value(1, 1)
+      db.update_sort_value(2, 2)
+      
+      assert_equal db.search("ナポリタン", :order => :desc).size, 2
+      assert_equal db.search("ナポリタン", :order => :desc)[0], 2
+      assert_equal db.search("ナポリタン", :order => :desc)[1], 1
+      
+      assert_equal db.search("ナポリタン", :order => :asc).size, 2
+      assert_equal db.search("ナポリタン", :order => :asc)[0], 1
+      assert_equal db.search("ナポリタン", :order => :asc)[1], 2
+    end
+  end
+  def test_create
+    db = FtsLite::Database.new(DB_FILE)
+    db.drop_table!
+    db.close
+    db = FtsLite::Database.new(DB_FILE, :table_name => "hogehgoe")
+    db.drop_table!
+    db.close
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,13 @@
+if (RUBY_VERSION < "1.9.0")
+  $KCODE = 'u'
+  begin
+    require 'rubygems'
+  rescue LoadError
+  end
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.expand_path(File.join('..', 'lib')))
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))
+
+require 'fts_lite'

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: fts_lite
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- nagadomi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bimyou_segmenter
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: sqlite3-ruby
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: simple full text search engine
+email:
+- nagadomi@nurs.or.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fts_lite.gemspec
+- lib/fts_lite.rb
+- lib/fts_lite/database.rb
+- lib/fts_lite/tokenizer.rb
+- lib/fts_lite/version.rb
+- test/fts_lite_test.rb
+- test/test_helper.rb
+homepage: https://github.com/nagadomi/fts_lite
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: simple full text search engine
+test_files:
+- test/fts_lite_test.rb
+- test/test_helper.rb