fts_lite 0.0.2 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +1 -15
- data/lib/fts_lite/{database.rb → index.rb} +15 -2
- data/lib/fts_lite/tokenizer.rb +3 -3
- data/lib/fts_lite/version.rb +1 -1
- data/lib/fts_lite.rb +1 -1
- data/test/fts_lite_test.rb +35 -34
- metadata +2 -2
    
        data/README.md
    CHANGED
    
    | @@ -18,18 +18,4 @@ Or install it yourself as: | |
| 18 18 |  | 
| 19 19 | 
             
            ## Usage
         | 
| 20 20 |  | 
| 21 | 
            -
             | 
| 22 | 
            -
                
         | 
| 23 | 
            -
                db = FtsLite::Database.new("./db.sqlite3", :tokenizer => :bigram, :cache_size => 64000)
         | 
| 24 | 
            -
                
         | 
| 25 | 
            -
                docid = 1
         | 
| 26 | 
            -
                text = "hoge piyo"
         | 
| 27 | 
            -
                sort_value = "2012-08-01"
         | 
| 28 | 
            -
                
         | 
| 29 | 
            -
                db.transaction do
         | 
| 30 | 
            -
                  db.update(docid, text, sort_value)
         | 
| 31 | 
            -
                end
         | 
| 32 | 
            -
                
         | 
| 33 | 
            -
                db.search('piyo', :order => :desc, :limit => 10).each do |docid|
         | 
| 34 | 
            -
                  p docid
         | 
| 35 | 
            -
                end
         | 
| 21 | 
            +
            TODO
         | 
| @@ -1,7 +1,7 @@ | |
| 1 1 | 
             
            require 'sqlite3'
         | 
| 2 2 |  | 
| 3 3 | 
             
            module FtsLite
         | 
| 4 | 
            -
              class  | 
| 4 | 
            +
              class Index
         | 
| 5 5 | 
             
                DEFAULT_TOKENIZER = :bigram
         | 
| 6 6 | 
             
                DEFAULT_JURNAL_MODE = "MEMORY"
         | 
| 7 7 | 
             
                DEFAULT_TEMP_STORE = "MEMORY"
         | 
| @@ -24,6 +24,19 @@ module FtsLite | |
| 24 24 | 
             
                  set_db_param(options)
         | 
| 25 25 | 
             
                  @tokenizer = Tokenizer.create(options[:tokenizer] || DEFAULT_TOKENIZER)
         | 
| 26 26 | 
             
                end
         | 
| 27 | 
            +
                def self.open(path, options = {})
         | 
| 28 | 
            +
                  if (block_given?)
         | 
| 29 | 
            +
                    index = Index.new(path, options)
         | 
| 30 | 
            +
                    begin
         | 
| 31 | 
            +
                      yield(index)
         | 
| 32 | 
            +
                    ensure
         | 
| 33 | 
            +
                      index.close
         | 
| 34 | 
            +
                    end
         | 
| 35 | 
            +
                  else
         | 
| 36 | 
            +
                    Index.new(path, options)
         | 
| 37 | 
            +
                  end
         | 
| 38 | 
            +
                end
         | 
| 39 | 
            +
                  
         | 
| 27 40 | 
             
                def close
         | 
| 28 41 | 
             
                  @db.close
         | 
| 29 42 | 
             
                end
         | 
| @@ -35,7 +48,7 @@ module FtsLite | |
| 35 48 | 
             
                    block.call
         | 
| 36 49 | 
             
                  end
         | 
| 37 50 | 
             
                end
         | 
| 38 | 
            -
                def  | 
| 51 | 
            +
                def set(docid, text, sort_value = nil)
         | 
| 39 52 | 
             
                  if (SQLITE_HAVE_FT4_REPLACE)
         | 
| 40 53 | 
             
                    @db.execute("INSERT OR REPLACE INTO #{@table_name} (docid, text, sort_value) VALUES(?, ?, ?);",
         | 
| 41 54 | 
             
                                [docid, @tokenizer.vector(text), sort_value])
         | 
    
        data/lib/fts_lite/tokenizer.rb
    CHANGED
    
    | @@ -70,9 +70,9 @@ module FtsLite | |
| 70 70 | 
             
                    split(text).join(" ")
         | 
| 71 71 | 
             
                  end
         | 
| 72 72 | 
             
                  def split(text)
         | 
| 73 | 
            -
                     | 
| 74 | 
            -
             | 
| 75 | 
            -
             | 
| 73 | 
            +
                    BimyouSegmenter.segment(Tokenizer.normalize(text),
         | 
| 74 | 
            +
                                            :white_space => false,
         | 
| 75 | 
            +
                                            :symbol => false).map {|word|
         | 
| 76 76 | 
             
                      if (word.size == 1)
         | 
| 77 77 | 
             
                        word
         | 
| 78 78 | 
             
                      else
         | 
    
        data/lib/fts_lite/version.rb
    CHANGED
    
    
    
        data/lib/fts_lite.rb
    CHANGED
    
    
    
        data/test/fts_lite_test.rb
    CHANGED
    
    | @@ -4,8 +4,8 @@ require 'test_helper' | |
| 4 4 | 
             
            class FtsLiteTest < Test::Unit::TestCase
         | 
| 5 5 | 
             
              DB_FILE = File.expand_path(File.join(File.dirname(__FILE__), "test.sqlite3"))
         | 
| 6 6 | 
             
              puts "RUBY_VERSION => #{RUBY_VERSION}"
         | 
| 7 | 
            -
              puts "SQLITE3_VERSION => #{FtsLite:: | 
| 8 | 
            -
              puts "SQLITE_HAVE_FT4_REPLACE => #{FtsLite:: | 
| 7 | 
            +
              puts "SQLITE3_VERSION => #{FtsLite::Index.sqlite3_version}"
         | 
| 8 | 
            +
              puts "SQLITE_HAVE_FT4_REPLACE => #{FtsLite::Index.have_ft4_replace}"
         | 
| 9 9 | 
             
              def setup
         | 
| 10 10 | 
             
                if (File.exist?(DB_FILE))
         | 
| 11 11 | 
             
                  File.unlink(DB_FILE)
         | 
| @@ -14,35 +14,36 @@ class FtsLiteTest < Test::Unit::TestCase | |
| 14 14 | 
             
              def teardown
         | 
| 15 15 | 
             
              end
         | 
| 16 16 | 
             
              def test_update
         | 
| 17 | 
            -
                 | 
| 18 | 
            -
             | 
| 19 | 
            -
             | 
| 17 | 
            +
                FtsLite::Index.open(DB_FILE, :tokenizer => :bigram) do |db|
         | 
| 18 | 
            +
                  db.transaction do 
         | 
| 19 | 
            +
                    db.delete_all
         | 
| 20 20 |  | 
| 21 | 
            -
             | 
| 22 | 
            -
             | 
| 23 | 
            -
             | 
| 24 | 
            -
             | 
| 25 | 
            -
             | 
| 26 | 
            -
             | 
| 27 | 
            -
             | 
| 28 | 
            -
             | 
| 29 | 
            -
             | 
| 30 | 
            -
             | 
| 31 | 
            -
             | 
| 32 | 
            -
             | 
| 33 | 
            -
             | 
| 34 | 
            -
             | 
| 35 | 
            -
             | 
| 36 | 
            -
             | 
| 21 | 
            +
                    assert_equal db.search("赤い").size, 0
         | 
| 22 | 
            +
                    db.set(1, "なぜナポリタンは赤いのだろうか ?", 2)
         | 
| 23 | 
            +
                    db.set(2, "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ", 1)
         | 
| 24 | 
            +
                    
         | 
| 25 | 
            +
                    assert_equal db.search("赤い").size, 1
         | 
| 26 | 
            +
                    assert_equal db.search("ナポリタン", :order => :desc).size, 2
         | 
| 27 | 
            +
                    assert_equal db.search("ナポリタン", :order => :desc)[0], 1
         | 
| 28 | 
            +
                    assert_equal db.search("ナポリタン", :order => :desc)[1], 2
         | 
| 29 | 
            +
                    
         | 
| 30 | 
            +
                    db.set(1, "なぜナポリタンは青いのだろうか ?", 0)
         | 
| 31 | 
            +
                    assert_equal db.search("赤い").size, 0
         | 
| 32 | 
            +
                    assert_equal db.search("青い").size, 1
         | 
| 33 | 
            +
                    
         | 
| 34 | 
            +
                    assert_equal db.search("ナポリタン", :order => :desc).size, 2
         | 
| 35 | 
            +
                    assert_equal db.search("ナポリタン", :order => :desc)[0], 2
         | 
| 36 | 
            +
                    assert_equal db.search("ナポリタン", :order => :desc)[1], 1
         | 
| 37 | 
            +
                  end
         | 
| 37 38 | 
             
                end
         | 
| 38 39 | 
             
              end
         | 
| 39 40 | 
             
              def test_bigram
         | 
| 40 | 
            -
                db = FtsLite:: | 
| 41 | 
            +
                db = FtsLite::Index.open(DB_FILE, :tokenizer => :bigram)
         | 
| 41 42 | 
             
                db.transaction do 
         | 
| 42 43 | 
             
                  db.delete_all
         | 
| 43 44 | 
             
                  p db.tokenize("なぜナポリタンは赤いのだろうか ?")
         | 
| 44 | 
            -
                  db. | 
| 45 | 
            -
                  db. | 
| 45 | 
            +
                  db.set(1, "なぜナポリタンは赤いのだろうか ?", 2)
         | 
| 46 | 
            +
                  db.set(2, "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ", 1)
         | 
| 46 47 |  | 
| 47 48 | 
             
                  assert_equal db.search("赤い").size, 1
         | 
| 48 49 | 
             
                  assert_equal db.search("赤い")[0], 1
         | 
| @@ -72,12 +73,12 @@ class FtsLiteTest < Test::Unit::TestCase | |
| 72 73 | 
             
                end
         | 
| 73 74 | 
             
              end
         | 
| 74 75 | 
             
              def test_trigram
         | 
| 75 | 
            -
                db = FtsLite:: | 
| 76 | 
            +
                db = FtsLite::Index.open(DB_FILE, :tokenizer => :trigram)
         | 
| 76 77 | 
             
                db.transaction do 
         | 
| 77 78 | 
             
                  db.delete_all
         | 
| 78 79 | 
             
                  p db.tokenize("なぜナポリタンは赤いのだろうか ?")
         | 
| 79 | 
            -
                  db. | 
| 80 | 
            -
                  db. | 
| 80 | 
            +
                  db.set(1, "なぜナポリタンは赤いのだろうか ?", 2)
         | 
| 81 | 
            +
                  db.set(2, "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ", 1)
         | 
| 81 82 |  | 
| 82 83 | 
             
                  assert_equal db.search("赤い").size, 0
         | 
| 83 84 |  | 
| @@ -106,12 +107,12 @@ class FtsLiteTest < Test::Unit::TestCase | |
| 106 107 | 
             
                end
         | 
| 107 108 | 
             
              end
         | 
| 108 109 | 
             
              def test_wakachi_bigram
         | 
| 109 | 
            -
                db = FtsLite:: | 
| 110 | 
            +
                db = FtsLite::Index.open(DB_FILE, :tokenizer => :wakachi_bigram)
         | 
| 110 111 | 
             
                db.transaction do 
         | 
| 111 112 | 
             
                  db.delete_all
         | 
| 112 113 | 
             
                  p db.tokenize("なぜナポリタンは赤いのだろうか ?")
         | 
| 113 | 
            -
                  db. | 
| 114 | 
            -
                  db. | 
| 114 | 
            +
                  db.set(1, "なぜナポリタンは赤いのだろうか ?", 2)
         | 
| 115 | 
            +
                  db.set(2, "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ", 1)
         | 
| 115 116 |  | 
| 116 117 | 
             
                  assert_equal db.search("赤い").size, 1
         | 
| 117 118 | 
             
                  assert_equal db.search("赤い")[0], 1
         | 
| @@ -141,11 +142,11 @@ class FtsLiteTest < Test::Unit::TestCase | |
| 141 142 | 
             
                end
         | 
| 142 143 | 
             
              end
         | 
| 143 144 | 
             
              def test_create
         | 
| 144 | 
            -
                db = FtsLite:: | 
| 145 | 
            -
                db.drop_table!
         | 
| 146 | 
            -
                db.close
         | 
| 147 | 
            -
                db = FtsLite::Database.new(DB_FILE, :table_name => "hogehgoe")
         | 
| 145 | 
            +
                db = FtsLite::Index.open(DB_FILE)
         | 
| 148 146 | 
             
                db.drop_table!
         | 
| 149 147 | 
             
                db.close
         | 
| 148 | 
            +
                FtsLite::Index.open(DB_FILE, :table_name => "hogehgoe") do |f|
         | 
| 149 | 
            +
                  f.drop_table!
         | 
| 150 | 
            +
                end
         | 
| 150 151 | 
             
              end
         | 
| 151 152 | 
             
            end
         | 
    
        metadata
    CHANGED
    
    | @@ -1,7 +1,7 @@ | |
| 1 1 | 
             
            --- !ruby/object:Gem::Specification
         | 
| 2 2 | 
             
            name: fts_lite
         | 
| 3 3 | 
             
            version: !ruby/object:Gem::Version
         | 
| 4 | 
            -
              version: 0.0 | 
| 4 | 
            +
              version: 0.1.0
         | 
| 5 5 | 
             
              prerelease: 
         | 
| 6 6 | 
             
            platform: ruby
         | 
| 7 7 | 
             
            authors:
         | 
| @@ -57,7 +57,7 @@ files: | |
| 57 57 | 
             
            - Rakefile
         | 
| 58 58 | 
             
            - fts_lite.gemspec
         | 
| 59 59 | 
             
            - lib/fts_lite.rb
         | 
| 60 | 
            -
            - lib/fts_lite/ | 
| 60 | 
            +
            - lib/fts_lite/index.rb
         | 
| 61 61 | 
             
            - lib/fts_lite/tokenizer.rb
         | 
| 62 62 | 
             
            - lib/fts_lite/version.rb
         | 
| 63 63 | 
             
            - test/fts_lite_test.rb
         |