fts_lite 0.0.2 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +1 -15
- data/lib/fts_lite/{database.rb → index.rb} +15 -2
- data/lib/fts_lite/tokenizer.rb +3 -3
- data/lib/fts_lite/version.rb +1 -1
- data/lib/fts_lite.rb +1 -1
- data/test/fts_lite_test.rb +35 -34
- metadata +2 -2
data/README.md
CHANGED
@@ -18,18 +18,4 @@ Or install it yourself as:
|
|
18
18
|
|
19
19
|
## Usage
|
20
20
|
|
21
|
-
|
22
|
-
|
23
|
-
db = FtsLite::Database.new("./db.sqlite3", :tokenizer => :bigram, :cache_size => 64000)
|
24
|
-
|
25
|
-
docid = 1
|
26
|
-
text = "hoge piyo"
|
27
|
-
sort_value = "2012-08-01"
|
28
|
-
|
29
|
-
db.transaction do
|
30
|
-
db.update(docid, text, sort_value)
|
31
|
-
end
|
32
|
-
|
33
|
-
db.search('piyo', :order => :desc, :limit => 10).each do |docid|
|
34
|
-
p docid
|
35
|
-
end
|
21
|
+
TODO
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'sqlite3'
|
2
2
|
|
3
3
|
module FtsLite
|
4
|
-
class
|
4
|
+
class Index
|
5
5
|
DEFAULT_TOKENIZER = :bigram
|
6
6
|
DEFAULT_JURNAL_MODE = "MEMORY"
|
7
7
|
DEFAULT_TEMP_STORE = "MEMORY"
|
@@ -24,6 +24,19 @@ module FtsLite
|
|
24
24
|
set_db_param(options)
|
25
25
|
@tokenizer = Tokenizer.create(options[:tokenizer] || DEFAULT_TOKENIZER)
|
26
26
|
end
|
27
|
+
def self.open(path, options = {})
|
28
|
+
if (block_given?)
|
29
|
+
index = Index.new(path, options)
|
30
|
+
begin
|
31
|
+
yield(index)
|
32
|
+
ensure
|
33
|
+
index.close
|
34
|
+
end
|
35
|
+
else
|
36
|
+
Index.new(path, options)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
27
40
|
def close
|
28
41
|
@db.close
|
29
42
|
end
|
@@ -35,7 +48,7 @@ module FtsLite
|
|
35
48
|
block.call
|
36
49
|
end
|
37
50
|
end
|
38
|
-
def
|
51
|
+
def set(docid, text, sort_value = nil)
|
39
52
|
if (SQLITE_HAVE_FT4_REPLACE)
|
40
53
|
@db.execute("INSERT OR REPLACE INTO #{@table_name} (docid, text, sort_value) VALUES(?, ?, ?);",
|
41
54
|
[docid, @tokenizer.vector(text), sort_value])
|
data/lib/fts_lite/tokenizer.rb
CHANGED
@@ -70,9 +70,9 @@ module FtsLite
|
|
70
70
|
split(text).join(" ")
|
71
71
|
end
|
72
72
|
def split(text)
|
73
|
-
|
74
|
-
|
75
|
-
|
73
|
+
BimyouSegmenter.segment(Tokenizer.normalize(text),
|
74
|
+
:white_space => false,
|
75
|
+
:symbol => false).map {|word|
|
76
76
|
if (word.size == 1)
|
77
77
|
word
|
78
78
|
else
|
data/lib/fts_lite/version.rb
CHANGED
data/lib/fts_lite.rb
CHANGED
data/test/fts_lite_test.rb
CHANGED
@@ -4,8 +4,8 @@ require 'test_helper'
|
|
4
4
|
class FtsLiteTest < Test::Unit::TestCase
|
5
5
|
DB_FILE = File.expand_path(File.join(File.dirname(__FILE__), "test.sqlite3"))
|
6
6
|
puts "RUBY_VERSION => #{RUBY_VERSION}"
|
7
|
-
puts "SQLITE3_VERSION => #{FtsLite::
|
8
|
-
puts "SQLITE_HAVE_FT4_REPLACE => #{FtsLite::
|
7
|
+
puts "SQLITE3_VERSION => #{FtsLite::Index.sqlite3_version}"
|
8
|
+
puts "SQLITE_HAVE_FT4_REPLACE => #{FtsLite::Index.have_ft4_replace}"
|
9
9
|
def setup
|
10
10
|
if (File.exist?(DB_FILE))
|
11
11
|
File.unlink(DB_FILE)
|
@@ -14,35 +14,36 @@ class FtsLiteTest < Test::Unit::TestCase
|
|
14
14
|
def teardown
|
15
15
|
end
|
16
16
|
def test_update
|
17
|
-
|
18
|
-
|
19
|
-
|
17
|
+
FtsLite::Index.open(DB_FILE, :tokenizer => :bigram) do |db|
|
18
|
+
db.transaction do
|
19
|
+
db.delete_all
|
20
20
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
21
|
+
assert_equal db.search("赤い").size, 0
|
22
|
+
db.set(1, "なぜナポリタンは赤いのだろうか ?", 2)
|
23
|
+
db.set(2, "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ", 1)
|
24
|
+
|
25
|
+
assert_equal db.search("赤い").size, 1
|
26
|
+
assert_equal db.search("ナポリタン", :order => :desc).size, 2
|
27
|
+
assert_equal db.search("ナポリタン", :order => :desc)[0], 1
|
28
|
+
assert_equal db.search("ナポリタン", :order => :desc)[1], 2
|
29
|
+
|
30
|
+
db.set(1, "なぜナポリタンは青いのだろうか ?", 0)
|
31
|
+
assert_equal db.search("赤い").size, 0
|
32
|
+
assert_equal db.search("青い").size, 1
|
33
|
+
|
34
|
+
assert_equal db.search("ナポリタン", :order => :desc).size, 2
|
35
|
+
assert_equal db.search("ナポリタン", :order => :desc)[0], 2
|
36
|
+
assert_equal db.search("ナポリタン", :order => :desc)[1], 1
|
37
|
+
end
|
37
38
|
end
|
38
39
|
end
|
39
40
|
def test_bigram
|
40
|
-
db = FtsLite::
|
41
|
+
db = FtsLite::Index.open(DB_FILE, :tokenizer => :bigram)
|
41
42
|
db.transaction do
|
42
43
|
db.delete_all
|
43
44
|
p db.tokenize("なぜナポリタンは赤いのだろうか ?")
|
44
|
-
db.
|
45
|
-
db.
|
45
|
+
db.set(1, "なぜナポリタンは赤いのだろうか ?", 2)
|
46
|
+
db.set(2, "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ", 1)
|
46
47
|
|
47
48
|
assert_equal db.search("赤い").size, 1
|
48
49
|
assert_equal db.search("赤い")[0], 1
|
@@ -72,12 +73,12 @@ class FtsLiteTest < Test::Unit::TestCase
|
|
72
73
|
end
|
73
74
|
end
|
74
75
|
def test_trigram
|
75
|
-
db = FtsLite::
|
76
|
+
db = FtsLite::Index.open(DB_FILE, :tokenizer => :trigram)
|
76
77
|
db.transaction do
|
77
78
|
db.delete_all
|
78
79
|
p db.tokenize("なぜナポリタンは赤いのだろうか ?")
|
79
|
-
db.
|
80
|
-
db.
|
80
|
+
db.set(1, "なぜナポリタンは赤いのだろうか ?", 2)
|
81
|
+
db.set(2, "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ", 1)
|
81
82
|
|
82
83
|
assert_equal db.search("赤い").size, 0
|
83
84
|
|
@@ -106,12 +107,12 @@ class FtsLiteTest < Test::Unit::TestCase
|
|
106
107
|
end
|
107
108
|
end
|
108
109
|
def test_wakachi_bigram
|
109
|
-
db = FtsLite::
|
110
|
+
db = FtsLite::Index.open(DB_FILE, :tokenizer => :wakachi_bigram)
|
110
111
|
db.transaction do
|
111
112
|
db.delete_all
|
112
113
|
p db.tokenize("なぜナポリタンは赤いのだろうか ?")
|
113
|
-
db.
|
114
|
-
db.
|
114
|
+
db.set(1, "なぜナポリタンは赤いのだろうか ?", 2)
|
115
|
+
db.set(2, "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ", 1)
|
115
116
|
|
116
117
|
assert_equal db.search("赤い").size, 1
|
117
118
|
assert_equal db.search("赤い")[0], 1
|
@@ -141,11 +142,11 @@ class FtsLiteTest < Test::Unit::TestCase
|
|
141
142
|
end
|
142
143
|
end
|
143
144
|
def test_create
|
144
|
-
db = FtsLite::
|
145
|
-
db.drop_table!
|
146
|
-
db.close
|
147
|
-
db = FtsLite::Database.new(DB_FILE, :table_name => "hogehgoe")
|
145
|
+
db = FtsLite::Index.open(DB_FILE)
|
148
146
|
db.drop_table!
|
149
147
|
db.close
|
148
|
+
FtsLite::Index.open(DB_FILE, :table_name => "hogehgoe") do |f|
|
149
|
+
f.drop_table!
|
150
|
+
end
|
150
151
|
end
|
151
152
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fts_lite
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -57,7 +57,7 @@ files:
|
|
57
57
|
- Rakefile
|
58
58
|
- fts_lite.gemspec
|
59
59
|
- lib/fts_lite.rb
|
60
|
-
- lib/fts_lite/
|
60
|
+
- lib/fts_lite/index.rb
|
61
61
|
- lib/fts_lite/tokenizer.rb
|
62
62
|
- lib/fts_lite/version.rb
|
63
63
|
- test/fts_lite_test.rb
|