fts_lite 0.0.2 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +1 -15
- data/lib/fts_lite/{database.rb → index.rb} +15 -2
- data/lib/fts_lite/tokenizer.rb +3 -3
- data/lib/fts_lite/version.rb +1 -1
- data/lib/fts_lite.rb +1 -1
- data/test/fts_lite_test.rb +35 -34
- metadata +2 -2
data/README.md
CHANGED
@@ -18,18 +18,4 @@ Or install it yourself as:
|
|
18
18
|
|
19
19
|
## Usage
|
20
20
|
|
21
|
-
|
22
|
-
|
23
|
-
db = FtsLite::Database.new("./db.sqlite3", :tokenizer => :bigram, :cache_size => 64000)
|
24
|
-
|
25
|
-
docid = 1
|
26
|
-
text = "hoge piyo"
|
27
|
-
sort_value = "2012-08-01"
|
28
|
-
|
29
|
-
db.transaction do
|
30
|
-
db.update(docid, text, sort_value)
|
31
|
-
end
|
32
|
-
|
33
|
-
db.search('piyo', :order => :desc, :limit => 10).each do |docid|
|
34
|
-
p docid
|
35
|
-
end
|
21
|
+
TODO
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'sqlite3'
|
2
2
|
|
3
3
|
module FtsLite
|
4
|
-
class
|
4
|
+
class Index
|
5
5
|
DEFAULT_TOKENIZER = :bigram
|
6
6
|
DEFAULT_JURNAL_MODE = "MEMORY"
|
7
7
|
DEFAULT_TEMP_STORE = "MEMORY"
|
@@ -24,6 +24,19 @@ module FtsLite
|
|
24
24
|
set_db_param(options)
|
25
25
|
@tokenizer = Tokenizer.create(options[:tokenizer] || DEFAULT_TOKENIZER)
|
26
26
|
end
|
27
|
+
def self.open(path, options = {})
|
28
|
+
if (block_given?)
|
29
|
+
index = Index.new(path, options)
|
30
|
+
begin
|
31
|
+
yield(index)
|
32
|
+
ensure
|
33
|
+
index.close
|
34
|
+
end
|
35
|
+
else
|
36
|
+
Index.new(path, options)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
27
40
|
def close
|
28
41
|
@db.close
|
29
42
|
end
|
@@ -35,7 +48,7 @@ module FtsLite
|
|
35
48
|
block.call
|
36
49
|
end
|
37
50
|
end
|
38
|
-
def
|
51
|
+
def set(docid, text, sort_value = nil)
|
39
52
|
if (SQLITE_HAVE_FT4_REPLACE)
|
40
53
|
@db.execute("INSERT OR REPLACE INTO #{@table_name} (docid, text, sort_value) VALUES(?, ?, ?);",
|
41
54
|
[docid, @tokenizer.vector(text), sort_value])
|
data/lib/fts_lite/tokenizer.rb
CHANGED
@@ -70,9 +70,9 @@ module FtsLite
|
|
70
70
|
split(text).join(" ")
|
71
71
|
end
|
72
72
|
def split(text)
|
73
|
-
|
74
|
-
|
75
|
-
|
73
|
+
BimyouSegmenter.segment(Tokenizer.normalize(text),
|
74
|
+
:white_space => false,
|
75
|
+
:symbol => false).map {|word|
|
76
76
|
if (word.size == 1)
|
77
77
|
word
|
78
78
|
else
|
data/lib/fts_lite/version.rb
CHANGED
data/lib/fts_lite.rb
CHANGED
data/test/fts_lite_test.rb
CHANGED
@@ -4,8 +4,8 @@ require 'test_helper'
|
|
4
4
|
class FtsLiteTest < Test::Unit::TestCase
|
5
5
|
DB_FILE = File.expand_path(File.join(File.dirname(__FILE__), "test.sqlite3"))
|
6
6
|
puts "RUBY_VERSION => #{RUBY_VERSION}"
|
7
|
-
puts "SQLITE3_VERSION => #{FtsLite::
|
8
|
-
puts "SQLITE_HAVE_FT4_REPLACE => #{FtsLite::
|
7
|
+
puts "SQLITE3_VERSION => #{FtsLite::Index.sqlite3_version}"
|
8
|
+
puts "SQLITE_HAVE_FT4_REPLACE => #{FtsLite::Index.have_ft4_replace}"
|
9
9
|
def setup
|
10
10
|
if (File.exist?(DB_FILE))
|
11
11
|
File.unlink(DB_FILE)
|
@@ -14,35 +14,36 @@ class FtsLiteTest < Test::Unit::TestCase
|
|
14
14
|
def teardown
|
15
15
|
end
|
16
16
|
def test_update
|
17
|
-
|
18
|
-
|
19
|
-
|
17
|
+
FtsLite::Index.open(DB_FILE, :tokenizer => :bigram) do |db|
|
18
|
+
db.transaction do
|
19
|
+
db.delete_all
|
20
20
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
21
|
+
assert_equal db.search("赤い").size, 0
|
22
|
+
db.set(1, "なぜナポリタンは赤いのだろうか ?", 2)
|
23
|
+
db.set(2, "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ", 1)
|
24
|
+
|
25
|
+
assert_equal db.search("赤い").size, 1
|
26
|
+
assert_equal db.search("ナポリタン", :order => :desc).size, 2
|
27
|
+
assert_equal db.search("ナポリタン", :order => :desc)[0], 1
|
28
|
+
assert_equal db.search("ナポリタン", :order => :desc)[1], 2
|
29
|
+
|
30
|
+
db.set(1, "なぜナポリタンは青いのだろうか ?", 0)
|
31
|
+
assert_equal db.search("赤い").size, 0
|
32
|
+
assert_equal db.search("青い").size, 1
|
33
|
+
|
34
|
+
assert_equal db.search("ナポリタン", :order => :desc).size, 2
|
35
|
+
assert_equal db.search("ナポリタン", :order => :desc)[0], 2
|
36
|
+
assert_equal db.search("ナポリタン", :order => :desc)[1], 1
|
37
|
+
end
|
37
38
|
end
|
38
39
|
end
|
39
40
|
def test_bigram
|
40
|
-
db = FtsLite::
|
41
|
+
db = FtsLite::Index.open(DB_FILE, :tokenizer => :bigram)
|
41
42
|
db.transaction do
|
42
43
|
db.delete_all
|
43
44
|
p db.tokenize("なぜナポリタンは赤いのだろうか ?")
|
44
|
-
db.
|
45
|
-
db.
|
45
|
+
db.set(1, "なぜナポリタンは赤いのだろうか ?", 2)
|
46
|
+
db.set(2, "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ", 1)
|
46
47
|
|
47
48
|
assert_equal db.search("赤い").size, 1
|
48
49
|
assert_equal db.search("赤い")[0], 1
|
@@ -72,12 +73,12 @@ class FtsLiteTest < Test::Unit::TestCase
|
|
72
73
|
end
|
73
74
|
end
|
74
75
|
def test_trigram
|
75
|
-
db = FtsLite::
|
76
|
+
db = FtsLite::Index.open(DB_FILE, :tokenizer => :trigram)
|
76
77
|
db.transaction do
|
77
78
|
db.delete_all
|
78
79
|
p db.tokenize("なぜナポリタンは赤いのだろうか ?")
|
79
|
-
db.
|
80
|
-
db.
|
80
|
+
db.set(1, "なぜナポリタンは赤いのだろうか ?", 2)
|
81
|
+
db.set(2, "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ", 1)
|
81
82
|
|
82
83
|
assert_equal db.search("赤い").size, 0
|
83
84
|
|
@@ -106,12 +107,12 @@ class FtsLiteTest < Test::Unit::TestCase
|
|
106
107
|
end
|
107
108
|
end
|
108
109
|
def test_wakachi_bigram
|
109
|
-
db = FtsLite::
|
110
|
+
db = FtsLite::Index.open(DB_FILE, :tokenizer => :wakachi_bigram)
|
110
111
|
db.transaction do
|
111
112
|
db.delete_all
|
112
113
|
p db.tokenize("なぜナポリタンは赤いのだろうか ?")
|
113
|
-
db.
|
114
|
-
db.
|
114
|
+
db.set(1, "なぜナポリタンは赤いのだろうか ?", 2)
|
115
|
+
db.set(2, "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ", 1)
|
115
116
|
|
116
117
|
assert_equal db.search("赤い").size, 1
|
117
118
|
assert_equal db.search("赤い")[0], 1
|
@@ -141,11 +142,11 @@ class FtsLiteTest < Test::Unit::TestCase
|
|
141
142
|
end
|
142
143
|
end
|
143
144
|
def test_create
|
144
|
-
db = FtsLite::
|
145
|
-
db.drop_table!
|
146
|
-
db.close
|
147
|
-
db = FtsLite::Database.new(DB_FILE, :table_name => "hogehgoe")
|
145
|
+
db = FtsLite::Index.open(DB_FILE)
|
148
146
|
db.drop_table!
|
149
147
|
db.close
|
148
|
+
FtsLite::Index.open(DB_FILE, :table_name => "hogehgoe") do |f|
|
149
|
+
f.drop_table!
|
150
|
+
end
|
150
151
|
end
|
151
152
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fts_lite
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -57,7 +57,7 @@ files:
|
|
57
57
|
- Rakefile
|
58
58
|
- fts_lite.gemspec
|
59
59
|
- lib/fts_lite.rb
|
60
|
-
- lib/fts_lite/
|
60
|
+
- lib/fts_lite/index.rb
|
61
61
|
- lib/fts_lite/tokenizer.rb
|
62
62
|
- lib/fts_lite/version.rb
|
63
63
|
- test/fts_lite_test.rb
|