fts_lite 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +2 -22
- data/lib/fts_lite/database.rb +23 -27
- data/lib/fts_lite/version.rb +1 -1
- data/test/fts_lite_test.rb +33 -13
- data/test/test_helper.rb +0 -7
- metadata +2 -2
data/README.md
CHANGED
@@ -1,11 +1,6 @@
|
|
1
1
|
# FtsLite
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
## Dependency
|
6
|
-
|
7
|
-
Ruby >= 1.9.2
|
8
|
-
SQLite3 >= 3.7.7 (FTS4 REPLACE support)
|
3
|
+
full text search index.
|
9
4
|
|
10
5
|
## Installation
|
11
6
|
|
@@ -32,24 +27,9 @@ Or install it yourself as:
|
|
32
27
|
sort_value = "2012-08-01"
|
33
28
|
|
34
29
|
db.transaction do
|
35
|
-
db.
|
36
|
-
db.batch_insert_or_replace([
|
37
|
-
{:docid => 30, :text => "hoge hoge", :sort_value => '2012-08-01'},
|
38
|
-
{:docid => 40, :text => "piyo piyo", :sort_value => '2012-08-02'}
|
39
|
-
])
|
30
|
+
db.update(docid, text, sort_value)
|
40
31
|
end
|
41
32
|
|
42
33
|
db.search('piyo', :order => :desc, :limit => 10).each do |docid|
|
43
34
|
p docid
|
44
35
|
end
|
45
|
-
|
46
|
-
|
47
|
-
db.batch_update_sort_value([
|
48
|
-
{:docid => 30, :sort_value => '2012-07-01'},
|
49
|
-
{:docid => 40, :sort_value => '2012-07-02'}
|
50
|
-
])
|
51
|
-
|
52
|
-
db.search('piyo', :order => :desc, :limit => 10).each do |docid|
|
53
|
-
p docid
|
54
|
-
end
|
55
|
-
|
data/lib/fts_lite/database.rb
CHANGED
@@ -6,7 +6,14 @@ module FtsLite
|
|
6
6
|
DEFAULT_JURNAL_MODE = "MEMORY"
|
7
7
|
DEFAULT_TEMP_STORE = "MEMORY"
|
8
8
|
DEFAULT_CACHE_SIZE = 32000
|
9
|
+
SQLITE_HAVE_FT4_REPLACE = SQLite3.libversion >= 3007007
|
9
10
|
|
11
|
+
def self.have_ft4_replace
|
12
|
+
SQLITE_HAVE_FT4_REPLACE
|
13
|
+
end
|
14
|
+
def self.sqlite3_version
|
15
|
+
SQLite3.libversion
|
16
|
+
end
|
10
17
|
class RuntimeError < ::RuntimeError
|
11
18
|
end
|
12
19
|
|
@@ -17,20 +24,30 @@ module FtsLite
|
|
17
24
|
set_db_param(options)
|
18
25
|
@tokenizer = Tokenizer.create(options[:tokenizer] || DEFAULT_TOKENIZER)
|
19
26
|
end
|
20
|
-
def tokenize(text)
|
21
|
-
@tokenizer.vector(text).split(" ")
|
22
|
-
end
|
23
27
|
def close
|
24
28
|
@db.close
|
25
29
|
end
|
30
|
+
def tokenize(text)
|
31
|
+
@tokenizer.vector(text).split(" ")
|
32
|
+
end
|
26
33
|
def transaction(&block)
|
27
34
|
@db.transaction do
|
28
35
|
block.call
|
29
36
|
end
|
30
37
|
end
|
31
|
-
def
|
32
|
-
|
33
|
-
|
38
|
+
def update(docid, text, sort_value = nil)
|
39
|
+
if (SQLITE_HAVE_FT4_REPLACE)
|
40
|
+
@db.execute("INSERT OR REPLACE INTO #{@table_name} (docid, text, sort_value) VALUES(?, ?, ?);",
|
41
|
+
[docid, @tokenizer.vector(text), sort_value])
|
42
|
+
else
|
43
|
+
begin
|
44
|
+
@db.execute("INSERT INTO #{@table_name} (docid, text, sort_value) VALUES(?, ?, ?);",
|
45
|
+
[docid, @tokenizer.vector(text), sort_value])
|
46
|
+
rescue SQLite3::ConstraintException
|
47
|
+
@db.execute("UPDATE #{@table_name} SET text = ?, sort_value = ? WHERE docid = ?;",
|
48
|
+
[@tokenizer.vector(text), sort_value, docid])
|
49
|
+
end
|
50
|
+
end
|
34
51
|
end
|
35
52
|
def update_sort_value(docid, sort_value)
|
36
53
|
@db.execute("UPDATE #{@table_name} SET sort_value = ? WHERE docid = ?;",
|
@@ -68,27 +85,6 @@ module FtsLite
|
|
68
85
|
def delete_all
|
69
86
|
@db.execute("DELETE FROM #{@table_name} ;")
|
70
87
|
end
|
71
|
-
def batch_insert(records)
|
72
|
-
@db.prepare("INSERT INTO #{@table_name} (docid, text, sort_value) VALUES(?, ?, ?);") do |stmt|
|
73
|
-
records.each do |rec|
|
74
|
-
stmt.execute([rec[:docid], @tokenizer.vector(rec[:text]), rec[:sort_value]])
|
75
|
-
end
|
76
|
-
end
|
77
|
-
end
|
78
|
-
def batch_insert_or_replace(records)
|
79
|
-
@db.prepare("INSERT OR REPLACE INTO #{@table_name} (docid, text, sort_value) VALUES(?, ?, ?);") do |stmt|
|
80
|
-
records.each do |rec|
|
81
|
-
stmt.execute([rec[:docid], @tokenizer.vector(rec[:text]), rec[:sort_value]])
|
82
|
-
end
|
83
|
-
end
|
84
|
-
end
|
85
|
-
def batch_update_sort_value(records)
|
86
|
-
@db.prepare("UPDATE #{@table_name} SET sort_value = ? WHERE docid = ?;") do |stmt|
|
87
|
-
records.each do |rec|
|
88
|
-
stmt.execute([rec[:sort_value], rec[:docid]])
|
89
|
-
end
|
90
|
-
end
|
91
|
-
end
|
92
88
|
def drop_table!
|
93
89
|
if (table_exist?)
|
94
90
|
@db.execute("DROP TABLE #{@table_name};")
|
data/lib/fts_lite/version.rb
CHANGED
data/test/fts_lite_test.rb
CHANGED
@@ -3,6 +3,9 @@ require 'test_helper'
|
|
3
3
|
|
4
4
|
class FtsLiteTest < Test::Unit::TestCase
|
5
5
|
DB_FILE = File.expand_path(File.join(File.dirname(__FILE__), "test.sqlite3"))
|
6
|
+
puts "RUBY_VERSION => #{RUBY_VERSION}"
|
7
|
+
puts "SQLITE3_VERSION => #{FtsLite::Database.sqlite3_version}"
|
8
|
+
puts "SQLITE_HAVE_FT4_REPLACE => #{FtsLite::Database.have_ft4_replace}"
|
6
9
|
def setup
|
7
10
|
if (File.exist?(DB_FILE))
|
8
11
|
File.unlink(DB_FILE)
|
@@ -10,13 +13,36 @@ class FtsLiteTest < Test::Unit::TestCase
|
|
10
13
|
end
|
11
14
|
def teardown
|
12
15
|
end
|
16
|
+
def test_update
|
17
|
+
db = FtsLite::Database.new(DB_FILE, :tokenizer => :bigram)
|
18
|
+
db.transaction do
|
19
|
+
db.delete_all
|
20
|
+
|
21
|
+
assert_equal db.search("赤い").size, 0
|
22
|
+
db.update(1, "なぜナポリタンは赤いのだろうか ?", 2)
|
23
|
+
db.update(2, "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ", 1)
|
24
|
+
|
25
|
+
assert_equal db.search("赤い").size, 1
|
26
|
+
assert_equal db.search("ナポリタン", :order => :desc).size, 2
|
27
|
+
assert_equal db.search("ナポリタン", :order => :desc)[0], 1
|
28
|
+
assert_equal db.search("ナポリタン", :order => :desc)[1], 2
|
29
|
+
|
30
|
+
db.update(1, "なぜナポリタンは青いのだろうか ?", 0)
|
31
|
+
assert_equal db.search("赤い").size, 0
|
32
|
+
assert_equal db.search("青い").size, 1
|
33
|
+
|
34
|
+
assert_equal db.search("ナポリタン", :order => :desc).size, 2
|
35
|
+
assert_equal db.search("ナポリタン", :order => :desc)[0], 2
|
36
|
+
assert_equal db.search("ナポリタン", :order => :desc)[1], 1
|
37
|
+
end
|
38
|
+
end
|
13
39
|
def test_bigram
|
14
40
|
db = FtsLite::Database.new(DB_FILE, :tokenizer => :bigram)
|
15
41
|
db.transaction do
|
16
42
|
db.delete_all
|
17
43
|
p db.tokenize("なぜナポリタンは赤いのだろうか ?")
|
18
|
-
db.
|
19
|
-
db.
|
44
|
+
db.update(1, "なぜナポリタンは赤いのだろうか ?", 2)
|
45
|
+
db.update(2, "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ", 1)
|
20
46
|
|
21
47
|
assert_equal db.search("赤い").size, 1
|
22
48
|
assert_equal db.search("赤い")[0], 1
|
@@ -50,8 +76,8 @@ class FtsLiteTest < Test::Unit::TestCase
|
|
50
76
|
db.transaction do
|
51
77
|
db.delete_all
|
52
78
|
p db.tokenize("なぜナポリタンは赤いのだろうか ?")
|
53
|
-
db.
|
54
|
-
db.
|
79
|
+
db.update(1, "なぜナポリタンは赤いのだろうか ?", 2)
|
80
|
+
db.update(2, "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ", 1)
|
55
81
|
|
56
82
|
assert_equal db.search("赤い").size, 0
|
57
83
|
|
@@ -84,15 +110,9 @@ class FtsLiteTest < Test::Unit::TestCase
|
|
84
110
|
db.transaction do
|
85
111
|
db.delete_all
|
86
112
|
p db.tokenize("なぜナポリタンは赤いのだろうか ?")
|
87
|
-
db.
|
88
|
-
|
89
|
-
|
90
|
-
},
|
91
|
-
{ :docid => 2,
|
92
|
-
:text => "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ",
|
93
|
-
:sort_value => 1
|
94
|
-
}
|
95
|
-
])
|
113
|
+
db.update(1, "なぜナポリタンは赤いのだろうか ?", 2)
|
114
|
+
db.update(2, "昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。 ", 1)
|
115
|
+
|
96
116
|
assert_equal db.search("赤い").size, 1
|
97
117
|
assert_equal db.search("赤い")[0], 1
|
98
118
|
|
data/test/test_helper.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fts_lite
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-08-
|
12
|
+
date: 2012-08-05 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bimyou_segmenter
|