odba 1.1.8 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/devenv.yml +30 -0
- data/.github/workflows/ruby.yml +8 -7
- data/.gitignore +11 -0
- data/{History.txt → History.md} +43 -19
- data/README.md +79 -2
- data/Rakefile +3 -15
- data/devenv.lock +171 -0
- data/devenv.nix +53 -0
- data/devenv.yaml +8 -0
- data/lib/odba/cache.rb +401 -344
- data/lib/odba/cache_entry.rb +42 -31
- data/lib/odba/connection_pool.rb +99 -72
- data/lib/odba/drbwrapper.rb +26 -18
- data/lib/odba/id_server.rb +20 -20
- data/lib/odba/index.rb +249 -215
- data/lib/odba/index_definition.rb +17 -17
- data/lib/odba/marshal.rb +15 -14
- data/lib/odba/odba.rb +40 -32
- data/lib/odba/odba_error.rb +4 -2
- data/lib/odba/persistable.rb +460 -414
- data/lib/odba/storage.rb +328 -277
- data/lib/odba/stub.rb +166 -153
- data/lib/odba/version.rb +1 -1
- data/lib/odba.rb +9 -9
- data/odba.gemspec +8 -3
- data/test/example.rb +47 -0
- data/test/helper.rb +6 -0
- data/test/suite.rb +7 -0
- data/test/test_array.rb +38 -33
- data/test/test_cache.rb +657 -622
- data/test/test_cache_entry.rb +51 -71
- data/test/test_connection_pool.rb +70 -25
- data/test/test_drbwrapper.rb +24 -19
- data/test/test_hash.rb +98 -89
- data/test/test_id_server.rb +30 -32
- data/test/test_index.rb +191 -158
- data/test/test_marshal.rb +15 -25
- data/test/test_persistable.rb +378 -369
- data/test/test_storage.rb +510 -471
- data/test/test_stub.rb +147 -135
- metadata +63 -20
- data/Guide.txt +0 -36
- data/Manifest.txt +0 -38
- data/install.rb +0 -1098
- data/lib/odba/18_19_loading_compatibility.rb +0 -77
data/lib/odba/storage.rb
CHANGED
|
@@ -1,118 +1,124 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
|
-
|
|
2
|
+
|
|
3
3
|
# ODBA::Storage -- odba -- 08.12.2011 -- mhatakeyama@ywesee.com
|
|
4
4
|
# ODBA::Storage -- odba -- 29.04.2004 -- hwyss@ywesee.com rwaltert@ywesee.com mwalder@ywesee.com
|
|
5
5
|
|
|
6
|
-
require
|
|
7
|
-
require
|
|
8
|
-
require 'dbi'
|
|
6
|
+
require "singleton"
|
|
7
|
+
require "dbi"
|
|
9
8
|
|
|
10
9
|
module ODBA
|
|
11
|
-
|
|
10
|
+
class Storage # :nodoc: all
|
|
12
11
|
include Singleton
|
|
13
|
-
|
|
14
|
-
|
|
12
|
+
attr_writer :dbi
|
|
13
|
+
BULK_FETCH_STEP = 2500
|
|
15
14
|
TABLES = [
|
|
16
15
|
# in table 'object', the isolated dumps of all objects are stored
|
|
17
|
-
[
|
|
18
|
-
CREATE TABLE IF NOT EXISTS object (
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
);
|
|
16
|
+
["object", <<~SQL],
|
|
17
|
+
CREATE TABLE IF NOT EXISTS object (
|
|
18
|
+
odba_id INTEGER NOT NULL, content TEXT,
|
|
19
|
+
name TEXT, prefetchable BOOLEAN, extent TEXT,
|
|
20
|
+
PRIMARY KEY(odba_id), UNIQUE(name)
|
|
21
|
+
);
|
|
23
22
|
SQL
|
|
24
|
-
[
|
|
25
|
-
CREATE INDEX IF NOT EXISTS prefetchable_index ON object(prefetchable);
|
|
23
|
+
["prefetchable_index", <<~SQL],
|
|
24
|
+
CREATE INDEX IF NOT EXISTS prefetchable_index ON object(prefetchable);
|
|
26
25
|
SQL
|
|
27
|
-
[
|
|
28
|
-
CREATE INDEX IF NOT EXISTS extent_index ON object(extent);
|
|
26
|
+
["extent_index", <<~SQL],
|
|
27
|
+
CREATE INDEX IF NOT EXISTS extent_index ON object(extent);
|
|
29
28
|
SQL
|
|
30
29
|
# helper table 'object_connection'
|
|
31
|
-
[
|
|
32
|
-
CREATE TABLE IF NOT EXISTS object_connection (
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
);
|
|
30
|
+
["object_connection", <<~SQL],
|
|
31
|
+
CREATE TABLE IF NOT EXISTS object_connection (
|
|
32
|
+
origin_id integer, target_id integer,
|
|
33
|
+
PRIMARY KEY(origin_id, target_id)
|
|
34
|
+
);
|
|
36
35
|
SQL
|
|
37
|
-
[
|
|
38
|
-
CREATE INDEX IF NOT EXISTS target_id_index ON object_connection(target_id);
|
|
36
|
+
["target_id_index", <<~SQL],
|
|
37
|
+
CREATE INDEX IF NOT EXISTS target_id_index ON object_connection(target_id);
|
|
39
38
|
SQL
|
|
40
39
|
# helper table 'collection'
|
|
41
|
-
[
|
|
42
|
-
CREATE TABLE IF NOT EXISTS collection (
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
);
|
|
40
|
+
["collection", <<~SQL]
|
|
41
|
+
CREATE TABLE IF NOT EXISTS collection (
|
|
42
|
+
odba_id integer NOT NULL, key text, value text,
|
|
43
|
+
PRIMARY KEY(odba_id, key)
|
|
44
|
+
);
|
|
46
45
|
SQL
|
|
47
46
|
]
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
47
|
+
def initialize
|
|
48
|
+
@id_mutex = Mutex.new
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def bulk_restore(bulk_fetch_ids)
|
|
52
|
+
if bulk_fetch_ids.empty?
|
|
53
|
+
[]
|
|
54
|
+
else
|
|
55
|
+
bulk_fetch_ids = bulk_fetch_ids.uniq
|
|
56
|
+
rows = []
|
|
57
|
+
until (ids = bulk_fetch_ids.slice!(0, BULK_FETCH_STEP)).empty?
|
|
58
|
+
sql = <<-SQL
|
|
59
59
|
SELECT odba_id, content FROM object
|
|
60
|
-
WHERE odba_id IN (#{ids.join(
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
60
|
+
WHERE odba_id IN (#{ids.join(",")})
|
|
61
|
+
SQL
|
|
62
|
+
rows.concat(dbi.select_all(sql))
|
|
63
|
+
end
|
|
64
|
+
rows
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
67
68
|
def collection_fetch(odba_id, key_dump)
|
|
68
69
|
sql = <<-SQL
|
|
69
70
|
SELECT value FROM collection
|
|
70
71
|
WHERE odba_id = ? AND key = ?
|
|
71
72
|
SQL
|
|
72
|
-
row =
|
|
73
|
+
row = dbi.select_one(sql, odba_id, key_dump)
|
|
73
74
|
row.first unless row.nil?
|
|
74
75
|
end
|
|
76
|
+
|
|
75
77
|
def collection_remove(odba_id, key_dump)
|
|
76
|
-
|
|
78
|
+
dbi.do <<-SQL, odba_id, key_dump
|
|
77
79
|
DELETE FROM collection
|
|
78
80
|
WHERE odba_id = ? AND key = ?
|
|
79
81
|
SQL
|
|
80
82
|
end
|
|
83
|
+
|
|
81
84
|
def collection_store(odba_id, key_dump, value_dump)
|
|
82
|
-
|
|
85
|
+
dbi.do <<-SQL, odba_id, key_dump, value_dump
|
|
83
86
|
INSERT INTO collection (odba_id, key, value)
|
|
84
87
|
VALUES (?, ?, ?)
|
|
85
88
|
SQL
|
|
86
89
|
end
|
|
87
|
-
|
|
88
|
-
|
|
90
|
+
|
|
91
|
+
def condition_index_delete(index_name, origin_id,
|
|
92
|
+
search_terms, target_id = nil)
|
|
89
93
|
values = []
|
|
90
94
|
sql = "DELETE FROM #{index_name}"
|
|
91
|
-
if
|
|
92
|
-
|
|
95
|
+
sql << if origin_id
|
|
96
|
+
" WHERE origin_id = ?"
|
|
93
97
|
else
|
|
94
|
-
|
|
98
|
+
" WHERE origin_id IS ?"
|
|
95
99
|
end
|
|
96
100
|
search_terms.each { |key, value|
|
|
97
101
|
sql << " AND %s = ?" % key
|
|
98
102
|
values << value
|
|
99
103
|
}
|
|
100
|
-
if
|
|
104
|
+
if target_id
|
|
101
105
|
sql << " AND target_id = ?"
|
|
102
106
|
values << target_id
|
|
103
107
|
end
|
|
104
|
-
|
|
108
|
+
dbi.do sql, origin_id, *values
|
|
105
109
|
end
|
|
110
|
+
|
|
106
111
|
def condition_index_ids(index_name, id, id_name)
|
|
107
112
|
sql = <<-SQL
|
|
108
113
|
SELECT DISTINCT *
|
|
109
114
|
FROM #{index_name}
|
|
110
115
|
WHERE #{id_name}=?
|
|
111
116
|
SQL
|
|
112
|
-
|
|
117
|
+
dbi.select_all(sql, id)
|
|
113
118
|
end
|
|
119
|
+
|
|
114
120
|
def create_dictionary_map(language)
|
|
115
|
-
|
|
121
|
+
dbi.do <<-SQL
|
|
116
122
|
ALTER TEXT SEARCH CONFIGURATION default_#{language}
|
|
117
123
|
ALTER MAPPING FOR
|
|
118
124
|
asciiword, asciihword, hword_asciipart,
|
|
@@ -120,122 +126,130 @@ CREATE TABLE IF NOT EXISTS collection (
|
|
|
120
126
|
numword, numhword
|
|
121
127
|
WITH #{language}_ispell, #{language}_stem;
|
|
122
128
|
SQL
|
|
123
|
-
|
|
129
|
+
dbi.do <<-SQL
|
|
124
130
|
ALTER TEXT SEARCH CONFIGURATION default_#{language}
|
|
125
131
|
ALTER MAPPING FOR
|
|
126
132
|
host, file, int, uint, version
|
|
127
133
|
WITH simple;
|
|
128
134
|
SQL
|
|
129
135
|
# drop from default setting
|
|
130
|
-
|
|
136
|
+
dbi.do <<-SQL
|
|
131
137
|
ALTER TEXT SEARCH CONFIGURATION default_#{language}
|
|
132
138
|
DROP MAPPING FOR
|
|
133
139
|
email, url, url_path, sfloat, float
|
|
134
140
|
SQL
|
|
135
141
|
end
|
|
142
|
+
|
|
136
143
|
def create_condition_index(table_name, definition)
|
|
137
|
-
|
|
138
|
-
CREATE TABLE IF NOT EXISTS #{table_name} (
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
);
|
|
143
|
-
SQL
|
|
144
|
-
#index origin_id
|
|
145
|
-
|
|
146
|
-
CREATE INDEX IF NOT EXISTS origin_id_#{table_name} ON #{table_name}(origin_id);
|
|
147
|
-
SQL
|
|
148
|
-
#index search_term
|
|
144
|
+
dbi.do <<~SQL
|
|
145
|
+
CREATE TABLE IF NOT EXISTS #{table_name} (
|
|
146
|
+
origin_id INTEGER,
|
|
147
|
+
#{definition.collect { |*pair| pair.join(" ") }.join(",\n ")},
|
|
148
|
+
target_id INTEGER
|
|
149
|
+
);
|
|
150
|
+
SQL
|
|
151
|
+
# index origin_id
|
|
152
|
+
dbi.do <<~SQL
|
|
153
|
+
CREATE INDEX IF NOT EXISTS origin_id_#{table_name} ON #{table_name}(origin_id);
|
|
154
|
+
SQL
|
|
155
|
+
# index search_term
|
|
149
156
|
definition.each { |name, datatype|
|
|
150
|
-
|
|
151
|
-
CREATE INDEX IF NOT EXISTS #{name}_#{table_name} ON #{table_name}(#{name});
|
|
157
|
+
dbi.do <<~SQL
|
|
158
|
+
CREATE INDEX IF NOT EXISTS #{name}_#{table_name} ON #{table_name}(#{name});
|
|
152
159
|
SQL
|
|
153
160
|
}
|
|
154
|
-
#index target_id
|
|
155
|
-
|
|
156
|
-
CREATE INDEX IF NOT EXISTS target_id_#{table_name} ON #{table_name}(target_id);
|
|
161
|
+
# index target_id
|
|
162
|
+
dbi.do <<~SQL
|
|
163
|
+
CREATE INDEX IF NOT EXISTS target_id_#{table_name} ON #{table_name}(target_id);
|
|
157
164
|
SQL
|
|
158
165
|
end
|
|
166
|
+
|
|
159
167
|
def create_fulltext_index(table_name)
|
|
160
|
-
|
|
161
|
-
DROP TABLE IF EXISTS #{table_name};
|
|
168
|
+
dbi.do <<~SQL
|
|
169
|
+
DROP TABLE IF EXISTS #{table_name};
|
|
162
170
|
SQL
|
|
163
|
-
|
|
164
|
-
CREATE TABLE IF NOT EXISTS #{table_name} (
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
)
|
|
171
|
+
dbi.do <<~SQL
|
|
172
|
+
CREATE TABLE IF NOT EXISTS #{table_name} (
|
|
173
|
+
origin_id INTEGER,
|
|
174
|
+
search_term tsvector,
|
|
175
|
+
target_id INTEGER
|
|
176
|
+
) ;
|
|
169
177
|
SQL
|
|
170
|
-
#index origin_id
|
|
171
|
-
|
|
172
|
-
CREATE INDEX IF NOT EXISTS origin_id_#{table_name} ON #{table_name}(origin_id);
|
|
178
|
+
# index origin_id
|
|
179
|
+
dbi.do <<~SQL
|
|
180
|
+
CREATE INDEX IF NOT EXISTS origin_id_#{table_name} ON #{table_name}(origin_id);
|
|
173
181
|
SQL
|
|
174
|
-
|
|
175
|
-
CREATE INDEX IF NOT EXISTS search_term_#{table_name}
|
|
176
|
-
ON #{table_name} USING gist(search_term);
|
|
182
|
+
dbi.do <<~SQL
|
|
183
|
+
CREATE INDEX IF NOT EXISTS search_term_#{table_name}
|
|
184
|
+
ON #{table_name} USING gist(search_term);
|
|
177
185
|
SQL
|
|
178
|
-
#index target_id
|
|
179
|
-
|
|
180
|
-
CREATE INDEX IF NOT EXISTS target_id_#{table_name} ON #{table_name}(target_id);
|
|
186
|
+
# index target_id
|
|
187
|
+
dbi.do <<~SQL
|
|
188
|
+
CREATE INDEX IF NOT EXISTS target_id_#{table_name} ON #{table_name}(target_id);
|
|
181
189
|
SQL
|
|
182
190
|
end
|
|
191
|
+
|
|
183
192
|
def create_index(table_name)
|
|
184
|
-
|
|
193
|
+
dbi.do <<-SQL
|
|
185
194
|
DROP TABLE IF EXISTS #{table_name};
|
|
186
195
|
SQL
|
|
187
|
-
|
|
196
|
+
dbi.do <<-SQL
|
|
188
197
|
CREATE TABLE IF NOT EXISTS #{table_name} (
|
|
189
198
|
origin_id INTEGER,
|
|
190
199
|
search_term TEXT,
|
|
191
200
|
target_id INTEGER
|
|
192
|
-
)
|
|
201
|
+
) ;
|
|
193
202
|
SQL
|
|
194
|
-
#index origin_id
|
|
195
|
-
|
|
203
|
+
# index origin_id
|
|
204
|
+
dbi.do <<-SQL
|
|
196
205
|
CREATE INDEX IF NOT EXISTS origin_id_#{table_name}
|
|
197
206
|
ON #{table_name}(origin_id)
|
|
198
207
|
SQL
|
|
199
|
-
#index search_term
|
|
200
|
-
|
|
208
|
+
# index search_term
|
|
209
|
+
dbi.do <<-SQL
|
|
201
210
|
CREATE INDEX IF NOT EXISTS search_term_#{table_name}
|
|
202
211
|
ON #{table_name}(search_term)
|
|
203
212
|
SQL
|
|
204
|
-
#index target_id
|
|
205
|
-
|
|
213
|
+
# index target_id
|
|
214
|
+
dbi.do <<-SQL
|
|
206
215
|
CREATE INDEX IF NOT EXISTS target_id_#{table_name}
|
|
207
216
|
ON #{table_name}(target_id)
|
|
208
217
|
SQL
|
|
209
218
|
end
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
219
|
+
|
|
220
|
+
def dbi
|
|
221
|
+
Thread.current[:txn] || @dbi
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def drop_index(index_name)
|
|
225
|
+
dbi.do "DROP TABLE IF EXISTS #{index_name}"
|
|
226
|
+
end
|
|
227
|
+
|
|
216
228
|
def delete_index_element(index_name, odba_id, id_name)
|
|
217
|
-
|
|
229
|
+
dbi.do <<-SQL, odba_id
|
|
218
230
|
DELETE FROM #{index_name} WHERE #{id_name} = ?
|
|
219
231
|
SQL
|
|
220
232
|
end
|
|
221
|
-
|
|
233
|
+
|
|
234
|
+
def delete_persistable(odba_id)
|
|
222
235
|
# delete origin from connections
|
|
223
|
-
|
|
236
|
+
dbi.do <<-SQL, odba_id
|
|
224
237
|
DELETE FROM object_connection WHERE origin_id = ?
|
|
225
|
-
|
|
238
|
+
SQL
|
|
226
239
|
# delete target from connections
|
|
227
|
-
|
|
240
|
+
dbi.do <<-SQL, odba_id
|
|
228
241
|
DELETE FROM object_connection WHERE target_id = ?
|
|
229
|
-
|
|
242
|
+
SQL
|
|
230
243
|
# delete from collections
|
|
231
|
-
|
|
244
|
+
dbi.do <<-SQL, odba_id
|
|
232
245
|
DELETE FROM collection WHERE odba_id = ?
|
|
233
|
-
|
|
246
|
+
SQL
|
|
234
247
|
# delete from objects
|
|
235
|
-
|
|
248
|
+
dbi.do <<-SQL, odba_id
|
|
236
249
|
DELETE FROM object WHERE odba_id = ?
|
|
237
|
-
|
|
238
|
-
|
|
250
|
+
SQL
|
|
251
|
+
end
|
|
252
|
+
|
|
239
253
|
def ensure_object_connections(origin_id, target_ids)
|
|
240
254
|
sql = <<-SQL
|
|
241
255
|
SELECT target_id FROM object_connection
|
|
@@ -243,24 +257,23 @@ CREATE INDEX IF NOT EXISTS target_id_#{table_name} ON #{table_name}(target_id);
|
|
|
243
257
|
SQL
|
|
244
258
|
target_ids.uniq!
|
|
245
259
|
update_ids = target_ids
|
|
246
|
-
old_ids = []
|
|
247
260
|
## use self.dbi instead of @dbi to get information about
|
|
248
261
|
## object_connections previously stored within this transaction
|
|
249
|
-
if(rows =
|
|
262
|
+
if (rows = dbi.select_all(sql, origin_id))
|
|
250
263
|
old_ids = rows.collect { |row| row[0] }
|
|
251
264
|
old_ids.uniq!
|
|
252
265
|
delete_ids = old_ids - target_ids
|
|
253
266
|
update_ids = target_ids - old_ids
|
|
254
|
-
unless
|
|
255
|
-
|
|
256
|
-
|
|
267
|
+
unless delete_ids.empty?
|
|
268
|
+
until (ids = delete_ids.slice!(0, BULK_FETCH_STEP)).empty?
|
|
269
|
+
dbi.do <<-SQL, origin_id
|
|
257
270
|
DELETE FROM object_connection
|
|
258
|
-
WHERE origin_id = ? AND target_id IN (#{ids.join(
|
|
271
|
+
WHERE origin_id = ? AND target_id IN (#{ids.join(",")})
|
|
259
272
|
SQL
|
|
260
273
|
end
|
|
261
274
|
end
|
|
262
275
|
end
|
|
263
|
-
sth =
|
|
276
|
+
sth = dbi.prepare <<-SQL
|
|
264
277
|
INSERT INTO object_connection (origin_id, target_id)
|
|
265
278
|
VALUES (?, ?)
|
|
266
279
|
SQL
|
|
@@ -269,59 +282,66 @@ CREATE INDEX IF NOT EXISTS target_id_#{table_name} ON #{table_name}(target_id);
|
|
|
269
282
|
}
|
|
270
283
|
sth.finish
|
|
271
284
|
end
|
|
285
|
+
|
|
272
286
|
def ensure_target_id_index(table_name)
|
|
273
|
-
#index target_id
|
|
274
|
-
|
|
287
|
+
# index target_id
|
|
288
|
+
dbi.do <<-SQL
|
|
275
289
|
CREATE INDEX IF NOT EXISTS target_id_#{table_name}
|
|
276
290
|
ON #{table_name}(target_id)
|
|
277
291
|
SQL
|
|
278
292
|
rescue
|
|
279
293
|
end
|
|
294
|
+
|
|
280
295
|
def extent_count(klass)
|
|
281
|
-
|
|
296
|
+
dbi.select_one(<<-EOQ, klass.to_s).first
|
|
282
297
|
SELECT COUNT(odba_id) FROM object WHERE extent = ?
|
|
283
298
|
EOQ
|
|
284
299
|
end
|
|
300
|
+
|
|
285
301
|
def extent_ids(klass)
|
|
286
|
-
|
|
302
|
+
dbi.select_all(<<-EOQ, klass.to_s).flatten
|
|
287
303
|
SELECT odba_id FROM object WHERE extent = ?
|
|
288
304
|
EOQ
|
|
289
305
|
end
|
|
306
|
+
|
|
290
307
|
def fulltext_index_delete(index_name, id, id_name)
|
|
291
|
-
|
|
308
|
+
dbi.do <<-SQL, id
|
|
292
309
|
DELETE FROM #{index_name}
|
|
293
310
|
WHERE #{id_name} = ?
|
|
294
311
|
SQL
|
|
295
312
|
end
|
|
313
|
+
|
|
296
314
|
def get_server_version
|
|
297
|
-
/\s([\d\.]+)\s/.match(
|
|
315
|
+
/\s([\d\.]+)\s/.match(dbi.select_all("select version();").first.first)[1]
|
|
298
316
|
end
|
|
317
|
+
|
|
299
318
|
def fulltext_index_target_ids(index_name, origin_id)
|
|
300
319
|
sql = <<-SQL
|
|
301
320
|
SELECT DISTINCT target_id
|
|
302
321
|
FROM #{index_name}
|
|
303
322
|
WHERE origin_id=?
|
|
304
323
|
SQL
|
|
305
|
-
|
|
324
|
+
dbi.select_all(sql, origin_id)
|
|
306
325
|
end
|
|
326
|
+
|
|
307
327
|
def generate_dictionary(language)
|
|
308
328
|
# postgres searches for the dictionary file in the directory share/tsearch_data of it installation location
|
|
309
329
|
# By default under gentoo, this is /usr/share/postgresql/tsearch_data/
|
|
310
330
|
# Use /usr/local/pgsql-10.1/bin/pg_config --sharedir to get the current value
|
|
311
331
|
# As we have no way to get the current installation path, we do not check whether the files are present or not
|
|
312
|
-
file=
|
|
332
|
+
file = "fulltext"
|
|
313
333
|
# setup configuration
|
|
314
|
-
|
|
334
|
+
dbi.do <<-SQL
|
|
315
335
|
DROP TEXT SEARCH DICTIONARY IF EXISTS public.default_#{language};
|
|
316
336
|
SQL
|
|
317
|
-
|
|
337
|
+
dbi.do <<-SQL
|
|
318
338
|
CREATE TEXT SEARCH CONFIGURATION public.default_#{language} ( COPY = pg_catalog.#{language} );
|
|
319
339
|
SQL
|
|
320
340
|
# ispell
|
|
321
|
-
|
|
341
|
+
dbi.do <<-SQL
|
|
322
342
|
DROP TEXT SEARCH DICTIONARY IF EXISTS #{language}_ispell;
|
|
323
343
|
SQL
|
|
324
|
-
|
|
344
|
+
dbi.do <<-SQL
|
|
325
345
|
CREATE TEXT SEARCH DICTIONARY #{language}_ispell (
|
|
326
346
|
TEMPLATE = ispell,
|
|
327
347
|
DictFile = #{language}_#{file},
|
|
@@ -332,35 +352,39 @@ CREATE INDEX IF NOT EXISTS target_id_#{table_name} ON #{table_name}(target_id);
|
|
|
332
352
|
# stem is already there.
|
|
333
353
|
create_dictionary_map(language)
|
|
334
354
|
end
|
|
355
|
+
|
|
335
356
|
def index_delete_origin(index_name, odba_id, term)
|
|
336
|
-
|
|
357
|
+
dbi.do <<-SQL, odba_id, term
|
|
337
358
|
DELETE FROM #{index_name}
|
|
338
359
|
WHERE origin_id = ?
|
|
339
360
|
AND search_term = ?
|
|
340
361
|
SQL
|
|
341
362
|
end
|
|
363
|
+
|
|
342
364
|
def index_delete_target(index_name, origin_id, search_term, target_id)
|
|
343
|
-
|
|
365
|
+
dbi.do <<-SQL, origin_id, search_term, target_id
|
|
344
366
|
DELETE FROM #{index_name}
|
|
345
367
|
WHERE origin_id = ?
|
|
346
368
|
AND search_term = ?
|
|
347
369
|
AND target_id = ?
|
|
348
370
|
SQL
|
|
349
371
|
end
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
372
|
+
|
|
373
|
+
def index_fetch_keys(index_name, length = nil)
|
|
374
|
+
expr = if length
|
|
375
|
+
"substr(search_term, 1, #{length})"
|
|
376
|
+
else
|
|
377
|
+
"search_term"
|
|
378
|
+
end
|
|
356
379
|
sql = <<-SQL
|
|
357
380
|
SELECT DISTINCT #{expr} AS key
|
|
358
381
|
FROM #{index_name}
|
|
359
382
|
ORDER BY key
|
|
360
383
|
SQL
|
|
361
|
-
|
|
384
|
+
dbi.select_all(sql).flatten
|
|
362
385
|
end
|
|
363
|
-
|
|
386
|
+
|
|
387
|
+
def index_matches(index_name, substring, limit = nil, offset = 0)
|
|
364
388
|
sql = <<-SQL
|
|
365
389
|
SELECT DISTINCT search_term AS key
|
|
366
390
|
FROM #{index_name}
|
|
@@ -373,41 +397,47 @@ CREATE INDEX IF NOT EXISTS target_id_#{table_name} ON #{table_name}(target_id);
|
|
|
373
397
|
if offset > 0
|
|
374
398
|
sql << "OFFSET #{offset}\n"
|
|
375
399
|
end
|
|
376
|
-
|
|
400
|
+
dbi.select_all(sql, substring + "%").flatten
|
|
377
401
|
end
|
|
402
|
+
|
|
378
403
|
def index_origin_ids(index_name, target_id)
|
|
379
404
|
sql = <<-SQL
|
|
380
405
|
SELECT DISTINCT origin_id, search_term
|
|
381
406
|
FROM #{index_name}
|
|
382
407
|
WHERE target_id=?
|
|
383
408
|
SQL
|
|
384
|
-
|
|
409
|
+
dbi.select_all(sql, target_id)
|
|
385
410
|
end
|
|
411
|
+
|
|
386
412
|
def index_target_ids(index_name, origin_id)
|
|
387
413
|
sql = <<-SQL
|
|
388
414
|
SELECT DISTINCT target_id, search_term
|
|
389
415
|
FROM #{index_name}
|
|
390
416
|
WHERE origin_id=?
|
|
391
417
|
SQL
|
|
392
|
-
|
|
418
|
+
dbi.select_all(sql, origin_id)
|
|
393
419
|
end
|
|
420
|
+
|
|
394
421
|
def max_id
|
|
395
422
|
@id_mutex.synchronize do
|
|
396
423
|
ensure_next_id_set
|
|
397
424
|
@next_id
|
|
398
425
|
end
|
|
399
426
|
end
|
|
427
|
+
|
|
400
428
|
def next_id
|
|
401
429
|
@id_mutex.synchronize do
|
|
402
430
|
ensure_next_id_set
|
|
403
431
|
@next_id += 1
|
|
404
432
|
end
|
|
405
433
|
end
|
|
434
|
+
|
|
406
435
|
def update_max_id(id)
|
|
407
436
|
@id_mutex.synchronize do
|
|
408
437
|
@next_id = id
|
|
409
438
|
end
|
|
410
439
|
end
|
|
440
|
+
|
|
411
441
|
def reserve_next_id(reserved_id)
|
|
412
442
|
@id_mutex.synchronize do
|
|
413
443
|
ensure_next_id_set
|
|
@@ -415,92 +445,98 @@ CREATE INDEX IF NOT EXISTS target_id_#{table_name} ON #{table_name}(target_id);
|
|
|
415
445
|
@next_id = reserved_id
|
|
416
446
|
else
|
|
417
447
|
raise OdbaDuplicateIdError,
|
|
418
|
-
|
|
448
|
+
"The id '#{reserved_id}' has already been assigned"
|
|
419
449
|
end
|
|
420
450
|
end
|
|
421
451
|
end
|
|
452
|
+
|
|
422
453
|
def remove_dictionary(language)
|
|
423
454
|
# remove configuration
|
|
424
|
-
|
|
455
|
+
dbi.do <<-SQL
|
|
425
456
|
DROP TEXT SEARCH CONFIGURATION IF EXISTS default_#{language}
|
|
426
457
|
SQL
|
|
427
458
|
# remove ispell dictionaries
|
|
428
|
-
|
|
459
|
+
dbi.do <<-SQL
|
|
429
460
|
DROP TEXT SEARCH DICTIONARY IF EXISTS #{language}_ispell;
|
|
430
461
|
SQL
|
|
431
462
|
end
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
463
|
+
|
|
464
|
+
def restore(odba_id)
|
|
465
|
+
row = dbi.select_one("SELECT content FROM object WHERE odba_id = ?", odba_id)
|
|
466
|
+
row.first unless row.nil?
|
|
467
|
+
end
|
|
468
|
+
|
|
469
|
+
def retrieve_connected_objects(target_id)
|
|
470
|
+
sql = <<-SQL
|
|
438
471
|
SELECT origin_id FROM object_connection
|
|
439
472
|
WHERE target_id = ?
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
473
|
+
SQL
|
|
474
|
+
dbi.select_all(sql, target_id)
|
|
475
|
+
end
|
|
476
|
+
|
|
477
|
+
def retrieve_from_condition_index(index_name, conditions, limit = nil)
|
|
444
478
|
sql = <<-EOQ
|
|
445
479
|
SELECT target_id, COUNT(target_id) AS relevance
|
|
446
480
|
FROM #{index_name}
|
|
447
481
|
WHERE TRUE
|
|
448
482
|
EOQ
|
|
449
483
|
values = []
|
|
450
|
-
|
|
484
|
+
conditions.collect { |name, info|
|
|
451
485
|
val = nil
|
|
452
486
|
condition = nil
|
|
453
|
-
if
|
|
454
|
-
condition = info[
|
|
455
|
-
if(val = info[
|
|
456
|
-
if
|
|
457
|
-
val +=
|
|
487
|
+
if info.is_a?(Hash)
|
|
488
|
+
condition = info["condition"]
|
|
489
|
+
if (val = info["value"])
|
|
490
|
+
if /i?like/i.match?(condition)
|
|
491
|
+
val += "%"
|
|
458
492
|
end
|
|
459
|
-
condition = "#{condition ||
|
|
493
|
+
condition = "#{condition || "="} ?"
|
|
460
494
|
values.push(val.to_s)
|
|
461
495
|
end
|
|
462
|
-
elsif
|
|
496
|
+
elsif info
|
|
463
497
|
condition = "= ?"
|
|
464
498
|
values.push(info.to_s)
|
|
465
499
|
end
|
|
466
500
|
sql << <<-EOQ
|
|
467
|
-
AND #{name} #{condition ||
|
|
501
|
+
AND #{name} #{condition || "IS NULL"}
|
|
468
502
|
EOQ
|
|
469
503
|
}
|
|
470
504
|
sql << " GROUP BY target_id\n"
|
|
471
|
-
if
|
|
505
|
+
if limit
|
|
472
506
|
sql << " LIMIT #{limit}"
|
|
473
507
|
end
|
|
474
|
-
|
|
508
|
+
dbi.select_all(sql, *values)
|
|
475
509
|
end
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
510
|
+
|
|
511
|
+
def retrieve_from_fulltext_index(index_name, search_term, limit = nil)
|
|
512
|
+
## this combination of gsub statements solves the problem of
|
|
513
|
+
# properly escaping strings of this form: "(2:1)" into
|
|
479
514
|
# '\(2\:1\)' (see test_retrieve_from_fulltext_index)
|
|
480
|
-
|
|
481
|
-
.gsub(/[():]/i, '\\ \\&').gsub(/\s/,
|
|
482
|
-
|
|
515
|
+
term = search_term.strip.gsub(/\s+/, "&").squeeze("&")
|
|
516
|
+
.gsub(/[():]/i, '\\ \\&').gsub(/\s/, "")
|
|
517
|
+
sql = <<-EOQ
|
|
483
518
|
SELECT target_id,
|
|
484
519
|
max(ts_rank(search_term, to_tsquery(?))) AS relevance
|
|
485
520
|
FROM #{index_name}
|
|
486
521
|
WHERE search_term @@ to_tsquery(?)
|
|
487
522
|
GROUP BY target_id
|
|
488
523
|
ORDER BY relevance DESC
|
|
489
|
-
|
|
490
|
-
if
|
|
524
|
+
EOQ
|
|
525
|
+
if limit
|
|
491
526
|
sql << " LIMIT #{limit}"
|
|
492
527
|
end
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
528
|
+
dbi.select_all(sql, term, term)
|
|
529
|
+
rescue DBI::ProgrammingError => e
|
|
530
|
+
warn("ODBA::Storage.retrieve_from_fulltext_index rescued a DBI::ProgrammingError(#{e.message}). Query:")
|
|
531
|
+
warn("self.dbi.select_all(#{sql}, #{term}, #{term})")
|
|
532
|
+
warn("returning empty result")
|
|
533
|
+
[]
|
|
534
|
+
end
|
|
535
|
+
|
|
536
|
+
def retrieve_from_index(index_name, search_term,
|
|
537
|
+
exact = nil, limit = nil)
|
|
538
|
+
unless exact
|
|
539
|
+
search_term += "%"
|
|
504
540
|
end
|
|
505
541
|
sql = <<-EOQ
|
|
506
542
|
SELECT target_id, COUNT(target_id) AS relevance
|
|
@@ -508,71 +544,85 @@ CREATE INDEX IF NOT EXISTS target_id_#{table_name} ON #{table_name}(target_id);
|
|
|
508
544
|
WHERE search_term LIKE ?
|
|
509
545
|
GROUP BY target_id
|
|
510
546
|
EOQ
|
|
511
|
-
if
|
|
547
|
+
if limit
|
|
512
548
|
sql << " LIMIT #{limit}"
|
|
513
549
|
end
|
|
514
|
-
|
|
550
|
+
dbi.select_all(sql, search_term)
|
|
515
551
|
end
|
|
516
|
-
|
|
517
|
-
|
|
552
|
+
|
|
553
|
+
def restore_collection(odba_id)
|
|
554
|
+
dbi.select_all <<-EOQ
|
|
518
555
|
SELECT key, value FROM collection WHERE odba_id = #{odba_id}
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
556
|
+
EOQ
|
|
557
|
+
end
|
|
558
|
+
|
|
559
|
+
def restore_named(name)
|
|
560
|
+
row = dbi.select_one("SELECT content FROM object WHERE name = ?",
|
|
561
|
+
name)
|
|
562
|
+
row.first unless row.nil?
|
|
563
|
+
end
|
|
564
|
+
|
|
565
|
+
def restore_prefetchable
|
|
566
|
+
dbi.select_all <<-EOQ
|
|
528
567
|
SELECT odba_id, content FROM object WHERE prefetchable = true
|
|
529
|
-
|
|
530
|
-
|
|
568
|
+
EOQ
|
|
569
|
+
end
|
|
570
|
+
|
|
531
571
|
def setup
|
|
572
|
+
old_stderr = $stderr.dup # We want to silence the annoying NOTICE: relation "object" already exists, skipping
|
|
573
|
+
$stderr.reopen("/dev/null", "w")
|
|
532
574
|
TABLES.each { |name, definition|
|
|
533
|
-
|
|
575
|
+
begin
|
|
576
|
+
dbi.do(definition)
|
|
577
|
+
rescue
|
|
578
|
+
$stderr = old_stderr
|
|
579
|
+
DBI::ProgrammingError
|
|
580
|
+
end
|
|
534
581
|
}
|
|
535
|
-
unless
|
|
536
|
-
|
|
537
|
-
ALTER TABLE object ADD COLUMN extent TEXT;
|
|
538
|
-
CREATE INDEX IF NOT EXISTS extent_index ON object(extent);
|
|
582
|
+
unless dbi.columns("object").any? { |col| col.name == "extent" }
|
|
583
|
+
dbi.do <<~EOS
|
|
584
|
+
ALTER TABLE object ADD COLUMN extent TEXT;
|
|
585
|
+
CREATE INDEX IF NOT EXISTS extent_index ON object(extent);
|
|
539
586
|
EOS
|
|
540
587
|
end
|
|
588
|
+
$stderr = old_stderr
|
|
541
589
|
end
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
590
|
+
|
|
591
|
+
def store(odba_id, dump, name, prefetchable, klass)
|
|
592
|
+
sql = "SELECT name FROM object WHERE odba_id = ?"
|
|
593
|
+
if (row = dbi.select_one(sql, odba_id))
|
|
594
|
+
name ||= row["name"]
|
|
595
|
+
dbi.do <<-SQL, dump, name, prefetchable, klass.to_s, odba_id
|
|
547
596
|
UPDATE object SET
|
|
548
597
|
content = ?,
|
|
549
598
|
name = ?,
|
|
550
599
|
prefetchable = ?,
|
|
551
600
|
extent = ?
|
|
552
601
|
WHERE odba_id = ?
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
602
|
+
SQL
|
|
603
|
+
else
|
|
604
|
+
dbi.do <<-SQL, odba_id, dump, name, prefetchable, klass.to_s
|
|
556
605
|
INSERT INTO object (odba_id, content, name, prefetchable, extent)
|
|
557
606
|
VALUES (?, ?, ?, ?, ?)
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
607
|
+
SQL
|
|
608
|
+
end
|
|
609
|
+
end
|
|
610
|
+
|
|
611
|
+
def transaction(&block)
|
|
612
|
+
retval = nil
|
|
613
|
+
@dbi.transaction { |dbi|
|
|
565
614
|
## this should not be necessary anymore:
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
615
|
+
# dbi['AutoCommit'] = false
|
|
616
|
+
Thread.current[:txn] = dbi
|
|
617
|
+
retval = block.call
|
|
618
|
+
}
|
|
619
|
+
retval
|
|
620
|
+
ensure
|
|
572
621
|
## this should not be necessary anymore:
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
622
|
+
# dbi['AutoCommit'] = true
|
|
623
|
+
Thread.current[:txn] = nil
|
|
624
|
+
end
|
|
625
|
+
|
|
576
626
|
def update_condition_index(index_name, origin_id, search_terms, target_id)
|
|
577
627
|
keys = []
|
|
578
628
|
vals = []
|
|
@@ -580,63 +630,64 @@ CREATE INDEX IF NOT EXISTS extent_index ON object(extent);
|
|
|
580
630
|
keys.push(key)
|
|
581
631
|
vals.push(val)
|
|
582
632
|
}
|
|
583
|
-
if
|
|
584
|
-
|
|
585
|
-
INSERT INTO #{index_name} (origin_id, target_id, #{keys.join(
|
|
586
|
-
VALUES (?, ?#{
|
|
633
|
+
if target_id
|
|
634
|
+
dbi.do <<~SQL, origin_id, target_id, *vals
|
|
635
|
+
INSERT INTO #{index_name} (origin_id, target_id, #{keys.join(", ")})
|
|
636
|
+
VALUES (?, ?#{", ?" * keys.size})
|
|
587
637
|
SQL
|
|
588
638
|
else
|
|
589
|
-
key_str = keys.collect { |key| "#{key}=?" }.join(
|
|
590
|
-
|
|
591
|
-
UPDATE #{index_name} SET #{key_str}
|
|
592
|
-
WHERE origin_id = ?
|
|
639
|
+
key_str = keys.collect { |key| "#{key}=?" }.join(", ")
|
|
640
|
+
dbi.do <<~SQL, *vals.push(origin_id)
|
|
641
|
+
UPDATE #{index_name} SET #{key_str}
|
|
642
|
+
WHERE origin_id = ?
|
|
593
643
|
SQL
|
|
594
644
|
end
|
|
595
645
|
end
|
|
646
|
+
|
|
596
647
|
def update_fulltext_index(index_name, origin_id, search_term, target_id)
|
|
597
|
-
search_term = search_term.gsub(/\s+/,
|
|
598
|
-
if
|
|
599
|
-
|
|
600
|
-
INSERT INTO #{index_name} (origin_id, search_term, target_id)
|
|
601
|
-
VALUES (?, to_tsvector(?), ?)
|
|
602
|
-
SQL
|
|
603
|
-
result = self.dbi.do <<-SQL, origin_id.to_s, search_term, target_id
|
|
604
|
-
INSERT INTO #{index_name} (origin_id, search_term, target_id)
|
|
605
|
-
VALUES (?, to_tsvector(?), ?)
|
|
648
|
+
search_term = search_term.gsub(/\s+/, " ").strip
|
|
649
|
+
if target_id
|
|
650
|
+
dbi.do <<~SQL, origin_id.to_s, search_term, target_id
|
|
651
|
+
INSERT INTO #{index_name} (origin_id, search_term, target_id)
|
|
652
|
+
VALUES (?, to_tsvector(?), ?)
|
|
606
653
|
SQL
|
|
607
654
|
else
|
|
608
|
-
|
|
609
|
-
UPDATE #{index_name} SET search_term=to_tsvector(?)
|
|
610
|
-
WHERE origin_id=?
|
|
655
|
+
dbi.do <<~SQL, search_term, origin_id
|
|
656
|
+
UPDATE #{index_name} SET search_term=to_tsvector(?)
|
|
657
|
+
WHERE origin_id=?
|
|
611
658
|
SQL
|
|
612
659
|
end
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
660
|
+
end
|
|
661
|
+
|
|
662
|
+
def update_index(index_name, origin_id, search_term, target_id)
|
|
663
|
+
if target_id
|
|
664
|
+
dbi.do <<-SQL, origin_id, search_term, target_id
|
|
617
665
|
INSERT INTO #{index_name} (origin_id, search_term, target_id)
|
|
618
666
|
VALUES (?, ?, ?)
|
|
619
667
|
SQL
|
|
620
668
|
else
|
|
621
|
-
|
|
669
|
+
dbi.do <<-SQL, search_term, origin_id
|
|
622
670
|
UPDATE #{index_name} SET search_term=?
|
|
623
671
|
WHERE origin_id=?
|
|
624
672
|
SQL
|
|
625
673
|
end
|
|
626
|
-
|
|
627
|
-
|
|
674
|
+
end
|
|
675
|
+
|
|
676
|
+
private
|
|
677
|
+
|
|
628
678
|
def ensure_next_id_set
|
|
629
|
-
if
|
|
679
|
+
if @next_id.nil?
|
|
630
680
|
@next_id = restore_max_id
|
|
631
681
|
end
|
|
632
682
|
end
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
683
|
+
|
|
684
|
+
def restore_max_id
|
|
685
|
+
row = dbi.select_one("SELECT odba_id FROM object ORDER BY odba_id DESC LIMIT 1")
|
|
686
|
+
if row.nil? || row.first.nil?
|
|
687
|
+
0
|
|
688
|
+
else
|
|
689
|
+
row.first
|
|
690
|
+
end
|
|
691
|
+
end
|
|
692
|
+
end
|
|
642
693
|
end
|