rufus-tokyo 0.1.14 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG.txt +9 -0
- data/CREDITS.txt +10 -1
- data/README.rdoc +9 -11
- data/TODO.txt +17 -5
- data/lib/rufus/edo/cabcore.rb +29 -6
- data/lib/rufus/edo/ntyrant/abstract.rb +1 -1
- data/lib/rufus/edo/ntyrant/table.rb +14 -0
- data/lib/rufus/edo/tabcore.rb +45 -18
- data/lib/rufus/tokyo.rb +1 -1
- data/lib/rufus/tokyo/cabinet/abstract.rb +33 -32
- data/lib/rufus/tokyo/cabinet/lib.rb +12 -17
- data/lib/rufus/tokyo/cabinet/table.rb +53 -20
- data/lib/rufus/tokyo/dystopia.rb +10 -7
- data/lib/rufus/tokyo/dystopia/core.rb +198 -0
- data/lib/rufus/tokyo/dystopia/lib.rb +65 -3
- data/lib/rufus/tokyo/dystopia/words.rb +1 -1
- data/lib/rufus/tokyo/outlen.rb +46 -0
- data/lib/rufus/tokyo/query.rb +9 -1
- data/lib/rufus/tokyo/tyrant/abstract.rb +4 -26
- data/lib/rufus/tokyo/tyrant/ext.rb +52 -0
- data/lib/rufus/tokyo/tyrant/lib.rb +6 -6
- data/lib/rufus/tokyo/tyrant/table.rb +3 -1
- data/lib/rufus/tokyo/utils.rb +44 -0
- data/spec/spec.rb +3 -1
- metadata +6 -2
data/lib/rufus/tokyo.rb
CHANGED
@@ -24,6 +24,7 @@
|
|
24
24
|
|
25
25
|
|
26
26
|
require 'rufus/tokyo/transactions'
|
27
|
+
require 'rufus/tokyo/outlen'
|
27
28
|
|
28
29
|
|
29
30
|
module Rufus::Tokyo
|
@@ -52,6 +53,7 @@ module Rufus::Tokyo
|
|
52
53
|
|
53
54
|
include HashMethods
|
54
55
|
include Transactions
|
56
|
+
include Outlen
|
55
57
|
|
56
58
|
# Creates/opens the cabinet, raises an exception in case of
|
57
59
|
# creation/opening failure.
|
@@ -189,8 +191,8 @@ module Rufus::Tokyo
|
|
189
191
|
|
190
192
|
name = name + params.collect { |k, v| "##{k}=#{v}" }.join('')
|
191
193
|
|
192
|
-
(lib.tcadbopen(@db, name) == 1) ||
|
193
|
-
|
194
|
+
(lib.tcadbopen(@db, name) == 1) || raise(
|
195
|
+
TokyoError.new("failed to open/create db '#{name}' #{params.inspect}"))
|
194
196
|
|
195
197
|
self.default = params[:default]
|
196
198
|
@default_proc ||= params[:default_proc]
|
@@ -249,6 +251,8 @@ module Rufus::Tokyo
|
|
249
251
|
#
|
250
252
|
def []= (k, v)
|
251
253
|
|
254
|
+
k = k.to_s; v = v.to_s
|
255
|
+
|
252
256
|
lib.abs_put(@db, k, Rufus::Tokyo.blen(k), v, Rufus::Tokyo.blen(v))
|
253
257
|
end
|
254
258
|
|
@@ -257,6 +261,8 @@ module Rufus::Tokyo
|
|
257
261
|
#
|
258
262
|
def putkeep (k, v)
|
259
263
|
|
264
|
+
k = k.to_s; v = v.to_s
|
265
|
+
|
260
266
|
(lib.abs_putkeep(
|
261
267
|
@db, k, Rufus::Tokyo.blen(k), v, Rufus::Tokyo.blen(v)) == 1)
|
262
268
|
end
|
@@ -265,6 +271,8 @@ module Rufus::Tokyo
|
|
265
271
|
#
|
266
272
|
def get (k)
|
267
273
|
|
274
|
+
k = k.to_s
|
275
|
+
|
268
276
|
outlen_op(:abs_get, k, Rufus::Tokyo.blen(k))
|
269
277
|
end
|
270
278
|
protected :get
|
@@ -274,6 +282,8 @@ module Rufus::Tokyo
|
|
274
282
|
#
|
275
283
|
def delete (k)
|
276
284
|
|
285
|
+
k = k.to_s
|
286
|
+
|
277
287
|
v = self[k]
|
278
288
|
|
279
289
|
(lib.abs_out(@db, k, Rufus::Tokyo.blen(k)) == 1) ? v : nil
|
@@ -412,6 +422,8 @@ module Rufus::Tokyo
|
|
412
422
|
#
|
413
423
|
def lget (keys)
|
414
424
|
|
425
|
+
keys = keys.collect { |k| k.to_s }
|
426
|
+
|
415
427
|
Hash[*call_misc('getlist', Rufus::Tokyo::List.new(keys))]
|
416
428
|
end
|
417
429
|
|
@@ -421,7 +433,12 @@ module Rufus::Tokyo
|
|
421
433
|
|
422
434
|
call_misc(
|
423
435
|
'putlist',
|
424
|
-
hash.inject(Rufus::Tokyo::List.new) { |l, (k, v)|
|
436
|
+
hash.inject(Rufus::Tokyo::List.new) { |l, (k, v)|
|
437
|
+
l << k.to_s
|
438
|
+
l << v.to_s
|
439
|
+
l
|
440
|
+
})
|
441
|
+
|
425
442
|
self
|
426
443
|
end
|
427
444
|
alias :lput :merge!
|
@@ -430,6 +447,8 @@ module Rufus::Tokyo
|
|
430
447
|
#
|
431
448
|
def ldelete (keys)
|
432
449
|
|
450
|
+
keys = keys.collect { |k| k.to_s }
|
451
|
+
|
433
452
|
call_misc('outlist', Rufus::Tokyo::List.new(keys))
|
434
453
|
end
|
435
454
|
|
@@ -440,6 +459,8 @@ module Rufus::Tokyo
|
|
440
459
|
#
|
441
460
|
def incr (key, inc=1)
|
442
461
|
|
462
|
+
key = key.to_s
|
463
|
+
|
443
464
|
v = inc.is_a?(Fixnum) ?
|
444
465
|
lib.addint(@db, key, Rufus::Tokyo.blen(key), inc) :
|
445
466
|
lib.adddouble(@db, key, Rufus::Tokyo.blen(key), inc)
|
@@ -475,7 +496,7 @@ module Rufus::Tokyo
|
|
475
496
|
#
|
476
497
|
def tranbegin
|
477
498
|
|
478
|
-
check_transaction_support
|
499
|
+
#check_transaction_support
|
479
500
|
|
480
501
|
libcall(:tcadbtranbegin)
|
481
502
|
end
|
@@ -487,7 +508,7 @@ module Rufus::Tokyo
|
|
487
508
|
#
|
488
509
|
def trancommit
|
489
510
|
|
490
|
-
check_transaction_support
|
511
|
+
#check_transaction_support
|
491
512
|
|
492
513
|
libcall(:tcadbtrancommit)
|
493
514
|
end
|
@@ -499,20 +520,19 @@ module Rufus::Tokyo
|
|
499
520
|
#
|
500
521
|
def tranabort
|
501
522
|
|
502
|
-
check_transaction_support
|
523
|
+
#check_transaction_support
|
503
524
|
|
504
525
|
libcall(:tcadbtranabort)
|
505
526
|
end
|
506
527
|
|
507
528
|
protected
|
508
529
|
|
509
|
-
def check_transaction_support
|
510
|
-
|
511
|
-
|
512
|
-
|
513
|
-
|
514
|
-
|
515
|
-
end
|
530
|
+
#def check_transaction_support
|
531
|
+
# raise(TokyoError.new(
|
532
|
+
# "The version of Tokyo Cabinet you're using doesn't support " +
|
533
|
+
# "transactions for non-table structures. Upgrade to TC >= 1.4.13.")
|
534
|
+
# ) unless lib.respond_to?(:tcadbtranbegin)
|
535
|
+
#end
|
516
536
|
|
517
537
|
# Wrapping tcadbmisc or tcrdbmisc
|
518
538
|
# (and taking care of freeing the list_pointer)
|
@@ -544,25 +564,6 @@ module Rufus::Tokyo
|
|
544
564
|
raise TokyoError.new("call to #{lib_method} failed")
|
545
565
|
end
|
546
566
|
|
547
|
-
# A wrapper for library returning a string (binary data potentially)
|
548
|
-
#
|
549
|
-
def outlen_op (method, *args)
|
550
|
-
|
551
|
-
args.unshift(@db)
|
552
|
-
|
553
|
-
outlen = FFI::MemoryPointer.new(:int)
|
554
|
-
args << outlen
|
555
|
-
|
556
|
-
out = lib.send(method, *args)
|
557
|
-
|
558
|
-
return nil if out.address == 0
|
559
|
-
|
560
|
-
return out.get_bytes(0, outlen.get_int(0))
|
561
|
-
|
562
|
-
ensure
|
563
|
-
|
564
|
-
outlen.free
|
565
|
-
end
|
566
567
|
end
|
567
568
|
end
|
568
569
|
|
@@ -29,18 +29,20 @@ module Rufus::Tokyo
|
|
29
29
|
# The libtokyocabinet.so methods get bound to this module
|
30
30
|
#
|
31
31
|
module CabinetLib #:nodoc#
|
32
|
+
|
32
33
|
extend FFI::Library
|
33
34
|
|
34
35
|
#
|
35
36
|
# find Tokyo Cabinet lib
|
36
37
|
|
37
|
-
paths = Array(ENV['TOKYO_CABINET_LIB'] || %w
|
38
|
+
paths = Array(ENV['TOKYO_CABINET_LIB'] || %w[
|
38
39
|
/usr/lib/libtokyocabinet.so
|
40
|
+
/usr/lib64/libtokyocabinet.so
|
39
41
|
/opt/local/lib/libtokyocabinet.dylib
|
40
42
|
/opt/local/lib/libtokyocabinet.so
|
41
43
|
/usr/local/lib/libtokyocabinet.dylib
|
42
44
|
/usr/local/lib/libtokyocabinet.so
|
43
|
-
|
45
|
+
])
|
44
46
|
|
45
47
|
begin
|
46
48
|
|
@@ -107,12 +109,10 @@ module Rufus::Tokyo
|
|
107
109
|
attfunc :addint, :tcadbaddint, [ :pointer, :string, :int, :int ], :int
|
108
110
|
attfunc :adddouble, :tcadbadddouble, [ :pointer, :string, :int, :double ], :double
|
109
111
|
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
rescue FFI::NotFoundError => nfe
|
115
|
-
end
|
112
|
+
# since TC 1.4.13
|
113
|
+
attfunc :tcadbtranbegin, [ :pointer ], :int
|
114
|
+
attfunc :tcadbtrancommit, [ :pointer ], :int
|
115
|
+
attfunc :tcadbtranabort, [ :pointer ], :int
|
116
116
|
|
117
117
|
#
|
118
118
|
# tctdb functions
|
@@ -125,10 +125,8 @@ module Rufus::Tokyo
|
|
125
125
|
attfunc :tctdbsetcache, [ :pointer, :uint32, :uint32, :uint32 ], :int
|
126
126
|
attfunc :tctdbsetxmsiz, [ :pointer, :uint64 ], :int
|
127
127
|
|
128
|
-
|
129
|
-
|
130
|
-
rescue FFI::NotFoundError => nfe
|
131
|
-
end
|
128
|
+
# since TC 1.4.21
|
129
|
+
attfunc :tctdbsetdfunit, [ :pointer, :uint32 ], :int
|
132
130
|
|
133
131
|
attfunc :tctdbopen, [ :pointer, :string, :int ], :int
|
134
132
|
|
@@ -189,10 +187,8 @@ module Rufus::Tokyo
|
|
189
187
|
attfunc :qry_search, :tctdbqrysearch, [ :pointer ], :pointer
|
190
188
|
attfunc :qry_searchout, :tctdbqrysearchout, [ :pointer ], :int
|
191
189
|
|
192
|
-
|
193
|
-
|
194
|
-
rescue FFI::NotFoundError => nfe
|
195
|
-
end
|
190
|
+
# since TC 1.4.12
|
191
|
+
attfunc :qry_count, :tctdbqrycount, [ :pointer ], :int
|
196
192
|
|
197
193
|
#
|
198
194
|
# tcmap functions
|
@@ -225,6 +221,5 @@ module Rufus::Tokyo
|
|
225
221
|
attfunc :tclistremove, [ :pointer, :int, :pointer ], :pointer
|
226
222
|
attfunc :tclistdel, [ :pointer ], :void
|
227
223
|
end
|
228
|
-
|
229
224
|
end
|
230
225
|
|
@@ -23,6 +23,7 @@
|
|
23
23
|
#++
|
24
24
|
|
25
25
|
|
26
|
+
require 'rufus/tokyo/utils'
|
26
27
|
require 'rufus/tokyo/query'
|
27
28
|
require 'rufus/tokyo/config'
|
28
29
|
require 'rufus/tokyo/transactions'
|
@@ -188,6 +189,10 @@ module Rufus::Tokyo
|
|
188
189
|
INDEX_TYPES = {
|
189
190
|
:lexical => 0,
|
190
191
|
:decimal => 1,
|
192
|
+
:token => 2,
|
193
|
+
:qgram => 3,
|
194
|
+
:opt => 9998,
|
195
|
+
:optimized => 9998,
|
191
196
|
:void => 9999,
|
192
197
|
:remove => 9999,
|
193
198
|
:keep => 1 << 24
|
@@ -195,7 +200,17 @@ module Rufus::Tokyo
|
|
195
200
|
|
196
201
|
# Sets an index on a column of the table.
|
197
202
|
#
|
198
|
-
# Types maybe be :lexical or :decimal
|
203
|
+
# Types maybe be :lexical or :decimal.
|
204
|
+
#
|
205
|
+
# Recently (TC 1.4.26 and 1.4.27) inverted indexes have been added,
|
206
|
+
# they are :token and :qgram. There is an :opt index as well.
|
207
|
+
#
|
208
|
+
# Sorry couldn't find any good doc about those inverted indexes apart from :
|
209
|
+
#
|
210
|
+
# http://alpha.mixi.co.jp/blog/?p=1147
|
211
|
+
# http://www.excite-webtl.jp/world/english/web/?wb_url=http%3A%2F%2Falpha.mixi.co.jp%2Fblog%2F%3Fp%3D1147&wb_lp=JAEN&wb_dis=2&wb_submit=+%96%7C+%96%F3+
|
212
|
+
#
|
213
|
+
# Use :keep to "add" and
|
199
214
|
# :remove (or :void) to "remove" an index.
|
200
215
|
#
|
201
216
|
# If column_name is :pk or "", the index will be set on the primary key.
|
@@ -204,7 +219,7 @@ module Rufus::Tokyo
|
|
204
219
|
#
|
205
220
|
def set_index (column_name, *types)
|
206
221
|
|
207
|
-
column_name =
|
222
|
+
column_name = column_name == :pk ? '' : column_name.to_s
|
208
223
|
|
209
224
|
i = types.inject(0) { |i, t| i = i | INDEX_TYPES[t]; i }
|
210
225
|
|
@@ -224,6 +239,9 @@ module Rufus::Tokyo
|
|
224
239
|
#
|
225
240
|
def []= (pk, h_or_a)
|
226
241
|
|
242
|
+
pk = pk.to_s
|
243
|
+
h_or_a = Rufus::Tokyo.h_or_a_to_s(h_or_a)
|
244
|
+
|
227
245
|
m = Rufus::Tokyo::Map[h_or_a]
|
228
246
|
|
229
247
|
r = lib.tab_put(@db, pk, Rufus::Tokyo.blen(pk), m.pointer)
|
@@ -242,6 +260,8 @@ module Rufus::Tokyo
|
|
242
260
|
#
|
243
261
|
def delete (k)
|
244
262
|
|
263
|
+
k = k.to_s
|
264
|
+
|
245
265
|
v = self[k]
|
246
266
|
return nil unless v
|
247
267
|
libcall(:tab_out, k, Rufus::Tokyo.blen(k))
|
@@ -305,7 +325,7 @@ module Rufus::Tokyo
|
|
305
325
|
|
306
326
|
ensure
|
307
327
|
|
308
|
-
outlen
|
328
|
+
outlen && outlen.free
|
309
329
|
end
|
310
330
|
|
311
331
|
# Deletes all the entries whose key begin with the given prefix.
|
@@ -321,7 +341,7 @@ module Rufus::Tokyo
|
|
321
341
|
ks = Rufus::Tokyo::List.new(ks)
|
322
342
|
ks.each { |k| self.delete(k) }
|
323
343
|
ensure
|
324
|
-
ks.free
|
344
|
+
ks && ks.free
|
325
345
|
end
|
326
346
|
end
|
327
347
|
|
@@ -331,7 +351,7 @@ module Rufus::Tokyo
|
|
331
351
|
#
|
332
352
|
def lget (keys)
|
333
353
|
|
334
|
-
keys.inject({}) { |h, k| v = self[k]; h[k] = v if v; h }
|
354
|
+
keys.inject({}) { |h, k| k = k.to_s; v = self[k]; h[k] = v if v; h }
|
335
355
|
end
|
336
356
|
|
337
357
|
# Returns the number of records in this table db
|
@@ -362,7 +382,7 @@ module Rufus::Tokyo
|
|
362
382
|
return rs
|
363
383
|
|
364
384
|
ensure
|
365
|
-
q.free
|
385
|
+
q && q.free
|
366
386
|
end
|
367
387
|
|
368
388
|
# Prepares and runs a query, returns an array of hashes (all Ruby)
|
@@ -376,7 +396,7 @@ module Rufus::Tokyo
|
|
376
396
|
return a
|
377
397
|
|
378
398
|
ensure
|
379
|
-
rs.free
|
399
|
+
rs && rs.free
|
380
400
|
end
|
381
401
|
|
382
402
|
# Prepares a query and then runs it and deletes all the results.
|
@@ -389,7 +409,7 @@ module Rufus::Tokyo
|
|
389
409
|
return rs
|
390
410
|
|
391
411
|
ensure
|
392
|
-
q.free
|
412
|
+
q && q.free
|
393
413
|
end
|
394
414
|
|
395
415
|
# Warning : this method is low-level, you probably only need
|
@@ -398,7 +418,6 @@ module Rufus::Tokyo
|
|
398
418
|
# Direct call for 'transaction begin'.
|
399
419
|
#
|
400
420
|
def tranbegin
|
401
|
-
|
402
421
|
libcall(:tctdbtranbegin)
|
403
422
|
end
|
404
423
|
|
@@ -443,10 +462,10 @@ module Rufus::Tokyo
|
|
443
462
|
|
444
463
|
def libcall (lib_method, *args)
|
445
464
|
|
446
|
-
|
465
|
+
(lib.send(lib_method, @db, *args) == 1) or raise_error
|
447
466
|
# stack level too deep with JRuby 1.1.6 :(
|
448
467
|
|
449
|
-
(eval(%{ lib.#{lib_method}(@db, *args) }) == 1) or raise_error
|
468
|
+
#(eval(%{ lib.#{lib_method}(@db, *args) }) == 1) or raise_error
|
450
469
|
# works with JRuby 1.1.6
|
451
470
|
end
|
452
471
|
|
@@ -552,8 +571,18 @@ module Rufus::Tokyo
|
|
552
571
|
#
|
553
572
|
# :numoreq # number which is equal to at least one token
|
554
573
|
#
|
574
|
+
# :ftsph # full-text phrase search
|
575
|
+
# :ftsphrase
|
576
|
+
# :phrase
|
577
|
+
# :ftsand # full-text AND
|
578
|
+
# :ftsor # full-text OR
|
579
|
+
# :ftsex # full-text with 'compound' expression
|
580
|
+
#
|
555
581
|
def add (colname, operator, val, affirmative=true, no_index=false)
|
556
582
|
|
583
|
+
colname = colname.to_s
|
584
|
+
val = val.to_s
|
585
|
+
|
557
586
|
op = operator.is_a?(Fixnum) ? operator : OPERATORS[operator]
|
558
587
|
op = op | TDBQCNEGATE unless affirmative
|
559
588
|
op = op | TDBQCNOIDX if no_index
|
@@ -585,13 +614,15 @@ module Rufus::Tokyo
|
|
585
614
|
# :numdesc
|
586
615
|
#
|
587
616
|
def order_by (colname, direction=:strasc)
|
588
|
-
|
617
|
+
|
618
|
+
lib.qry_setorder(@query, colname.to_s, DIRECTIONS[direction])
|
589
619
|
end
|
590
620
|
|
591
621
|
# When set to true, only the primary keys of the matching records will
|
592
622
|
# be returned.
|
593
623
|
#
|
594
624
|
def pk_only (on=true)
|
625
|
+
|
595
626
|
@opts[:pk_only] = on
|
596
627
|
end
|
597
628
|
|
@@ -658,8 +689,8 @@ module Rufus::Tokyo
|
|
658
689
|
#
|
659
690
|
def run
|
660
691
|
|
661
|
-
|
662
|
-
|
692
|
+
#@last_resultset =
|
693
|
+
TableResultSet.new(@table, lib.qry_search(@query), @opts)
|
663
694
|
end
|
664
695
|
|
665
696
|
# Runs this query AND let all the matching records get deleted.
|
@@ -675,16 +706,17 @@ module Rufus::Tokyo
|
|
675
706
|
#
|
676
707
|
def count
|
677
708
|
|
678
|
-
if lib.respond_to?(:qry_count)
|
679
|
-
|
680
|
-
else
|
681
|
-
|
682
|
-
end
|
709
|
+
#if lib.respond_to?(:qry_count)
|
710
|
+
lib.qry_count(@query)
|
711
|
+
#else
|
712
|
+
# @last_resultset ? @last_resultset.size : 0
|
713
|
+
#end
|
683
714
|
end
|
684
715
|
|
685
716
|
# Frees this data structure
|
686
717
|
#
|
687
718
|
def free
|
719
|
+
|
688
720
|
lib.qry_del(@query)
|
689
721
|
@query = nil
|
690
722
|
end
|
@@ -719,6 +751,7 @@ module Rufus::Tokyo
|
|
719
751
|
# The classical each
|
720
752
|
#
|
721
753
|
def each
|
754
|
+
|
722
755
|
(0..size-1).each do |i|
|
723
756
|
pk = @list[i]
|
724
757
|
if @opts[:pk_only]
|
@@ -745,9 +778,9 @@ module Rufus::Tokyo
|
|
745
778
|
@list.free
|
746
779
|
@list = nil
|
747
780
|
end
|
748
|
-
|
749
781
|
alias :close :free
|
750
782
|
alias :destroy :free
|
783
|
+
|
751
784
|
end
|
752
785
|
end
|
753
786
|
|