xapian_db 1.3.3.1 → 1.3.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/README.rdoc +28 -4
- data/lib/xapian_db.rb +1 -1
- data/lib/xapian_db/config.rb +13 -2
- data/lib/xapian_db/index_writers/sidekiq_worker.rb +43 -0
- data/lib/xapian_db/index_writers/sidekiq_writer.rb +43 -0
- data/lib/xapian_db/railtie.rb +2 -0
- metadata +18 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 629bfb4aa9ec303fad2535c704879fa91a3130e2
|
4
|
+
data.tar.gz: 70f8b7b084498b194891372992ec54cddcf8c51e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5e1d4a205c21a0e988ba43cace4d94921f5375f91d130845c8207152397f92f394e606be71d88684253ce22f9814c01fa030b6afb83efd8c35970ca0a2cdb369
|
7
|
+
data.tar.gz: 6b4785131589a8110202a805c8711dc1ac7b765af193d41ee22c0fa6245eb4e023fdee582e2e1e60e725bf40412de9bdab7bca3db6b636f222c5a64f3e9b7d31
|
data/CHANGELOG.md
CHANGED
data/README.rdoc
CHANGED
@@ -84,8 +84,8 @@ You can override these defaults by placing a config file named 'xapian_db.yml' i
|
|
84
84
|
- language: any iso language code, default: :none (activates spelling corrections, stemmer and stop words if an iso language code ist set)
|
85
85
|
- term_min_length: <n>, default: 1 (do not index terms shorter than n)
|
86
86
|
- term_splitter_count: <n>, default: 0 (see chapter Term Splitting)
|
87
|
-
-
|
88
|
-
-
|
87
|
+
- enabled_query_flags: <list of flags, separated by colons>
|
88
|
+
- disabled_query_flags: <list of flags, separated by colons>
|
89
89
|
|
90
90
|
The following query flags are enabled by default:
|
91
91
|
|
@@ -94,7 +94,7 @@ You can override these defaults by placing a config file named 'xapian_db.yml' i
|
|
94
94
|
- FLAG_BOOLEAN_ANY_CASE
|
95
95
|
- FLAG_SPELLING_CORRECTION
|
96
96
|
|
97
|
-
See the xapian docs for all available query flags
|
97
|
+
See the xapian docs for all available query flags; if you use the enabled_query_flags option, you must list all query flags that you want to enable since enabled_query_flags overwrites the defaults
|
98
98
|
|
99
99
|
If you do not configure settings for an environment in this file, xapian_db applies the defaults.
|
100
100
|
|
@@ -398,7 +398,7 @@ If you want to build a realtime search showing results while the user types, you
|
|
398
398
|
|
399
399
|
Since Xapian allows only one database instance to write to the index, the default setup of XapianDb will not work
|
400
400
|
with multiple app instances trying to write to the same database (you will get lock errors).
|
401
|
-
Therefore, XapianDb provides
|
401
|
+
Therefore, XapianDb provides three solutions based on queueing systems to overcome this. The first solution uses beanstalk, the second one uses resque and the third uses sidekiq.
|
402
402
|
|
403
403
|
== Installation with beanstalk
|
404
404
|
|
@@ -487,3 +487,27 @@ Be sure to specify the correct queue name when starting the worker.
|
|
487
487
|
|
488
488
|
<b>If you don't provide a queue name, it WON'T take 'xapian_db' by default! Do not start multiple
|
489
489
|
instances of this worker!</b>
|
490
|
+
|
491
|
+
== Installation with Sidekiq
|
492
|
+
|
493
|
+
=== 1. Install and start redis
|
494
|
+
|
495
|
+
Install and start redis as described on the {resque github page}[https://github.com/defunkt/resque].
|
496
|
+
|
497
|
+
=== 2. Add the sidekiq gem to your config
|
498
|
+
|
499
|
+
gem 'sidekiq'
|
500
|
+
bundle install
|
501
|
+
|
502
|
+
=== 3. Configure XapianDb to use sidekiq in production
|
503
|
+
|
504
|
+
production:
|
505
|
+
database: db/xapian_db/production
|
506
|
+
writer: sidekiq
|
507
|
+
sidekiq_queue: my_queue
|
508
|
+
|
509
|
+
If you don't specify a queue name XapianDb will use 'xapian_db' by default.
|
510
|
+
|
511
|
+
=== 4. Start sidekiq
|
512
|
+
|
513
|
+
RAILS_ENV=production bundle exec sidekiq
|
data/lib/xapian_db.rb
CHANGED
@@ -10,7 +10,7 @@ require 'xapian'
|
|
10
10
|
require 'json'
|
11
11
|
|
12
12
|
do_not_require = %w(update_stopwords railtie base_adapter generic_adapter active_record_adapter datamapper_adapter
|
13
|
-
beanstalk_writer resque_writer utilities install_generator datamapper)
|
13
|
+
beanstalk_writer resque_writer sidekiq_writer utilities install_generator datamapper)
|
14
14
|
files = Dir.glob("#{File.dirname(__FILE__)}/**/*.rb").reject{|path| do_not_require.include?(File.basename(path, ".rb"))}
|
15
15
|
# Require these first
|
16
16
|
require "#{File.dirname(__FILE__)}/xapian_db/utilities"
|
data/lib/xapian_db/config.rb
CHANGED
@@ -49,6 +49,10 @@ module XapianDb
|
|
49
49
|
@config.instance_variable_get("@_resque_queue") || 'xapian_db'
|
50
50
|
end
|
51
51
|
|
52
|
+
def sidekiq_queue
|
53
|
+
@config.instance_variable_get("@_sidekiq_queue") || 'xapian_db'
|
54
|
+
end
|
55
|
+
|
52
56
|
def term_min_length
|
53
57
|
@config.instance_variable_get("@_term_min_length") || 1
|
54
58
|
end
|
@@ -70,8 +74,8 @@ module XapianDb
|
|
70
74
|
# DSL methods
|
71
75
|
# ---------------------------------------------------------------------------------
|
72
76
|
|
73
|
-
attr_reader :_database, :_adapter, :_writer, :_beanstalk_daemon, :_resque_queue, :
|
74
|
-
:_term_min_length, :_term_splitter_count, :_enabled_query_flags
|
77
|
+
attr_reader :_database, :_adapter, :_writer, :_beanstalk_daemon, :_resque_queue, :_sidekiq_queue,
|
78
|
+
:_stemmer, :_stopper, :_term_min_length, :_term_splitter_count, :_enabled_query_flags
|
75
79
|
|
76
80
|
# Set the global database to use
|
77
81
|
# @param [String] path The path to the database. Either apply a file sytem path or :memory
|
@@ -115,6 +119,7 @@ module XapianDb
|
|
115
119
|
# - :direct ({XapianDb::IndexWriters::DirectWriter})
|
116
120
|
# - :beanstalk ({XapianDb::IndexWriters::BeanstalkWriter})
|
117
121
|
# - :resque ({XapianDb::IndexWriters::ResqueWriter})
|
122
|
+
# - :sidekiq ({XapianDb::IndexWriters::SidekiqWriter})
|
118
123
|
def writer(type)
|
119
124
|
# We try to guess the writer name
|
120
125
|
begin
|
@@ -138,6 +143,12 @@ module XapianDb
|
|
138
143
|
@_resque_queue = name
|
139
144
|
end
|
140
145
|
|
146
|
+
# Set the name of the sidekiq queue
|
147
|
+
# @param [String] name The name of the sidekiq queue
|
148
|
+
def sidekiq_queue(name)
|
149
|
+
@_sidekiq_queue = name
|
150
|
+
end
|
151
|
+
|
141
152
|
# Set the language.
|
142
153
|
# @param [Symbol] lang The language; apply the two letter ISO639 code for the language
|
143
154
|
# @example
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
module XapianDb
|
3
|
+
module IndexWriters
|
4
|
+
# Worker to update the Xapian index; the worker will be called by sidekiq
|
5
|
+
# and uses the DirectWriter to do the real work
|
6
|
+
# @author Michael Stämpfli and John Bradley
|
7
|
+
class SidekiqWorker
|
8
|
+
|
9
|
+
extend XapianDb::Utilities
|
10
|
+
|
11
|
+
APPROVED_TASKS = [:index, :delete_doc, :reindex_class]
|
12
|
+
|
13
|
+
def perform(task, options)
|
14
|
+
self.class.send(task, options) if APPROVED_TASKS.include?(task.to_sym)
|
15
|
+
end
|
16
|
+
|
17
|
+
class << self
|
18
|
+
def queue
|
19
|
+
XapianDb::Config.sidekiq_queue
|
20
|
+
end
|
21
|
+
|
22
|
+
def perform(task, options)
|
23
|
+
send(task, options) if APPROVED_TASKS.include?(task.to_sym)
|
24
|
+
end
|
25
|
+
|
26
|
+
def index(options)
|
27
|
+
klass = constantize options['class']
|
28
|
+
obj = klass.respond_to?('get') ? klass.get(options['id']) : klass.find(options['id'])
|
29
|
+
DirectWriter.index obj
|
30
|
+
end
|
31
|
+
|
32
|
+
def delete_doc(options)
|
33
|
+
DirectWriter.delete_doc_with options['xapian_id']
|
34
|
+
end
|
35
|
+
|
36
|
+
def reindex_class(options)
|
37
|
+
klass = constantize options['class']
|
38
|
+
DirectWriter.reindex_class klass, :verbose => false
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
# This writer uses sidekiq to enqueue index jobs
|
3
|
+
# @author Michael Stämpfli and John Bradley
|
4
|
+
|
5
|
+
require 'sidekiq'
|
6
|
+
|
7
|
+
module XapianDb
|
8
|
+
module IndexWriters
|
9
|
+
class SidekiqWriter
|
10
|
+
|
11
|
+
SidekiqWorker.class_eval do
|
12
|
+
include Sidekiq::Worker
|
13
|
+
end
|
14
|
+
|
15
|
+
class << self
|
16
|
+
|
17
|
+
# Update an object in the index
|
18
|
+
# @param [Object] obj An instance of a class with a blueprint configuration
|
19
|
+
def index(obj, commit=true)
|
20
|
+
Sidekiq::Client.enqueue worker_class, :index, :class => obj.class.name, :id => obj.id
|
21
|
+
end
|
22
|
+
|
23
|
+
# Remove an object from the index
|
24
|
+
# @param [String] xapian_id The document id
|
25
|
+
def delete_doc_with(xapian_id, commit=true)
|
26
|
+
Sidekiq::Client.enqueue worker_class, :delete_doc, :xapian_id => xapian_id
|
27
|
+
end
|
28
|
+
|
29
|
+
# Reindex all objects of a given class
|
30
|
+
# @param [Class] klass The class to reindex
|
31
|
+
def reindex_class(klass, options = {})
|
32
|
+
Sidekiq::Client.enqueue worker_class, :reindex_class, :class => klass.name
|
33
|
+
end
|
34
|
+
|
35
|
+
def worker_class
|
36
|
+
SidekiqWorker
|
37
|
+
end
|
38
|
+
private :worker_class
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
data/lib/xapian_db/railtie.rb
CHANGED
@@ -49,6 +49,7 @@ module XapianDb
|
|
49
49
|
config.writer @writer.try(:to_sym)
|
50
50
|
config.beanstalk_daemon_url @beanstalk_daemon
|
51
51
|
config.resque_queue @resque_queue
|
52
|
+
config.sidekiq_queue @sidekiq_queue
|
52
53
|
config.language @language.try(:to_sym)
|
53
54
|
config.term_min_length @term_min_length
|
54
55
|
config.term_splitter_count @term_splitter_count
|
@@ -74,6 +75,7 @@ module XapianDb
|
|
74
75
|
@writer = env_config["writer"] || :direct
|
75
76
|
@beanstalk_daemon_url = env_config["beanstalk_daemon"]
|
76
77
|
@resque_queue = env_config["resque_queue"]
|
78
|
+
@sidekiq_queue = env_config["sidekiq_queue"]
|
77
79
|
@language = env_config["language"]
|
78
80
|
@term_min_length = env_config["term_min_length"]
|
79
81
|
@enable_phrase_search = env_config["enable_phrase_search"] == true
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: xapian_db
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.3.
|
4
|
+
version: 1.3.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Gernot Kogler
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-
|
11
|
+
date: 2013-07-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: daemons
|
@@ -122,6 +122,20 @@ dependencies:
|
|
122
122
|
- - '>='
|
123
123
|
- !ruby/object:Gem::Version
|
124
124
|
version: 1.19.0
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: sidekiq
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - '>='
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: 2.13.0
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - '>='
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: 2.13.0
|
125
139
|
- !ruby/object:Gem::Dependency
|
126
140
|
name: xapian-ruby
|
127
141
|
requirement: !ruby/object:Gem::Requirement
|
@@ -173,6 +187,8 @@ files:
|
|
173
187
|
- lib/xapian_db/index_writers/no_op_writer.rb
|
174
188
|
- lib/xapian_db/index_writers/resque_worker.rb
|
175
189
|
- lib/xapian_db/index_writers/resque_writer.rb
|
190
|
+
- lib/xapian_db/index_writers/sidekiq_worker.rb
|
191
|
+
- lib/xapian_db/index_writers/sidekiq_writer.rb
|
176
192
|
- lib/xapian_db/index_writers/transactional_writer.rb
|
177
193
|
- lib/xapian_db/indexer.rb
|
178
194
|
- lib/xapian_db/model_extenders/active_record.rb
|