flying-sphinx 0.8.5 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +1 -1
- data/HISTORY +7 -0
- data/flying-sphinx.gemspec +4 -4
- data/lib/flying_sphinx.rb +36 -10
- data/lib/flying_sphinx/action.rb +90 -0
- data/lib/flying_sphinx/api.rb +16 -19
- data/lib/flying_sphinx/binary.rb +7 -0
- data/lib/flying_sphinx/binary/translator.rb +48 -0
- data/lib/flying_sphinx/cli.rb +36 -29
- data/lib/flying_sphinx/configuration.rb +16 -29
- data/lib/flying_sphinx/controller.rb +91 -0
- data/lib/flying_sphinx/rails.rb +1 -11
- data/lib/flying_sphinx/railtie.rb +4 -10
- data/lib/flying_sphinx/setting_files.rb +8 -7
- data/lib/flying_sphinx/sinatra.rb +4 -9
- data/lib/flying_sphinx/sphinxql.rb +7 -0
- data/lib/flying_sphinx/sphinxql/translator.rb +26 -0
- data/lib/flying_sphinx/tasks.rb +5 -8
- data/lib/flying_sphinx/version.rb +1 -1
- data/spec/specs/action_spec.rb +101 -0
- data/spec/specs/api_spec.rb +13 -7
- data/spec/specs/configuration_spec.rb +10 -18
- data/spec/specs/setting_files_spec.rb +27 -31
- metadata +198 -187
- data/lib/flying_sphinx/delayed_delta.rb +0 -89
- data/lib/flying_sphinx/flag_as_deleted_job.rb +0 -41
- data/lib/flying_sphinx/heroku_shared_adapter.rb +0 -5
- data/lib/flying_sphinx/index_request.rb +0 -110
- data/lib/flying_sphinx/sphinx_configuration.rb +0 -30
- data/spec/specs/delayed_delta_spec.rb +0 -147
- data/spec/specs/flag_as_deleted_job_spec.rb +0 -45
- data/spec/specs/index_request_spec.rb +0 -172
- data/spec/specs/sphinx_configuration_spec.rb +0 -42
@@ -1,89 +0,0 @@
|
|
1
|
-
class FlyingSphinx::DelayedDelta < ThinkingSphinx::Deltas::DefaultDelta
|
2
|
-
# Adds a job to the queue, if it doesn't already exist. This is to ensure
|
3
|
-
# multiple indexing requests for the same delta index don't get added, as the
|
4
|
-
# index only needs to be processed once.
|
5
|
-
#
|
6
|
-
# Because indexing jobs are all the same object, they all get serialised to
|
7
|
-
# the same YAML value.
|
8
|
-
#
|
9
|
-
# @param [Object] object The job, which must respond to the #perform method.
|
10
|
-
# @param [Integer] priority (0)
|
11
|
-
#
|
12
|
-
def self.enqueue(object, priority = 0)
|
13
|
-
return if duplicates_exist? object
|
14
|
-
|
15
|
-
enqueue_without_duplicates_check object, priority
|
16
|
-
end
|
17
|
-
|
18
|
-
def self.enqueue_without_duplicates_check(object, priority = 0)
|
19
|
-
if defined?(Rails) && Rails.version.to_i <= 2
|
20
|
-
::Delayed::Job.enqueue(object, priority)
|
21
|
-
else
|
22
|
-
::Delayed::Job.enqueue(object, :priority => priority)
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
# Checks whether a given job already exists in the queue.
|
27
|
-
#
|
28
|
-
# @param [Object] object The job
|
29
|
-
# @return [Boolean] True if a duplicate of the job already exists in the queue
|
30
|
-
#
|
31
|
-
def self.duplicates_exist?(object)
|
32
|
-
::Delayed::Job.count(
|
33
|
-
:conditions => {
|
34
|
-
:handler => object.to_yaml,
|
35
|
-
:locked_at => nil
|
36
|
-
}
|
37
|
-
) > 0
|
38
|
-
end
|
39
|
-
|
40
|
-
# Adds a job to the queue for processing the given model's delta index. A job
|
41
|
-
# for hiding the instance in the core index is also created, if an instance is
|
42
|
-
# provided.
|
43
|
-
#
|
44
|
-
# Neither job will be queued if updates or deltas are disabled, or if the
|
45
|
-
# instance (when given) is not toggled to be in the delta index. The first two
|
46
|
-
# options are controlled via ThinkingSphinx.updates_enabled? and
|
47
|
-
# ThinkingSphinx.deltas_enabled?.
|
48
|
-
#
|
49
|
-
# @param [Class] model the ActiveRecord model to index.
|
50
|
-
# @param [ActiveRecord::Base] instance the instance of the given model that
|
51
|
-
# has changed. Optional.
|
52
|
-
# @return [Boolean] true
|
53
|
-
#
|
54
|
-
def index(model, instance = nil)
|
55
|
-
return true if skip? instance
|
56
|
-
|
57
|
-
self.class.enqueue(
|
58
|
-
FlyingSphinx::IndexRequest.new(model.delta_index_names, true),
|
59
|
-
delayed_job_priority
|
60
|
-
)
|
61
|
-
|
62
|
-
self.class.enqueue_without_duplicates_check(
|
63
|
-
FlyingSphinx::FlagAsDeletedJob.new(
|
64
|
-
model.core_index_names, instance.sphinx_document_id
|
65
|
-
),
|
66
|
-
delayed_job_priority
|
67
|
-
) if instance
|
68
|
-
|
69
|
-
true
|
70
|
-
end
|
71
|
-
|
72
|
-
private
|
73
|
-
|
74
|
-
def delayed_job_priority
|
75
|
-
ThinkingSphinx::Configuration.instance.delayed_job_priority
|
76
|
-
end
|
77
|
-
|
78
|
-
# Checks whether jobs should be enqueued. Only true if updates and deltas are
|
79
|
-
# enabled, and the instance (if there is one) is toggled.
|
80
|
-
#
|
81
|
-
# @param [ActiveRecord::Base, NilClass] instance
|
82
|
-
# @return [Boolean]
|
83
|
-
#
|
84
|
-
def skip?(instance)
|
85
|
-
!ThinkingSphinx.updates_enabled? ||
|
86
|
-
!ThinkingSphinx.deltas_enabled? ||
|
87
|
-
(instance && !toggled(instance))
|
88
|
-
end
|
89
|
-
end
|
@@ -1,41 +0,0 @@
|
|
1
|
-
# A simple job for flagging a specified Sphinx document in a given index as
|
2
|
-
# 'deleted'.
|
3
|
-
#
|
4
|
-
class FlyingSphinx::FlagAsDeletedJob
|
5
|
-
attr_accessor :indices, :document_id
|
6
|
-
|
7
|
-
# Initialises the object with an index name and document id. Please note that
|
8
|
-
# the document id is Sphinx's unique identifier, and will almost certainly not
|
9
|
-
# be the model instance's primary key value.
|
10
|
-
#
|
11
|
-
# @param [String] index The index name
|
12
|
-
# @param [Integer] document_id The document id
|
13
|
-
#
|
14
|
-
def initialize(indices, document_id)
|
15
|
-
@indices, @document_id = indices, document_id
|
16
|
-
end
|
17
|
-
|
18
|
-
# Updates the sphinx_deleted attribute for the given document, setting the
|
19
|
-
# value to 1 (true). This is not a special attribute in Sphinx, but is used
|
20
|
-
# by Thinking Sphinx to ignore deleted values between full re-indexing. It's
|
21
|
-
# particularly useful in this situation to avoid old values in the core index
|
22
|
-
# and just use the new values in the delta index as a reference point.
|
23
|
-
#
|
24
|
-
# @return [Boolean] true
|
25
|
-
#
|
26
|
-
def perform
|
27
|
-
indices.each do |index|
|
28
|
-
client.update index, ['sphinx_deleted'], {@document_id => [1]}
|
29
|
-
end
|
30
|
-
rescue Riddle::ConnectionError
|
31
|
-
# If it fails here, so be it.
|
32
|
-
ensure
|
33
|
-
true
|
34
|
-
end
|
35
|
-
|
36
|
-
private
|
37
|
-
|
38
|
-
def client
|
39
|
-
@client ||= ThinkingSphinx::Configuration.instance.client
|
40
|
-
end
|
41
|
-
end
|
@@ -1,110 +0,0 @@
|
|
1
|
-
class FlyingSphinx::IndexRequest
|
2
|
-
attr_reader :index_id, :indices, :async
|
3
|
-
|
4
|
-
INDEX_COMPLETE_CHECKING_INTERVAL = 3
|
5
|
-
|
6
|
-
# Remove all Delta jobs from the queue. If the
|
7
|
-
# delayed_jobs table does not exist, this method will do nothing.
|
8
|
-
#
|
9
|
-
def self.cancel_jobs
|
10
|
-
return unless defined?(::Delayed) && ::Delayed::Job.table_exists?
|
11
|
-
|
12
|
-
::Delayed::Job.delete_all "handler LIKE '--- !ruby/object:FlyingSphinx::%'"
|
13
|
-
end
|
14
|
-
|
15
|
-
def self.output_last_index
|
16
|
-
index = FlyingSphinx::Configuration.new.api.get('indices/last').body
|
17
|
-
puts "Index Job Status: #{index.status}"
|
18
|
-
puts "Index Log:\n#{index.log}"
|
19
|
-
end
|
20
|
-
|
21
|
-
def initialize(indices = [], async = false)
|
22
|
-
@indices, @async = indices, async
|
23
|
-
end
|
24
|
-
|
25
|
-
# Shows index name in Delayed::Job#name.
|
26
|
-
#
|
27
|
-
def display_name
|
28
|
-
"#{self.class.name} for #{indices.join(', ')}"
|
29
|
-
end
|
30
|
-
|
31
|
-
def index
|
32
|
-
begin_request
|
33
|
-
return if async
|
34
|
-
|
35
|
-
while !request_complete?
|
36
|
-
sleep 3
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
def status_message
|
41
|
-
raise "Index Request failed to start. Something's not right!" if @index_id.nil?
|
42
|
-
|
43
|
-
status = request_status
|
44
|
-
case status
|
45
|
-
when 'FINISHED'
|
46
|
-
"Index Request has completed:\n#{request_log}"
|
47
|
-
when 'FAILED'
|
48
|
-
'Index Request failed.'
|
49
|
-
when 'PENDING'
|
50
|
-
'Index Request is still pending - something has gone wrong.'
|
51
|
-
else
|
52
|
-
"Unknown index response: '#{status}'."
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
# Runs Sphinx's indexer tool to process the index. Currently assumes Sphinx is
|
57
|
-
# running.
|
58
|
-
#
|
59
|
-
# @return [Boolean] true
|
60
|
-
#
|
61
|
-
def perform
|
62
|
-
index
|
63
|
-
true
|
64
|
-
end
|
65
|
-
|
66
|
-
private
|
67
|
-
|
68
|
-
def configuration
|
69
|
-
@configuration ||= FlyingSphinx::Configuration.new
|
70
|
-
end
|
71
|
-
|
72
|
-
def begin_request
|
73
|
-
path = 'indices'
|
74
|
-
path << '/unique' if async
|
75
|
-
response = api.post path, :indices => indices.join(',')
|
76
|
-
|
77
|
-
@index_id = response.body.id
|
78
|
-
@request_begun = true
|
79
|
-
|
80
|
-
raise RuntimeError, 'Your account does not support delta indexing. Upgrading plans is probably the best way around this.' if response.body.status == 'BLOCKED'
|
81
|
-
end
|
82
|
-
|
83
|
-
def request_begun?
|
84
|
-
@request_begun
|
85
|
-
end
|
86
|
-
|
87
|
-
def request_complete?
|
88
|
-
case request_status
|
89
|
-
when 'FINISHED', 'FAILED'
|
90
|
-
true
|
91
|
-
when 'PENDING'
|
92
|
-
false
|
93
|
-
else
|
94
|
-
raise "Unknown index response: '#{response.body}'"
|
95
|
-
end
|
96
|
-
end
|
97
|
-
|
98
|
-
def request_log
|
99
|
-
@request.log
|
100
|
-
end
|
101
|
-
|
102
|
-
def request_status
|
103
|
-
@request = api.get("indices/#{index_id}").body
|
104
|
-
@request.status
|
105
|
-
end
|
106
|
-
|
107
|
-
def api
|
108
|
-
configuration.api
|
109
|
-
end
|
110
|
-
end
|
@@ -1,30 +0,0 @@
|
|
1
|
-
class FlyingSphinx::SphinxConfiguration
|
2
|
-
def initialize(thinking_sphinx = ThinkingSphinx::Configuration.instance)
|
3
|
-
@thinking_sphinx = thinking_sphinx
|
4
|
-
end
|
5
|
-
|
6
|
-
def upload_to(api)
|
7
|
-
api.put '/',
|
8
|
-
:configuration => content,
|
9
|
-
:sphinx_version => thinking_sphinx.version
|
10
|
-
end
|
11
|
-
|
12
|
-
def upload_file_to(api, path)
|
13
|
-
api.put '/',
|
14
|
-
:configuration => File.read(path),
|
15
|
-
:sphinx_version => '2.0.4'
|
16
|
-
end
|
17
|
-
|
18
|
-
private
|
19
|
-
|
20
|
-
attr_reader :thinking_sphinx
|
21
|
-
|
22
|
-
def content
|
23
|
-
@content ||= begin
|
24
|
-
thinking_sphinx.generate
|
25
|
-
thinking_sphinx.configuration.searchd.client_key =
|
26
|
-
FlyingSphinx::Configuration.new.client_key
|
27
|
-
thinking_sphinx.configuration.render
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
@@ -1,147 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FlyingSphinx::DelayedDelta do
|
4
|
-
describe '.enqueue' do
|
5
|
-
before :each do
|
6
|
-
Delayed::Job.stub!(:count => 0)
|
7
|
-
end
|
8
|
-
|
9
|
-
it "should enqueue if there's no existing jobs for the same index" do
|
10
|
-
Delayed::Job.should_receive(:enqueue)
|
11
|
-
|
12
|
-
FlyingSphinx::DelayedDelta.enqueue(stub('object'))
|
13
|
-
end
|
14
|
-
|
15
|
-
it "should not enqueue the job if there's an existing job already" do
|
16
|
-
Delayed::Job.stub!(:count => 1)
|
17
|
-
Delayed::Job.should_not_receive(:enqueue)
|
18
|
-
|
19
|
-
FlyingSphinx::DelayedDelta.enqueue(stub('object'))
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
describe '#index' do
|
24
|
-
let(:config) { ThinkingSphinx::Configuration.instance }
|
25
|
-
let(:delayed_delta) { FlyingSphinx::DelayedDelta.new stub('instance'), {} }
|
26
|
-
let(:model) {
|
27
|
-
stub 'foo',
|
28
|
-
:name => 'foo',
|
29
|
-
:core_index_names => ['foo_core'],
|
30
|
-
:delta_index_names => ['foo_delta']
|
31
|
-
}
|
32
|
-
let(:instance) { stub('instance', :sphinx_document_id => 42) }
|
33
|
-
|
34
|
-
before :each do
|
35
|
-
ThinkingSphinx.updates_enabled = true
|
36
|
-
ThinkingSphinx.deltas_enabled = true
|
37
|
-
|
38
|
-
config.delayed_job_priority = 2
|
39
|
-
|
40
|
-
FlyingSphinx::DelayedDelta.stub!(:enqueue => true)
|
41
|
-
Delayed::Job.stub!(:enqueue => true, :inspect => 'Delayed::Job')
|
42
|
-
|
43
|
-
delayed_delta.stub!(:toggled => true)
|
44
|
-
end
|
45
|
-
|
46
|
-
context 'updates disabled' do
|
47
|
-
before :each do
|
48
|
-
ThinkingSphinx.updates_enabled = false
|
49
|
-
end
|
50
|
-
|
51
|
-
it "should not enqueue a delta job" do
|
52
|
-
FlyingSphinx::DelayedDelta.should_not_receive(:enqueue)
|
53
|
-
|
54
|
-
delayed_delta.index model
|
55
|
-
end
|
56
|
-
|
57
|
-
it "should not enqueue a flag as deleted job" do
|
58
|
-
Delayed::Job.should_not_receive(:enqueue)
|
59
|
-
|
60
|
-
delayed_delta.index model
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
|
-
context 'deltas disabled' do
|
65
|
-
before :each do
|
66
|
-
ThinkingSphinx.deltas_enabled = false
|
67
|
-
end
|
68
|
-
|
69
|
-
it "should not enqueue a delta job" do
|
70
|
-
FlyingSphinx::DelayedDelta.should_not_receive(:enqueue)
|
71
|
-
|
72
|
-
delayed_delta.index model
|
73
|
-
end
|
74
|
-
|
75
|
-
it "should not enqueue a flag as deleted job" do
|
76
|
-
Delayed::Job.should_not_receive(:enqueue)
|
77
|
-
|
78
|
-
delayed_delta.index model
|
79
|
-
end
|
80
|
-
end
|
81
|
-
|
82
|
-
context "instance isn't toggled" do
|
83
|
-
before :each do
|
84
|
-
delayed_delta.stub!(:toggled => false)
|
85
|
-
end
|
86
|
-
|
87
|
-
it "should not enqueue a delta job" do
|
88
|
-
FlyingSphinx::DelayedDelta.should_not_receive(:enqueue)
|
89
|
-
|
90
|
-
delayed_delta.index model, instance
|
91
|
-
end
|
92
|
-
|
93
|
-
it "should not enqueue a flag as deleted job" do
|
94
|
-
Delayed::Job.should_not_receive(:enqueue)
|
95
|
-
|
96
|
-
delayed_delta.index model, instance
|
97
|
-
end
|
98
|
-
end
|
99
|
-
|
100
|
-
it "should enqueue a delta job for the appropriate indexes" do
|
101
|
-
FlyingSphinx::DelayedDelta.should_receive(:enqueue) do |job, priority|
|
102
|
-
job.indices.should == ['foo_delta']
|
103
|
-
job.async.should be_true
|
104
|
-
end
|
105
|
-
|
106
|
-
delayed_delta.index model
|
107
|
-
end
|
108
|
-
|
109
|
-
it "should use the defined priority for the delta job" do
|
110
|
-
FlyingSphinx::DelayedDelta.should_receive(:enqueue) do |job, priority|
|
111
|
-
priority.should == 2
|
112
|
-
end
|
113
|
-
|
114
|
-
delayed_delta.index model
|
115
|
-
end
|
116
|
-
|
117
|
-
it "should enqueue a flag-as-deleted job for the appropriate indexes" do
|
118
|
-
Delayed::Job.should_receive(:enqueue) do |job, options|
|
119
|
-
job.indices.should == ['foo_core']
|
120
|
-
end
|
121
|
-
|
122
|
-
delayed_delta.index model, instance
|
123
|
-
end
|
124
|
-
|
125
|
-
it "should enqueue a flag-as-deleted job for the appropriate id" do
|
126
|
-
Delayed::Job.should_receive(:enqueue) do |job, options|
|
127
|
-
job.document_id.should == 42
|
128
|
-
end
|
129
|
-
|
130
|
-
delayed_delta.index model, instance
|
131
|
-
end
|
132
|
-
|
133
|
-
it "should use the defined priority for the flag-as-deleted job" do
|
134
|
-
Delayed::Job.should_receive(:enqueue) do |job, options|
|
135
|
-
options[:priority].should == 2
|
136
|
-
end
|
137
|
-
|
138
|
-
delayed_delta.index model, instance
|
139
|
-
end
|
140
|
-
|
141
|
-
it "should not enqueue a flag-as-deleted job if no instance is provided" do
|
142
|
-
Delayed::Job.should_not_receive(:enqueue)
|
143
|
-
|
144
|
-
delayed_delta.index model
|
145
|
-
end
|
146
|
-
end
|
147
|
-
end
|
@@ -1,45 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FlyingSphinx::FlagAsDeletedJob do
|
4
|
-
describe '#perform' do
|
5
|
-
let(:config) { ThinkingSphinx::Configuration.instance }
|
6
|
-
let(:client) { stub('client', :update => true) }
|
7
|
-
let(:job) { FlyingSphinx::FlagAsDeletedJob.new(['foo_core'], 12) }
|
8
|
-
|
9
|
-
before :each do
|
10
|
-
config.stub!(:client => client)
|
11
|
-
end
|
12
|
-
|
13
|
-
it "should update the specified index" do
|
14
|
-
client.should_receive(:update) do |index, attributes, values|
|
15
|
-
index.should == 'foo_core'
|
16
|
-
end
|
17
|
-
|
18
|
-
job.perform
|
19
|
-
end
|
20
|
-
|
21
|
-
it "should update all specified indexes" do
|
22
|
-
job.indices = ['foo_core', 'bar_core']
|
23
|
-
client.should_receive(:update).with('foo_core', anything, anything)
|
24
|
-
client.should_receive(:update).with('bar_core', anything, anything)
|
25
|
-
|
26
|
-
job.perform
|
27
|
-
end
|
28
|
-
|
29
|
-
it "should update the sphinx_deleted attribute" do
|
30
|
-
client.should_receive(:update) do |index, attributes, values|
|
31
|
-
attributes.should == ['sphinx_deleted']
|
32
|
-
end
|
33
|
-
|
34
|
-
job.perform
|
35
|
-
end
|
36
|
-
|
37
|
-
it "should set sphinx_deleted for the given document to true" do
|
38
|
-
client.should_receive(:update) do |index, attributes, values|
|
39
|
-
values[12].should == [1]
|
40
|
-
end
|
41
|
-
|
42
|
-
job.perform
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|