ts-resque-delta 1.2.4 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -1
  3. data/.travis.yml +14 -14
  4. data/Appraisals +4 -6
  5. data/Gemfile +1 -1
  6. data/README.markdown +31 -39
  7. data/Rakefile +6 -13
  8. data/gemfiles/binary.gemfile +7 -0
  9. data/gemfiles/sphinxql.gemfile +7 -0
  10. data/lib/thinking_sphinx/deltas/resque_delta.rb +66 -76
  11. data/lib/thinking_sphinx/deltas/resque_delta/delta_job.rb +1 -113
  12. data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_job.rb +7 -0
  13. data/lib/ts-resque-delta.rb +0 -2
  14. data/spec/acceptance/resque_deltas_spec.rb +50 -0
  15. data/spec/acceptance/spec_helper.rb +4 -0
  16. data/spec/acceptance/support/database_cleaner.rb +11 -0
  17. data/spec/acceptance/support/sphinx_controller.rb +61 -0
  18. data/spec/acceptance/support/sphinx_helpers.rb +36 -0
  19. data/spec/internal/.gitignore +2 -0
  20. data/spec/internal/app/indices/book_index.rb +3 -0
  21. data/spec/internal/app/models/book.rb +7 -0
  22. data/spec/internal/config/database.yml +5 -0
  23. data/spec/internal/db/schema.rb +10 -0
  24. data/spec/internal/log/.gitignore +1 -0
  25. data/spec/spec_helper.rb +16 -12
  26. data/spec/thinking_sphinx/deltas/resque_delta/delta_job_spec.rb +11 -155
  27. data/spec/thinking_sphinx/deltas/resque_delta_spec.rb +3 -166
  28. data/ts-resque-delta.gemspec +19 -28
  29. metadata +90 -230
  30. data/Guardfile +0 -17
  31. data/features/resque_deltas.feature +0 -62
  32. data/features/smart_indexing.feature +0 -43
  33. data/features/step_definitions/common_steps.rb +0 -76
  34. data/features/step_definitions/resque_delta_steps.rb +0 -33
  35. data/features/step_definitions/smart_indexing_steps.rb +0 -3
  36. data/features/support/env.rb +0 -41
  37. data/features/thinking_sphinx/database.example.yml +0 -4
  38. data/features/thinking_sphinx/db/migrations/create_delayed_betas.rb +0 -4
  39. data/features/thinking_sphinx/models/delayed_beta.rb +0 -6
  40. data/gemfiles/activerecord2.gemfile +0 -8
  41. data/gemfiles/activerecord2.gemfile.lock +0 -114
  42. data/gemfiles/activerecord3.gemfile +0 -8
  43. data/gemfiles/activerecord3.gemfile.lock +0 -123
  44. data/lib/flying_sphinx/resque_delta.rb +0 -35
  45. data/lib/flying_sphinx/resque_delta/delta_job.rb +0 -14
  46. data/lib/flying_sphinx/resque_delta/flag_as_deleted_job.rb +0 -7
  47. data/lib/thinking_sphinx/deltas/resque_delta/core_index.rb +0 -116
  48. data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set.rb +0 -59
  49. data/lib/thinking_sphinx/deltas/resque_delta/index_utils.rb +0 -47
  50. data/lib/thinking_sphinx/deltas/resque_delta/railtie.rb +0 -8
  51. data/lib/thinking_sphinx/deltas/resque_delta/tasks.rb +0 -38
  52. data/lib/thinking_sphinx/deltas/resque_delta/version.rb +0 -7
  53. data/spec/flying_sphinx/resque_delta/delta_job_spec.rb +0 -32
  54. data/spec/flying_sphinx/resque_delta/flag_as_deleted_job_spec.rb +0 -23
  55. data/spec/flying_sphinx/resque_delta_spec.rb +0 -130
  56. data/spec/thinking_sphinx/deltas/resque_delta/core_index_spec.rb +0 -210
  57. data/spec/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set_spec.rb +0 -126
  58. data/spec/thinking_sphinx/deltas/resque_delta/index_utils_spec.rb +0 -67
@@ -1,35 +0,0 @@
1
- require 'thinking_sphinx/deltas/resque_delta'
2
-
3
- class FlyingSphinx::ResqueDelta < ThinkingSphinx::Deltas::ResqueDelta
4
- def self.job_types
5
- [
6
- FlyingSphinx::ResqueDelta::DeltaJob,
7
- FlyingSphinx::ResqueDelta::FlagAsDeletedJob
8
- ]
9
- end
10
-
11
- def self.job_prefix
12
- 'fs-delta'
13
- end
14
-
15
- def index(model, instance = nil)
16
- return true if skip?(instance)
17
-
18
- model.delta_index_names.each do |delta|
19
- next if self.class.locked?(delta)
20
-
21
- Resque.enqueue FlyingSphinx::ResqueDelta::DeltaJob, delta
22
- end
23
-
24
- Resque.enqueue(
25
- FlyingSphinx::ResqueDelta::FlagAsDeletedJob,
26
- model.core_index_names,
27
- instance.sphinx_document_id
28
- ) if instance
29
-
30
- true
31
- end
32
- end
33
-
34
- require 'flying_sphinx/resque_delta/delta_job'
35
- require 'flying_sphinx/resque_delta/flag_as_deleted_job'
@@ -1,14 +0,0 @@
1
- class FlyingSphinx::ResqueDelta::DeltaJob < ThinkingSphinx::Deltas::ResqueDelta::DeltaJob
2
- @queue = :fs_delta
3
-
4
- # Runs Sphinx's indexer tool to process the index. Currently assumes Sphinx
5
- # is running.
6
- #
7
- # @param [String] index the name of the Sphinx index
8
- #
9
- def self.perform(indices)
10
- return if skip?(indices)
11
-
12
- FlyingSphinx::IndexRequest.new([indices]).perform
13
- end
14
- end
@@ -1,7 +0,0 @@
1
- class FlyingSphinx::ResqueDelta::FlagAsDeletedJob
2
- @queue = :fs_delta
3
-
4
- def self.perform(indices, document_id)
5
- FlyingSphinx::FlagAsDeletedJob.new(indices, document_id).perform
6
- end
7
- end
@@ -1,116 +0,0 @@
1
- class ThinkingSphinx::Deltas::ResqueDelta::CoreIndex
2
-
3
- def sphinx_indices
4
- unless @sphinx_indices
5
- @ts_config ||= ThinkingSphinx::Configuration.instance
6
- @ts_config.generate
7
- @sphinx_indices = @ts_config.configuration.indices.collect { |i| i.name }
8
- # The collected indices look like:
9
- # ["foo_core", "foo_delta", "foo", "bar_core", "bar_delta", "bar"]
10
- @sphinx_indices.reject! { |i| i =~ /_(core|delta)$/}
11
- # Now we have:
12
- # ["foo", "bar"]
13
- end
14
- @sphinx_indices
15
- end
16
-
17
- # Public: Lock a delta index against indexing or new index jobs.
18
- #
19
- # index_name - The String index prefix.
20
- #
21
- # Examples
22
- #
23
- # lock_delta('foo')
24
- #
25
- # Returns nothing.
26
- def lock_delta(index_name)
27
- ThinkingSphinx::Deltas::ResqueDelta.lock("#{index_name}_delta")
28
- end
29
-
30
- # Public: Unlock a delta index for indexing or new index jobs.
31
- #
32
- # index_name - The String index prefix.
33
- #
34
- # Examples
35
- #
36
- # unlock_delta('foo')
37
- #
38
- # Returns nothing.
39
- def unlock_delta(index_name)
40
- ThinkingSphinx::Deltas::ResqueDelta.unlock("#{index_name}_delta")
41
- end
42
-
43
- # Public: Lock all delta indexes against indexing or new index jobs.
44
- #
45
- # Returns nothing.
46
- def lock_deltas
47
- sphinx_indices.each { |index_name| lock_delta(index_name) }
48
- end
49
-
50
- # Public: Unlock all delta indexes for indexing or new index jobs.
51
- #
52
- # Returns nothing.
53
- def unlock_deltas
54
- sphinx_indices.each { |index_name| unlock_delta(index_name) }
55
- end
56
-
57
- # Public: Index all indices while locking each delta as we index the corresponding core index.
58
- #
59
- # Returns true on success; false on failure.
60
- def smart_index(opts = {})
61
- verbose = opts.fetch(:verbose, true)
62
- verbose = false if ENV['SILENT'] == 'true'
63
-
64
- # Load config like ts:in.
65
- unless ENV['INDEX_ONLY'] == 'true'
66
- puts "Generating Configuration to #{ts_config.config_file}" if verbose
67
- ts_config.build
68
- end
69
- FileUtils.mkdir_p(ts_config.searchd_file_path)
70
-
71
- # Index each core, one at a time. Wrap with delta locking logic.
72
- index_prefixes.each do |index_name|
73
- ret = nil
74
-
75
- with_delta_index_lock(index_name) do
76
- ThinkingSphinx::Deltas::ResqueDelta.prepare_for_core_index(index_name)
77
- ts_config.controller.index("#{index_name}_core", :verbose => verbose)
78
- ret = $?
79
- end
80
-
81
- return false if ret.to_i != 0
82
-
83
- Resque.enqueue(
84
- ThinkingSphinx::Deltas::ResqueDelta::DeltaJob,
85
- "#{index_name}_delta"
86
- )
87
- end
88
-
89
- true
90
- end
91
-
92
- # Public: Wraps the passed block with a delta index lock
93
- #
94
- # index_name - The String index prefix.
95
- #
96
- # Examples
97
- #
98
- # with_delta_index_lock('foo')
99
- #
100
- # Returns nothing.
101
- def with_delta_index_lock(index_name)
102
- lock_delta(index_name)
103
- yield
104
- unlock_delta(index_name)
105
- end
106
-
107
- private
108
-
109
- def ts_config
110
- ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.ts_config
111
- end
112
-
113
- def index_prefixes
114
- ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.index_prefixes
115
- end
116
- end
@@ -1,59 +0,0 @@
1
- class ThinkingSphinx::Deltas::ResqueDelta < ThinkingSphinx::Deltas::DefaultDelta
2
- module FlagAsDeletedSet
3
- extend self
4
-
5
- def set_name(core_name)
6
- "#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:flag.deleted:#{core_name}:set"
7
- end
8
-
9
- def temp_name(core_name)
10
- "#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:flag.deleted:#{core_name}:temp"
11
- end
12
-
13
- def processing_name(core_name)
14
- "#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:flag.deleted:#{core_name}:processing"
15
- end
16
-
17
- def add(core_name, document_id)
18
- Resque.redis.sadd(set_name(core_name), document_id)
19
- end
20
-
21
- def clear!(core_name)
22
- Resque.redis.del(set_name(core_name))
23
-
24
- #Clear processing set as well
25
- delta_name = ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.core_to_delta(core_name)
26
- ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.around_perform_lock(delta_name) do
27
- Resque.redis.del(processing_name(core_name))
28
- end
29
- end
30
-
31
- def clear_all!
32
- ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.core_indices.each do |core_index|
33
- clear!(core_index)
34
- end
35
- end
36
-
37
- def get_subset_for_processing(core_name)
38
- # Use a transaction to keep from losing set members if interrupted
39
- Resque.redis.multi do
40
- # Copy set to temp
41
- Resque.redis.sunionstore temp_name(core_name), set_name(core_name)
42
- # Store (set - temp) into set. This removes all items we copied into temp from set.
43
- Resque.redis.sdiffstore set_name(core_name), set_name(core_name), temp_name(core_name)
44
- # Merge processing and temp together and store into processing.
45
- Resque.redis.sunionstore processing_name(core_name), processing_name(core_name), temp_name(core_name)
46
-
47
- Resque.redis.del temp_name(core_name)
48
- end
49
- end
50
-
51
- def processing_members(core_name)
52
- Resque.redis.smembers(processing_name(core_name)).collect(&:to_i)
53
- end
54
-
55
- def clear_processing(core_name)
56
- Resque.redis.del(processing_name(core_name))
57
- end
58
- end
59
- end
@@ -1,47 +0,0 @@
1
- module ThinkingSphinx::Deltas::ResqueDelta::IndexUtils
2
- extend self
3
-
4
- # Public: Return a list of index prefixes (i.e. without "_core"/"_delta").
5
- #
6
- # Examples
7
- #
8
- # sphinx_indices
9
- # # => ['foo', 'bar']
10
- #
11
- # Returns an Array of index prefixes.
12
- def index_prefixes
13
- @prefixes ||= indices.reject { |i| i =~ /_(core|delta)$/ }
14
- end
15
-
16
- def core_indices
17
- @core_indices ||= indices.select { |i| i =~ /_core$/ }
18
- end
19
-
20
- def delta_indices
21
- @delta_indices ||= indices.select { |i| i =~ /_delta$/ }
22
- end
23
-
24
- def reload!
25
- @ts_config = @indices = @prefixes = @core_indices = @delta_indices = nil
26
- end
27
-
28
- def delta_to_core(delta_name)
29
- delta_name.sub(/_delta$/, '_core')
30
- end
31
-
32
- def core_to_delta(core_name)
33
- core_name.sub(/_core$/, '_delta')
34
- end
35
-
36
- def ts_config
37
- @ts_config ||= ThinkingSphinx::Configuration.instance.tap do |config|
38
- config.generate
39
- end
40
- end
41
-
42
- private
43
- def indices
44
- @indices ||= ts_config.configuration.indices.collect { |i| i.name }
45
- end
46
-
47
- end
@@ -1,8 +0,0 @@
1
- require 'rails'
2
-
3
- class ThinkingSphinx::Deltas::ResqueDelta::Railtie < Rails::Railtie
4
-
5
- rake_tasks do
6
- load File.expand_path('../tasks.rb', __FILE__)
7
- end
8
- end
@@ -1,38 +0,0 @@
1
- require 'thinking_sphinx/deltas/resque_delta'
2
-
3
- namespace :thinking_sphinx do
4
- desc 'Lock all delta indices (Resque will not run indexer or place new jobs on the :ts_delta queue).'
5
- task :lock_deltas do
6
- ThinkingSphinx::Deltas::ResqueDelta::CoreIndex.new.lock_deltas
7
- end
8
-
9
- desc 'Unlock all delta indices.'
10
- task :unlock_deltas do
11
- ThinkingSphinx::Deltas::ResqueDelta::CoreIndex.new.unlock_deltas
12
- end
13
-
14
- desc 'Like `rake thinking_sphinx:index`, but locks one index at a time.'
15
- task :smart_index => :app_env do
16
- ret = ThinkingSphinx::Deltas::ResqueDelta::CoreIndex.new.smart_index
17
-
18
- abort("Indexing failed.") if ret != true
19
- end
20
- end
21
-
22
- namespace :ts do
23
- desc 'Like `rake thinking_sphinx:index`, but locks one index at a time.'
24
- task :si => 'thinking_sphinx:smart_index'
25
- end
26
-
27
- unless Rake::Task.task_defined?('thinking_sphinx:index')
28
- require 'thinking_sphinx/tasks'
29
- end
30
-
31
- # Ensure that indexing does not conflict with ts-resque-delta delta jobs.
32
- Rake::Task['thinking_sphinx:index'].enhance ['thinking_sphinx:lock_deltas'] do
33
- Rake::Task['thinking_sphinx:unlock_deltas'].invoke
34
- end
35
-
36
- Rake::Task['thinking_sphinx:reindex'].enhance ['thinking_sphinx:lock_deltas'] do
37
- Rake::Task['thinking_sphinx:unlock_deltas'].invoke
38
- end
@@ -1,7 +0,0 @@
1
- module ThinkingSphinx
2
- module Deltas
3
- class ResqueDeltaInfo
4
- VERSION = '1.2.4'
5
- end
6
- end
7
- end
@@ -1,32 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FlyingSphinx::ResqueDelta::DeltaJob do
4
- describe '@queue' do
5
- it "uses the fs_delta queue" do
6
- FlyingSphinx::ResqueDelta::DeltaJob.instance_variable_get(:@queue).
7
- should == :fs_delta
8
- end
9
- end
10
-
11
- describe '.perform' do
12
- it "doesn't create an index request when skipping" do
13
- FlyingSphinx::ResqueDelta::DeltaJob.stub!(:skip? => true)
14
-
15
- FlyingSphinx::IndexRequest.should_not_receive(:new)
16
-
17
- FlyingSphinx::ResqueDelta::DeltaJob.perform 'foo_delta'
18
- end
19
-
20
- it "performs an index request when not skipping" do
21
- request = double('index request', :perform => true)
22
- FlyingSphinx::ResqueDelta::DeltaJob.stub!(:skip? => false)
23
-
24
- FlyingSphinx::IndexRequest.should_receive(:new).
25
- with(['foo_delta']).
26
- and_return(request)
27
- request.should_receive(:perform)
28
-
29
- FlyingSphinx::ResqueDelta::DeltaJob.perform 'foo_delta'
30
- end
31
- end
32
- end
@@ -1,23 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FlyingSphinx::ResqueDelta::FlagAsDeletedJob do
4
- describe '@queue' do
5
- it "uses the fs_delta queue" do
6
- FlyingSphinx::ResqueDelta::FlagAsDeletedJob.
7
- instance_variable_get(:@queue).should == :fs_delta
8
- end
9
- end
10
-
11
- describe '.perform' do
12
- it "performs a flag-as-deleted job" do
13
- job = double('flag as deleted job', :perform => true)
14
-
15
- FlyingSphinx::FlagAsDeletedJob.should_receive(:new).
16
- with(['foo_core'], 5).
17
- and_return(job)
18
- job.should_receive(:perform)
19
-
20
- FlyingSphinx::ResqueDelta::FlagAsDeletedJob.perform ['foo_core'], 5
21
- end
22
- end
23
- end
@@ -1,130 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FlyingSphinx::ResqueDelta do
4
- describe '.job_types' do
5
- it "contains just the Flying Sphinx delta and delete jobs" do
6
- FlyingSphinx::ResqueDelta.job_types.should == [
7
- FlyingSphinx::ResqueDelta::DeltaJob,
8
- FlyingSphinx::ResqueDelta::FlagAsDeletedJob
9
- ]
10
- end
11
- end
12
-
13
- describe '.job_prefix' do
14
- it "is fs-delta" do
15
- FlyingSphinx::ResqueDelta.job_prefix.should == 'fs-delta'
16
- end
17
- end
18
-
19
- describe '#index' do
20
- before :each do
21
- ThinkingSphinx.updates_enabled = true
22
- ThinkingSphinx.deltas_enabled = true
23
-
24
- Resque.stub(:enqueue => true)
25
-
26
- @delayed_delta = FlyingSphinx::ResqueDelta.new(
27
- stub('instance'), {}
28
- )
29
- @delayed_delta.stub(:toggled).and_return(true)
30
-
31
- FlyingSphinx::ResqueDelta.stub(:lock)
32
- FlyingSphinx::ResqueDelta.stub(:unlock)
33
- FlyingSphinx::ResqueDelta.stub(:locked?).and_return(false)
34
-
35
- @model = stub('foo')
36
- @model.stub(:core_index_names => ['foo_core'])
37
- @model.stub(:delta_index_names => ['foo_delta'])
38
-
39
- @instance = stub('instance')
40
- @instance.stub(:sphinx_document_id => 42)
41
- end
42
-
43
- context 'updates disabled' do
44
- before :each do
45
- ThinkingSphinx.updates_enabled = false
46
- end
47
-
48
- it "should not enqueue a delta job" do
49
- Resque.should_not_receive(:enqueue)
50
- @delayed_delta.index(@model)
51
- end
52
-
53
- it "should not enqueue a flag as deleted job" do
54
- Resque.should_not_receive(:enqueue)
55
- @delayed_delta.index(@model)
56
- end
57
- end
58
-
59
- context 'deltas disabled' do
60
- before :each do
61
- ThinkingSphinx.deltas_enabled = false
62
- end
63
-
64
- it "should not enqueue a delta job" do
65
- Resque.should_not_receive(:enqueue)
66
- @delayed_delta.index(@model)
67
- end
68
-
69
- it "should not enqueue a flag as deleted job" do
70
- Resque.should_not_receive(:enqueue)
71
- @delayed_delta.index(@model)
72
- end
73
- end
74
-
75
- context "instance isn't toggled" do
76
- before :each do
77
- @delayed_delta.stub(:toggled => false)
78
- end
79
-
80
- it "should not enqueue a delta job" do
81
- Resque.should_not_receive(:enqueue)
82
- @delayed_delta.index(@model, @instance)
83
- end
84
-
85
- it "should not enqueue a flag as deleted job" do
86
- Resque.should_not_receive(:enqueue)
87
- @delayed_delta.index(@model, @instance)
88
- end
89
- end
90
-
91
- it "should enqueue a delta job" do
92
- Resque.should_receive(:enqueue).at_least(:once).with(
93
- FlyingSphinx::ResqueDelta::DeltaJob, 'foo_delta'
94
- )
95
- @delayed_delta.index(@model)
96
- end
97
-
98
- it "should enqueue a flag-as-deleted job" do
99
- Resque.should_receive(:enqueue).at_least(:once).with(
100
- FlyingSphinx::ResqueDelta::FlagAsDeletedJob,
101
- ['foo_core'],
102
- 42
103
- )
104
- @delayed_delta.index(@model, @instance)
105
- end
106
-
107
- context "delta index is locked" do
108
- before :each do
109
- FlyingSphinx::ResqueDelta.stub(:locked?).and_return(true)
110
- end
111
-
112
- it "should not enqueue a delta job" do
113
- Resque.should_not_receive(:enqueue).with(
114
- FlyingSphinx::ResqueDelta::DeltaJob,
115
- ['foo_delta']
116
- )
117
- @delayed_delta.index(@model, @instance)
118
- end
119
-
120
- it "should enqueue a flag-as-deleted job" do
121
- Resque.should_receive(:enqueue).at_least(:once).with(
122
- FlyingSphinx::ResqueDelta::FlagAsDeletedJob,
123
- ['foo_core'],
124
- 42
125
- )
126
- @delayed_delta.index(@model, @instance)
127
- end
128
- end
129
- end
130
- end