ts-resque-delta 1.2.4 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (58) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -1
  3. data/.travis.yml +14 -14
  4. data/Appraisals +4 -6
  5. data/Gemfile +1 -1
  6. data/README.markdown +31 -39
  7. data/Rakefile +6 -13
  8. data/gemfiles/binary.gemfile +7 -0
  9. data/gemfiles/sphinxql.gemfile +7 -0
  10. data/lib/thinking_sphinx/deltas/resque_delta.rb +66 -76
  11. data/lib/thinking_sphinx/deltas/resque_delta/delta_job.rb +1 -113
  12. data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_job.rb +7 -0
  13. data/lib/ts-resque-delta.rb +0 -2
  14. data/spec/acceptance/resque_deltas_spec.rb +50 -0
  15. data/spec/acceptance/spec_helper.rb +4 -0
  16. data/spec/acceptance/support/database_cleaner.rb +11 -0
  17. data/spec/acceptance/support/sphinx_controller.rb +61 -0
  18. data/spec/acceptance/support/sphinx_helpers.rb +36 -0
  19. data/spec/internal/.gitignore +2 -0
  20. data/spec/internal/app/indices/book_index.rb +3 -0
  21. data/spec/internal/app/models/book.rb +7 -0
  22. data/spec/internal/config/database.yml +5 -0
  23. data/spec/internal/db/schema.rb +10 -0
  24. data/spec/internal/log/.gitignore +1 -0
  25. data/spec/spec_helper.rb +16 -12
  26. data/spec/thinking_sphinx/deltas/resque_delta/delta_job_spec.rb +11 -155
  27. data/spec/thinking_sphinx/deltas/resque_delta_spec.rb +3 -166
  28. data/ts-resque-delta.gemspec +19 -28
  29. metadata +90 -230
  30. data/Guardfile +0 -17
  31. data/features/resque_deltas.feature +0 -62
  32. data/features/smart_indexing.feature +0 -43
  33. data/features/step_definitions/common_steps.rb +0 -76
  34. data/features/step_definitions/resque_delta_steps.rb +0 -33
  35. data/features/step_definitions/smart_indexing_steps.rb +0 -3
  36. data/features/support/env.rb +0 -41
  37. data/features/thinking_sphinx/database.example.yml +0 -4
  38. data/features/thinking_sphinx/db/migrations/create_delayed_betas.rb +0 -4
  39. data/features/thinking_sphinx/models/delayed_beta.rb +0 -6
  40. data/gemfiles/activerecord2.gemfile +0 -8
  41. data/gemfiles/activerecord2.gemfile.lock +0 -114
  42. data/gemfiles/activerecord3.gemfile +0 -8
  43. data/gemfiles/activerecord3.gemfile.lock +0 -123
  44. data/lib/flying_sphinx/resque_delta.rb +0 -35
  45. data/lib/flying_sphinx/resque_delta/delta_job.rb +0 -14
  46. data/lib/flying_sphinx/resque_delta/flag_as_deleted_job.rb +0 -7
  47. data/lib/thinking_sphinx/deltas/resque_delta/core_index.rb +0 -116
  48. data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set.rb +0 -59
  49. data/lib/thinking_sphinx/deltas/resque_delta/index_utils.rb +0 -47
  50. data/lib/thinking_sphinx/deltas/resque_delta/railtie.rb +0 -8
  51. data/lib/thinking_sphinx/deltas/resque_delta/tasks.rb +0 -38
  52. data/lib/thinking_sphinx/deltas/resque_delta/version.rb +0 -7
  53. data/spec/flying_sphinx/resque_delta/delta_job_spec.rb +0 -32
  54. data/spec/flying_sphinx/resque_delta/flag_as_deleted_job_spec.rb +0 -23
  55. data/spec/flying_sphinx/resque_delta_spec.rb +0 -130
  56. data/spec/thinking_sphinx/deltas/resque_delta/core_index_spec.rb +0 -210
  57. data/spec/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set_spec.rb +0 -126
  58. data/spec/thinking_sphinx/deltas/resque_delta/index_utils_spec.rb +0 -67
data/Guardfile DELETED
@@ -1,17 +0,0 @@
1
- guard 'bundler' do
2
- watch('Gemfile')
3
- watch(/^.+\.gemspec/)
4
- end
5
-
6
- guard 'rspec', :version => 2, :cli => "-c --format progress", :all_on_start => false do
7
- watch(%r{^spec/.+_spec\.rb$})
8
- watch(%r{^lib/(.+)\.rb$}) { |m| "spec/#{m[1]}_spec.rb" }
9
- watch('spec/spec_helper.rb') { "spec" }
10
- end
11
-
12
- guard 'cucumber', :all_on_start => false do
13
- watch(%r{^features/.+\.feature$})
14
- watch(%r{^features/support/.+$}) { 'features' }
15
- watch(%r{^features/step_definitions/(.+)_steps\.rb$}) { |m| Dir[File.join("**/#{m[1]}.feature")][0] || 'features' }
16
- end
17
-
@@ -1,62 +0,0 @@
1
- Feature: Resque Delta Indexing
2
- In order to have delta indexing on frequently-updated sites
3
- Developers
4
- Should be able to use Resque to handle delta indices to lower system load
5
-
6
- Background:
7
- Given Sphinx is running
8
- And I am searching on delayed betas
9
- And I have data and it has been indexed
10
-
11
- Scenario: Delta Index should not fire automatically
12
- When I search for one
13
- Then I should get 1 result
14
-
15
- When I change the name of delayed beta one to eleven
16
- And I wait for Sphinx to catch up
17
- And I search for one
18
- Then I should get 1 result
19
-
20
- When I search for eleven
21
- Then I should get 0 results
22
-
23
- Scenario: Delta Index should fire when jobs are run
24
- When I search for one
25
- Then I should get 1 result
26
-
27
- When I change the name of delayed beta two to twelve
28
- And I wait for Sphinx to catch up
29
- And I search for twelve
30
- Then I should get 0 results
31
-
32
- When I run the delayed jobs
33
- And I wait for Sphinx to catch up
34
- And I search for twelve
35
- Then I should get 1 result
36
-
37
- When I search for two
38
- Then I should get 0 results
39
-
40
- Scenario: ensuring that duplicate jobs are deleted
41
- When I change the name of delayed beta two to fifty
42
- And I change the name of delayed beta five to twelve
43
- And I change the name of delayed beta one to fifteen
44
- And I change the name of delayed beta six to twenty
45
- And I run one delayed job
46
- Then there should be no more DeltaJobs on the Resque queue
47
-
48
- When I run the delayed jobs
49
- And I wait for Sphinx to catch up
50
- And I search for fifty
51
- Then I should get 1 result
52
-
53
- When I search for two
54
- Then I should get 0 results
55
-
56
- Scenario: canceling jobs
57
- When I change the name of delayed beta two to fifty
58
- And I cancel the jobs
59
- And I run the delayed jobs
60
- And I wait for Sphinx to catch up
61
- And I search for fifty
62
- Then I should get 0 results
@@ -1,43 +0,0 @@
1
- Feature: Smart Indexing
2
- In order to have core indexing that works well with resque delta indexing
3
- Developers
4
- Should be able to use smart index to update core indices
5
-
6
- Background:
7
- Given Sphinx is running
8
- And I am searching on delayed betas
9
- And I have data
10
-
11
- Scenario: Smart indexing should update core indices
12
- When I run the smart indexer
13
- And I wait for Sphinx to catch up
14
- And I search for one
15
- Then I should get 1 result
16
-
17
- Scenario: Smart indexing should reset the delta index
18
- Given I have indexed
19
- When I change the name of delayed beta one to eleven
20
- And I run the delayed jobs
21
- And I wait for Sphinx to catch up
22
-
23
- When I change the name of delayed beta eleven to one
24
- And I run the smart indexer
25
- And I run the delayed jobs
26
- And I wait for Sphinx to catch up
27
-
28
- When I search for eleven
29
- Then I should get 0 results
30
-
31
- Scenario: Delta Index running after smart indexing should not hide records
32
- When I run the smart indexer
33
- And I run the delayed jobs
34
- And I wait for Sphinx to catch up
35
-
36
- When I search for two
37
- Then I should get 1 result
38
-
39
- Scenario: Smart index should remove existing delta jobs
40
- When I run the smart indexer
41
- And I run one delayed job
42
- And I wait for Sphinx to catch up
43
- Then there should be no more DeltaJobs on the Resque queue
@@ -1,76 +0,0 @@
1
- Before do
2
- $queries_executed = []
3
- ThinkingSphinx::Deltas::ResqueDelta.clear!
4
- @model = nil
5
- @method = :search
6
- @query = ""
7
- @conditions = {}
8
- @with = {}
9
- @without = {}
10
- @with_all = {}
11
- @options = {}
12
- @results = nil
13
- end
14
-
15
- Given "Sphinx is running" do
16
- ThinkingSphinx::Configuration.instance.controller.should be_running
17
- end
18
-
19
- Given /^I am searching on (.+)$/ do |model|
20
- @model = model.gsub(/\s/, '_').singularize.camelize.constantize
21
- end
22
-
23
- Given "I have data" do
24
- DelayedBeta.create(:name => "one")
25
- DelayedBeta.create(:name => "two")
26
- DelayedBeta.create(:name => "three")
27
- DelayedBeta.create(:name => "four")
28
- DelayedBeta.create(:name => "five")
29
- DelayedBeta.create(:name => "six")
30
- DelayedBeta.create(:name => "seven")
31
- DelayedBeta.create(:name => "eight")
32
- DelayedBeta.create(:name => "nine")
33
- DelayedBeta.create(:name => "ten")
34
- end
35
-
36
- Given "I have indexed" do
37
- ThinkingSphinx::Deltas::ResqueDelta.clear!
38
- ThinkingSphinx::Configuration.instance.controller.index
39
- sleep(1.5)
40
- end
41
-
42
- Given "I have data and it has been indexed" do
43
- step "I have data"
44
- step "I have indexed"
45
- end
46
-
47
- When "I wait for Sphinx to catch up" do
48
- sleep(0.5)
49
- end
50
-
51
- When /^I search for (\w+)$/ do |query|
52
- @results = nil
53
- @query = query
54
- end
55
-
56
- Then /^I should get (\d+) results?$/ do |count|
57
- results.length.should == count.to_i
58
- end
59
-
60
- Then /^I debug$/ do
61
- debugger
62
- 0
63
- end
64
-
65
- def results
66
- @results ||= (@model || ThinkingSphinx).send(
67
- @method,
68
- @query,
69
- @options.merge(
70
- :conditions => @conditions,
71
- :with => @with,
72
- :without => @without,
73
- :with_all => @with_all
74
- )
75
- )
76
- end
@@ -1,33 +0,0 @@
1
- When /^I run the delayed jobs$/ do
2
- unless @resque_worker
3
- @resque_worker = Resque::Worker.new("ts_delta")
4
- @resque_worker.register_worker
5
- end
6
- while job = @resque_worker.reserve
7
- @resque_worker.perform(job)
8
- end
9
- end
10
-
11
- When /^I run one delayed job$/ do
12
- unless @resque_worker
13
- @resque_worker = Resque::Worker.new("ts_delta")
14
- @resque_worker.register_worker
15
- end
16
- job = @resque_worker.reserve
17
- @resque_worker.perform(job)
18
- end
19
-
20
- When /^I cancel the jobs$/ do
21
- ThinkingSphinx::Deltas::ResqueDelta.clear!
22
- end
23
-
24
- When /^I change the name of delayed beta (\w+) to (\w+)$/ do |current, replacement|
25
- DelayedBeta.find_by_name(current).update_attributes(:name => replacement)
26
- end
27
-
28
- Then /^there should be no more DeltaJobs on the Resque queue$/ do
29
- job_classes = Resque.redis.lrange("queue:ts_delta", 0, -1).collect do |j|
30
- Resque.decode(j)["class"]
31
- end
32
- job_classes.should_not include("ThinkingSphinx::Deltas::ResqueDelta::DeltaJob")
33
- end
@@ -1,3 +0,0 @@
1
- When /^I run the smart indexer$/ do
2
- ThinkingSphinx::Deltas::ResqueDelta::CoreIndex.new.smart_index(:verbose => false)
3
- end
@@ -1,41 +0,0 @@
1
- require 'cucumber'
2
- require 'rspec/expectations'
3
- require 'fileutils'
4
- require 'active_record'
5
- require 'mock_redis'
6
-
7
- PROJECT_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
8
-
9
- $:.unshift(File.join(PROJECT_ROOT, 'lib'))
10
- $:.unshift(File.dirname(__FILE__))
11
-
12
- require 'cucumber/thinking_sphinx/internal_world'
13
-
14
- ActiveRecord::Base.default_timezone = :utc
15
-
16
- world = Cucumber::ThinkingSphinx::InternalWorld.new
17
- world.configure_database
18
-
19
- require 'thinking_sphinx'
20
- require 'thinking_sphinx/deltas/resque_delta'
21
-
22
- ActiveRecord::Base.send(:include, ThinkingSphinx::ActiveRecord)
23
-
24
- ActiveSupport::Inflector.inflections do |inflect|
25
- inflect.plural /^(.*)beta$/i, '\1betas'
26
- inflect.singular /^(.*)betas$/i, '\1beta'
27
- end
28
-
29
- world.setup
30
-
31
- ThinkingSphinx.updates_enabled = true
32
- ThinkingSphinx.deltas_enabled = true
33
-
34
- Resque.redis = MockRedis.new
35
- Before do
36
- Resque.redis.flushall
37
- end
38
-
39
- require 'database_cleaner'
40
- require 'database_cleaner/cucumber'
41
- DatabaseCleaner.strategy = :truncation
@@ -1,4 +0,0 @@
1
- username: "root"
2
- host: "localhost"
3
- password:
4
- adapter: "mysql2"
@@ -1,4 +0,0 @@
1
- ActiveRecord::Base.connection.create_table :delayed_betas, :force => true do |t|
2
- t.column :name, :string, :null => false
3
- t.column :delta, :boolean, :null => false, :default => false
4
- end
@@ -1,6 +0,0 @@
1
- class DelayedBeta < ActiveRecord::Base
2
- define_index do
3
- indexes :name, :sortable => true
4
- set_property :delta => ThinkingSphinx::Deltas::ResqueDelta
5
- end
6
- end
@@ -1,8 +0,0 @@
1
- # This file was generated by Appraisal
2
-
3
- source "http://rubygems.org"
4
-
5
- gem "activerecord", "~> 2.3.14"
6
- gem "mysql2", "~> 0.2.7"
7
-
8
- gemspec :path=>"../"
@@ -1,114 +0,0 @@
1
- PATH
2
- remote: /Users/aarongibralter/Dropbox/Documents/source-code/ts-resque-delta
3
- specs:
4
- ts-resque-delta (1.2.3)
5
- resque (~> 1.10)
6
- resque-lock-timeout (~> 0.4)
7
- riddle (>= 1.5)
8
- thinking-sphinx (>= 1.4.1)
9
-
10
- GEM
11
- remote: http://rubygems.org/
12
- specs:
13
- activerecord (2.3.14)
14
- activesupport (= 2.3.14)
15
- activesupport (2.3.14)
16
- after_commit (1.0.10)
17
- activerecord (>= 1.15.6, < 3.0.0)
18
- appraisal (0.4.1)
19
- bundler
20
- rake
21
- builder (3.1.4)
22
- cucumber (1.2.1)
23
- builder (>= 2.1.2)
24
- diff-lcs (>= 1.1.3)
25
- gherkin (~> 2.11.0)
26
- json (>= 1.4.6)
27
- database_cleaner (0.9.1)
28
- diff-lcs (1.1.3)
29
- fakefs (0.4.2)
30
- faraday (0.8.4)
31
- multipart-post (~> 1.1)
32
- faraday_middleware (0.9.0)
33
- faraday (>= 0.7.4, < 0.9)
34
- flying-sphinx (0.8.4)
35
- faraday_middleware (~> 0.7)
36
- multi_json (>= 1.0.1)
37
- rash (~> 0.3.0)
38
- riddle (>= 1.5.0)
39
- thinking-sphinx
40
- gherkin (2.11.5)
41
- json (>= 1.4.6)
42
- guard (0.8.8)
43
- thor (~> 0.14.6)
44
- guard-bundler (0.1.3)
45
- bundler (>= 1.0.0)
46
- guard (>= 0.2.2)
47
- guard-cucumber (0.7.5)
48
- cucumber (>= 0.10)
49
- guard (>= 0.8.3)
50
- guard-rspec (0.5.11)
51
- guard (>= 0.8.4)
52
- hashie (1.2.0)
53
- json (1.7.5)
54
- mock_redis (0.3.0)
55
- multi_json (1.3.7)
56
- multipart-post (1.1.5)
57
- mysql2 (0.2.18)
58
- rack (1.4.1)
59
- rack-protection (1.2.0)
60
- rack
61
- rake (10.0.2)
62
- rash (0.3.2)
63
- hashie (~> 1.2.0)
64
- redis (3.0.2)
65
- redis-namespace (1.2.1)
66
- redis (~> 3.0.0)
67
- resque (1.23.0)
68
- multi_json (~> 1.0)
69
- redis-namespace (~> 1.0)
70
- sinatra (>= 0.9.2)
71
- vegas (~> 0.1.2)
72
- resque-lock-timeout (0.4.1)
73
- resque (>= 1.8.0)
74
- riddle (1.5.3)
75
- rspec (2.10.0)
76
- rspec-core (~> 2.10.0)
77
- rspec-expectations (~> 2.10.0)
78
- rspec-mocks (~> 2.10.0)
79
- rspec-core (2.10.1)
80
- rspec-expectations (2.10.0)
81
- diff-lcs (~> 1.1.3)
82
- rspec-mocks (2.10.1)
83
- sinatra (1.3.3)
84
- rack (~> 1.3, >= 1.3.6)
85
- rack-protection (~> 1.2)
86
- tilt (~> 1.3, >= 1.3.3)
87
- thinking-sphinx (1.4.13)
88
- activerecord (>= 1.15.6, < 3.0.0)
89
- after_commit (>= 1.0.7)
90
- riddle (>= 1.5.3)
91
- thor (0.14.6)
92
- tilt (1.3.3)
93
- vegas (0.1.11)
94
- rack (>= 1.0.0)
95
-
96
- PLATFORMS
97
- ruby
98
-
99
- DEPENDENCIES
100
- activerecord (~> 2.3.14)
101
- appraisal (~> 0.4.1)
102
- cucumber
103
- database_cleaner (>= 0.5.2)
104
- fakefs (~> 0.4.0)
105
- flying-sphinx (>= 0.5.1)
106
- guard (~> 0.8.8)
107
- guard-bundler (~> 0.1.3)
108
- guard-cucumber (~> 0.7.4)
109
- guard-rspec (~> 0.5.8)
110
- mock_redis (~> 0.3.0)
111
- mysql2 (~> 0.2.7)
112
- rake (>= 0.8.7)
113
- rspec (~> 2.10.0)
114
- ts-resque-delta!
@@ -1,8 +0,0 @@
1
- # This file was generated by Appraisal
2
-
3
- source "http://rubygems.org"
4
-
5
- gem "activerecord", "~> 3.2.9"
6
- gem "mysql2", "~> 0.3.11"
7
-
8
- gemspec :path=>"../"
@@ -1,123 +0,0 @@
1
- PATH
2
- remote: /Users/aarongibralter/Dropbox/Documents/source-code/ts-resque-delta
3
- specs:
4
- ts-resque-delta (1.2.3)
5
- resque (~> 1.10)
6
- resque-lock-timeout (~> 0.4)
7
- riddle (>= 1.5)
8
- thinking-sphinx (>= 1.4.1)
9
-
10
- GEM
11
- remote: http://rubygems.org/
12
- specs:
13
- activemodel (3.2.9)
14
- activesupport (= 3.2.9)
15
- builder (~> 3.0.0)
16
- activerecord (3.2.9)
17
- activemodel (= 3.2.9)
18
- activesupport (= 3.2.9)
19
- arel (~> 3.0.2)
20
- tzinfo (~> 0.3.29)
21
- activesupport (3.2.9)
22
- i18n (~> 0.6)
23
- multi_json (~> 1.0)
24
- appraisal (0.4.1)
25
- bundler
26
- rake
27
- arel (3.0.2)
28
- builder (3.0.4)
29
- cucumber (1.2.1)
30
- builder (>= 2.1.2)
31
- diff-lcs (>= 1.1.3)
32
- gherkin (~> 2.11.0)
33
- json (>= 1.4.6)
34
- database_cleaner (0.9.1)
35
- diff-lcs (1.1.3)
36
- fakefs (0.4.2)
37
- faraday (0.8.4)
38
- multipart-post (~> 1.1)
39
- faraday_middleware (0.9.0)
40
- faraday (>= 0.7.4, < 0.9)
41
- flying-sphinx (0.8.4)
42
- faraday_middleware (~> 0.7)
43
- multi_json (>= 1.0.1)
44
- rash (~> 0.3.0)
45
- riddle (>= 1.5.0)
46
- thinking-sphinx
47
- gherkin (2.11.5)
48
- json (>= 1.4.6)
49
- guard (0.8.8)
50
- thor (~> 0.14.6)
51
- guard-bundler (0.1.3)
52
- bundler (>= 1.0.0)
53
- guard (>= 0.2.2)
54
- guard-cucumber (0.7.5)
55
- cucumber (>= 0.10)
56
- guard (>= 0.8.3)
57
- guard-rspec (0.5.11)
58
- guard (>= 0.8.4)
59
- hashie (1.2.0)
60
- i18n (0.6.1)
61
- json (1.7.5)
62
- mock_redis (0.3.0)
63
- multi_json (1.3.7)
64
- multipart-post (1.1.5)
65
- mysql2 (0.3.11)
66
- rack (1.4.1)
67
- rack-protection (1.2.0)
68
- rack
69
- rake (10.0.2)
70
- rash (0.3.2)
71
- hashie (~> 1.2.0)
72
- redis (3.0.2)
73
- redis-namespace (1.2.1)
74
- redis (~> 3.0.0)
75
- resque (1.23.0)
76
- multi_json (~> 1.0)
77
- redis-namespace (~> 1.0)
78
- sinatra (>= 0.9.2)
79
- vegas (~> 0.1.2)
80
- resque-lock-timeout (0.4.1)
81
- resque (>= 1.8.0)
82
- riddle (1.5.3)
83
- rspec (2.10.0)
84
- rspec-core (~> 2.10.0)
85
- rspec-expectations (~> 2.10.0)
86
- rspec-mocks (~> 2.10.0)
87
- rspec-core (2.10.1)
88
- rspec-expectations (2.10.0)
89
- diff-lcs (~> 1.1.3)
90
- rspec-mocks (2.10.1)
91
- sinatra (1.3.3)
92
- rack (~> 1.3, >= 1.3.6)
93
- rack-protection (~> 1.2)
94
- tilt (~> 1.3, >= 1.3.3)
95
- thinking-sphinx (2.0.13)
96
- activerecord (>= 3.0.3)
97
- builder (>= 2.1.2)
98
- riddle (>= 1.5.3)
99
- thor (0.14.6)
100
- tilt (1.3.3)
101
- tzinfo (0.3.35)
102
- vegas (0.1.11)
103
- rack (>= 1.0.0)
104
-
105
- PLATFORMS
106
- ruby
107
-
108
- DEPENDENCIES
109
- activerecord (~> 3.2.9)
110
- appraisal (~> 0.4.1)
111
- cucumber
112
- database_cleaner (>= 0.5.2)
113
- fakefs (~> 0.4.0)
114
- flying-sphinx (>= 0.5.1)
115
- guard (~> 0.8.8)
116
- guard-bundler (~> 0.1.3)
117
- guard-cucumber (~> 0.7.4)
118
- guard-rspec (~> 0.5.8)
119
- mock_redis (~> 0.3.0)
120
- mysql2 (~> 0.3.11)
121
- rake (>= 0.8.7)
122
- rspec (~> 2.10.0)
123
- ts-resque-delta!