ts-delayed-delta 1.1.3 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. data/.gitignore +1 -0
  2. data/.travis.yml +14 -12
  3. data/Appraisals +11 -0
  4. data/Gemfile +1 -3
  5. data/LICENSE +1 -1
  6. data/README.textile +29 -26
  7. data/Rakefile +3 -16
  8. data/gemfiles/binary.gemfile +7 -0
  9. data/gemfiles/sphinxql.gemfile +7 -0
  10. data/lib/thinking_sphinx/deltas/delayed_delta.rb +117 -47
  11. data/lib/thinking_sphinx/deltas/delayed_delta/delta_job.rb +7 -16
  12. data/lib/thinking_sphinx/deltas/delayed_delta/flag_as_deleted_job.rb +8 -19
  13. data/lib/ts-delayed-delta.rb +0 -3
  14. data/spec/acceptance/delayed_job_deltas_spec.rb +46 -0
  15. data/spec/acceptance/spec_helper.rb +4 -0
  16. data/spec/acceptance/support/database_cleaner.rb +11 -0
  17. data/spec/acceptance/support/sphinx_controller.rb +61 -0
  18. data/spec/acceptance/support/sphinx_helpers.rb +31 -0
  19. data/spec/internal/.gitignore +2 -0
  20. data/spec/internal/app/indices/book_index.rb +3 -0
  21. data/spec/internal/app/models/book.rb +7 -0
  22. data/spec/internal/config/database.yml +5 -0
  23. data/spec/internal/db/schema.rb +24 -0
  24. data/spec/internal/log/.gitignore +1 -0
  25. data/spec/spec_helper.rb +9 -7
  26. data/ts-delayed-delta.gemspec +15 -11
  27. metadata +129 -86
  28. data/VERSION +0 -1
  29. data/features/delayed_deltas.feature +0 -37
  30. data/features/step_definitions/common_steps.rb +0 -48
  31. data/features/step_definitions/delayed_delta_steps.rb +0 -11
  32. data/features/support/env.rb +0 -33
  33. data/features/thinking_sphinx/database.example.yml +0 -3
  34. data/features/thinking_sphinx/db/fixtures/delayed_betas.rb +0 -10
  35. data/features/thinking_sphinx/db/migrations/create_delayed_betas.rb +0 -17
  36. data/features/thinking_sphinx/models/delayed_beta.rb +0 -7
  37. data/lib/thinking_sphinx/deltas/delayed_delta/job.rb +0 -65
  38. data/lib/thinking_sphinx/deltas/delayed_delta/railtie.rb +0 -5
  39. data/lib/thinking_sphinx/deltas/delayed_delta/tasks.rb +0 -23
  40. data/lib/thinking_sphinx/deltas/delayed_delta/version.rb +0 -5
  41. data/spec/thinking_sphinx/deltas/delayed_delta/delta_job_spec.rb +0 -53
  42. data/spec/thinking_sphinx/deltas/delayed_delta/flag_as_deleted_job_spec.rb +0 -78
  43. data/spec/thinking_sphinx/deltas/delayed_delta/job_spec.rb +0 -52
  44. data/spec/thinking_sphinx/deltas/delayed_delta_spec.rb +0 -128
  45. data/tasks/rails.rake +0 -1
data/VERSION DELETED
@@ -1 +0,0 @@
1
- 1.1.2
@@ -1,37 +0,0 @@
1
- Feature: Delayed Delta Indexing
2
- In order to have delta indexing on frequently-updated sites
3
- Developers
4
- Should be able to use delayed_job to handle delta indexes to lower system load
5
-
6
- Scenario: Delta Index should not fire automatically
7
- Given Sphinx is running
8
- And I am searching on delayed betas
9
- When I search for one
10
- Then I should get 1 result
11
-
12
- When I change the name of delayed beta one to eleven
13
- And I wait for Sphinx to catch up
14
- And I search for one
15
- Then I should get 1 result
16
-
17
- When I search for eleven
18
- Then I should get 0 results
19
-
20
- Scenario: Delta Index should fire when jobs are run
21
- Given Sphinx is running
22
- And I am searching on delayed betas
23
- When I search for one
24
- Then I should get 1 result
25
-
26
- When I change the name of delayed beta two to twelve
27
- And I wait for Sphinx to catch up
28
- And I search for twelve
29
- Then I should get 0 results
30
-
31
- When I run the delayed jobs
32
- And I wait for Sphinx to catch up
33
- And I search for twelve
34
- Then I should get 1 result
35
-
36
- When I search for two
37
- Then I should get 0 results
@@ -1,48 +0,0 @@
1
- Before do
2
- $queries_executed = []
3
- ThinkingSphinx::Deltas::Job.cancel_thinking_sphinx_jobs
4
-
5
- @model = nil
6
- @method = :search
7
- @query = ""
8
- @conditions = {}
9
- @with = {}
10
- @without = {}
11
- @with_all = {}
12
- @options = {}
13
- @results = nil
14
- end
15
-
16
- Given "Sphinx is running" do
17
- ThinkingSphinx::Configuration.instance.controller.should be_running
18
- end
19
-
20
- Given /^I am searching on (.+)$/ do |model|
21
- @model = model.gsub(/\s/, '_').singularize.camelize.constantize
22
- end
23
-
24
- When "I wait for Sphinx to catch up" do
25
- sleep(0.25)
26
- end
27
-
28
- When /^I search for (\w+)$/ do |query|
29
- @results = nil
30
- @query = query
31
- end
32
-
33
- Then /^I should get (\d+) results?$/ do |count|
34
- results.length.should == count.to_i
35
- end
36
-
37
- def results
38
- @results ||= (@model || ThinkingSphinx).send(
39
- @method,
40
- @query,
41
- @options.merge(
42
- :conditions => @conditions,
43
- :with => @with,
44
- :without => @without,
45
- :with_all => @with_all
46
- )
47
- )
48
- end
@@ -1,11 +0,0 @@
1
- When /^I run the delayed jobs$/ do
2
- if Delayed::Worker.respond_to? :backend
3
- Delayed::Worker.new(:quiet => true).work_off
4
- else
5
- Delayed::Job.work_off
6
- end
7
- end
8
-
9
- When /^I change the name of delayed beta (\w+) to (\w+)$/ do |current, replacement|
10
- DelayedBeta.find_by_name(current).update_attributes(:name => replacement)
11
- end
@@ -1,33 +0,0 @@
1
- require 'rubygems'
2
- require 'fileutils'
3
- require 'bundler'
4
-
5
- Bundler.require :default, :development
6
-
7
- require 'active_record'
8
- require 'thinking_sphinx'
9
- require 'delayed_job'
10
-
11
- ActiveRecord::Base.send(:include, ThinkingSphinx::ActiveRecord)
12
- Delayed::Worker.backend = :active_record
13
-
14
- ActiveSupport::Inflector.inflections do |inflect|
15
- inflect.plural /^(.*)beta$/i, '\1betas'
16
- inflect.singular /^(.*)betas$/i, '\1beta'
17
- end
18
-
19
- $:.unshift File.dirname(__FILE__) + '/../../lib'
20
-
21
- require 'cucumber/thinking_sphinx/internal_world'
22
-
23
- # Time.zone_default = Time.__send__(:get_zone, 'Melbourne')
24
- # ActiveRecord::Base.time_zone_aware_attributes = true
25
- # ActiveRecord::Base.default_timezone = :utc
26
- #
27
- world = Cucumber::ThinkingSphinx::InternalWorld.new
28
- world.configure_database
29
-
30
- require "thinking_sphinx"
31
- require 'thinking_sphinx/deltas/delayed_delta'
32
-
33
- world.setup
@@ -1,3 +0,0 @@
1
- username: root
2
- host: localhost
3
- password:
@@ -1,10 +0,0 @@
1
- DelayedBeta.create :name => "one"
2
- DelayedBeta.create :name => "two"
3
- DelayedBeta.create :name => "three"
4
- DelayedBeta.create :name => "four"
5
- DelayedBeta.create :name => "five"
6
- DelayedBeta.create :name => "six"
7
- DelayedBeta.create :name => "seven"
8
- DelayedBeta.create :name => "eight"
9
- DelayedBeta.create :name => "nine"
10
- DelayedBeta.create :name => "ten"
@@ -1,17 +0,0 @@
1
- ActiveRecord::Base.connection.create_table :delayed_betas, :force => true do |t|
2
- t.column :name, :string, :null => false
3
- t.column :delta, :boolean, :null => false, :default => false
4
- end
5
-
6
- ActiveRecord::Base.connection.create_table :delayed_jobs, :force => true do |t|
7
- t.column :priority, :integer, :default => 0
8
- t.column :attempts, :integer, :default => 0
9
- t.column :handler, :text
10
- t.column :last_error, :string
11
- t.column :run_at, :datetime
12
- t.column :locked_at, :datetime
13
- t.column :failed_at, :datetime
14
- t.column :locked_by, :string
15
- t.column :created_at, :datetime
16
- t.column :updated_at, :datetime
17
- end
@@ -1,7 +0,0 @@
1
- class DelayedBeta < ActiveRecord::Base
2
- define_index do
3
- indexes :name, :sortable => true
4
-
5
- set_property :delta => :delayed
6
- end
7
- end
@@ -1,65 +0,0 @@
1
- module Delayed
2
- module Backend
3
- module ActiveRecord
4
- class Job < ::ActiveRecord::Base
5
- self.table_name = "delayed_jobs"
6
- end
7
- end
8
- end
9
- end
10
-
11
- # A custom job model, subclassed from Delayed::Job. The two things it does
12
- # differently is that it checks for duplicate tasks before enqueuing them, and
13
- # provides the option to remove all Delayed Delta jobs from the queue.
14
- #
15
- # As such, this class should not be used for any other tasks.
16
- #
17
- class ThinkingSphinx::Deltas::Job < Delayed::Backend::ActiveRecord::Job
18
- self.table_name = "delayed_jobs"
19
- # Adds a job to the queue, if it doesn't already exist. This is to ensure
20
- # multiple indexing requests for the same delta index don't get added, as the
21
- # index only needs to be processed once.
22
- #
23
- # Because indexing jobs are all the same object, with a single instance
24
- # variable (the index name), they all get serialised to the same YAML value.
25
- #
26
- # @param [Object] object The job, which must respond to the #perform method.
27
- # @param [Integer] priority (0)
28
- #
29
- def self.enqueue(object, priority = 0)
30
- ::Delayed::Job.enqueue(object, :priority => priority) unless duplicates_exist(object)
31
- end
32
-
33
- # Remove all Thinking Sphinx/Delayed Delta jobs from the queue. If the
34
- # delayed_jobs table does not exist, this method will do nothing.
35
- #
36
- def self.cancel_thinking_sphinx_jobs
37
- if connection.tables.include?("delayed_jobs")
38
- delete_all("handler LIKE '--- !ruby/object:ThinkingSphinx::Deltas::%'")
39
- end
40
- end
41
-
42
- # This is to stop ActiveRecord complaining about a missing database when
43
- # running specs (otherwise printing failure messages raises confusing stack
44
- # traces).
45
- #
46
- def self.inspect
47
- "Job"
48
- end
49
-
50
- private
51
-
52
- # Checks whether a given job already exists in the queue.
53
- #
54
- # @param [Object] object The job
55
- # @return [Boolean] True if a duplicate of the job already exists in the queue
56
- #
57
- def self.duplicates_exist(object)
58
- count(
59
- :conditions => {
60
- :handler => object.to_yaml,
61
- :locked_at => nil
62
- }
63
- ) > 0
64
- end
65
- end
@@ -1,5 +0,0 @@
1
- class ThinkingSphinx::Deltas::DelayedDelta::Railtie < Rails::Railtie
2
- rake_tasks do
3
- load 'thinking_sphinx/deltas/delayed_delta/tasks.rb'
4
- end
5
- end
@@ -1,23 +0,0 @@
1
- namespace :thinking_sphinx do
2
- task :index do
3
- require 'thinking_sphinx/deltas/delayed_delta'
4
- ThinkingSphinx::Deltas::Job.cancel_thinking_sphinx_jobs
5
- end
6
-
7
- desc "Process stored delta index requests"
8
- task :delayed_delta => :app_env do
9
- require 'delayed_job'
10
- require 'delayed/worker'
11
- require 'thinking_sphinx/deltas/delayed_delta'
12
-
13
- Delayed::Worker.new(
14
- :min_priority => ENV['MIN_PRIORITY'],
15
- :max_priority => ENV['MAX_PRIORITY']
16
- ).start
17
- end
18
- end
19
-
20
- namespace :ts do
21
- desc "Process stored delta index requests"
22
- task :dd => "thinking_sphinx:delayed_delta"
23
- end
@@ -1,5 +0,0 @@
1
- module ThinkingSphinx
2
- module DelayedDelta
3
- Version = '1.1.3'
4
- end
5
- end
@@ -1,53 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe ThinkingSphinx::Deltas::DeltaJob do
4
- describe '#perform' do
5
- before :each do
6
- ThinkingSphinx.suppress_delta_output = false
7
-
8
- @delta_job = ThinkingSphinx::Deltas::DeltaJob.new(['foo_core'])
9
- @delta_job.stub! :` => true
10
- @delta_job.stub! :puts => nil
11
- end
12
-
13
- it "should output the delta indexing by default" do
14
- @delta_job.should_receive(:puts)
15
-
16
- @delta_job.perform
17
- end
18
-
19
- it "should not output the delta indexing if requested" do
20
- ThinkingSphinx.suppress_delta_output = true
21
- @delta_job.should_not_receive(:puts)
22
-
23
- @delta_job.perform
24
- end
25
-
26
- it "should process just the requested indices" do
27
- @delta_job.should_receive(:`) do |command|
28
- command.should match(/foo_core/)
29
- command.should_not match(/--all/)
30
- end
31
-
32
- @delta_job.perform
33
- end
34
-
35
- context 'multiple indices' do
36
- it "should process all requested indices" do
37
- @delta_job.indices = ['foo_core', 'bar_core']
38
- @delta_job.should_receive(:`) do |command|
39
- command.should match(/foo_core bar_core/)
40
- end
41
-
42
- @delta_job.perform
43
- end
44
- end
45
- end
46
-
47
- describe "#display_name" do
48
- it "should display class name with all indices" do
49
- @delta_job = ThinkingSphinx::Deltas::DeltaJob.new(['foo_core', 'bar_core'])
50
- @delta_job.display_name.should == "ThinkingSphinx::Deltas::DeltaJob for foo_core, bar_core"
51
- end
52
- end
53
- end
@@ -1,78 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe ThinkingSphinx::Deltas::FlagAsDeletedJob do
4
- describe '#perform' do
5
- before :each do
6
- ThinkingSphinx.updates_enabled = true
7
- @client = stub('client', :update => true)
8
-
9
- ThinkingSphinx::Configuration.instance.stub!(:client => @client)
10
- ThinkingSphinx.stub!(:search_for_id => true)
11
- ThinkingSphinx.stub!(:sphinx_running? => true)
12
-
13
- @job = ThinkingSphinx::Deltas::FlagAsDeletedJob.new(['foo_core'], 12)
14
- end
15
-
16
- it "should not update if Sphinx isn't running" do
17
- ThinkingSphinx.stub!(:sphinx_running? => false)
18
- @client.should_not_receive(:update)
19
-
20
- @job.perform
21
- end
22
-
23
- it "should not update if the document isn't in the index" do
24
- ThinkingSphinx.stub!(:search_for_id => false)
25
- @client.should_not_receive(:update)
26
-
27
- @job.perform
28
- end
29
-
30
- it "should update the specified index" do
31
- @client.should_receive(:update) do |index, attributes, values|
32
- index.should == 'foo_core'
33
- end
34
-
35
- @job.perform
36
- end
37
-
38
- it "should update all specified indices" do
39
- @job.indices = ['foo_core', 'bar_core']
40
- @client.should_receive(:update).with('foo_core', anything, anything)
41
- @client.should_receive(:update).with('bar_core', anything, anything)
42
-
43
- @job.perform
44
- end
45
-
46
- it "should update the sphinx_deleted attribute" do
47
- @client.should_receive(:update) do |index, attributes, values|
48
- attributes.should == ['sphinx_deleted']
49
- end
50
-
51
- @job.perform
52
- end
53
-
54
- it "should set sphinx_deleted for the given document to true" do
55
- @client.should_receive(:update) do |index, attributes, values|
56
- values[12].should == [1]
57
- end
58
-
59
- @job.perform
60
- end
61
-
62
- it "should check for the existence of the document in the specified index" do
63
- ThinkingSphinx.should_receive(:search_for_id) do |id, index|
64
- index.should == 'foo_core'
65
- end
66
-
67
- @job.perform
68
- end
69
-
70
- it "should check for the existence of the given document id" do
71
- ThinkingSphinx.should_receive(:search_for_id) do |id, index|
72
- id.should == 12
73
- end
74
-
75
- @job.perform
76
- end
77
- end
78
- end
@@ -1,52 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe ThinkingSphinx::Deltas::Job do
4
- describe '.enqueue' do
5
- before :each do
6
- ThinkingSphinx::Deltas::Job.stub!(:count => 0)
7
- end
8
-
9
- it "should enqueue if there's no existing jobs for the same index" do
10
- Delayed::Job.should_receive(:enqueue)
11
-
12
- ThinkingSphinx::Deltas::Job.enqueue(stub('object'))
13
- end
14
-
15
- it "should not enqueue the job if there's an existing job already" do
16
- ThinkingSphinx::Deltas::Job.stub!(:count => 1)
17
- Delayed::Job.should_not_receive(:enqueue)
18
-
19
- ThinkingSphinx::Deltas::Job.enqueue(stub('object'))
20
- end
21
- end
22
-
23
- describe '.cancel_thinking_sphinx_jobs' do
24
- before :each do
25
- ThinkingSphinx::Deltas::Job.stub!(:connection => double('connection'))
26
- ThinkingSphinx::Deltas::Job.stub!(:delete_all => true)
27
- end
28
-
29
- it "should not delete any rows if the delayed_jobs table does not exist" do
30
- ThinkingSphinx::Deltas::Job.connection.stub!(:tables => [])
31
- ThinkingSphinx::Deltas::Job.should_not_receive(:delete_all)
32
-
33
- ThinkingSphinx::Deltas::Job.cancel_thinking_sphinx_jobs
34
- end
35
-
36
- it "should delete rows if the delayed_jobs table does exist" do
37
- ThinkingSphinx::Deltas::Job.connection.stub!(:tables => ['delayed_jobs'])
38
- ThinkingSphinx::Deltas::Job.should_receive(:delete_all)
39
-
40
- ThinkingSphinx::Deltas::Job.cancel_thinking_sphinx_jobs
41
- end
42
-
43
- it "should delete only Thinking Sphinx jobs" do
44
- ThinkingSphinx::Deltas::Job.connection.stub!(:tables => ['delayed_jobs'])
45
- ThinkingSphinx::Deltas::Job.should_receive(:delete_all) do |sql|
46
- sql.should match(/handler LIKE '--- !ruby\/object:ThinkingSphinx::Deltas::\%'/)
47
- end
48
-
49
- ThinkingSphinx::Deltas::Job.cancel_thinking_sphinx_jobs
50
- end
51
- end
52
- end