ts-resque-delta 1.2.4 → 2.0.0

data/lib/flying_sphinx/resque_delta.rb

@@ -1,35 +0,0 @@
-require 'thinking_sphinx/deltas/resque_delta'
-
-class FlyingSphinx::ResqueDelta < ThinkingSphinx::Deltas::ResqueDelta
-  def self.job_types
-    [
-      FlyingSphinx::ResqueDelta::DeltaJob,
-      FlyingSphinx::ResqueDelta::FlagAsDeletedJob
-    ]
-  end
-
-  def self.job_prefix
-    'fs-delta'
-  end
-
-  def index(model, instance = nil)
-    return true if skip?(instance)
-
-    model.delta_index_names.each do |delta|
-      next if self.class.locked?(delta)
-
-      Resque.enqueue FlyingSphinx::ResqueDelta::DeltaJob, delta
-    end
-
-    Resque.enqueue(
-      FlyingSphinx::ResqueDelta::FlagAsDeletedJob,
-      model.core_index_names,
-      instance.sphinx_document_id
-    ) if instance
-
-    true
-  end
-end
-
-require 'flying_sphinx/resque_delta/delta_job'
-require 'flying_sphinx/resque_delta/flag_as_deleted_job'

data/lib/flying_sphinx/resque_delta/delta_job.rb

@@ -1,14 +0,0 @@
-class FlyingSphinx::ResqueDelta::DeltaJob < ThinkingSphinx::Deltas::ResqueDelta::DeltaJob
-  @queue = :fs_delta
-
-  # Runs Sphinx's indexer tool to process the index. Currently assumes Sphinx
-  # is running.
-  #
-  # @param [String] index the name of the Sphinx index
-  #
-  def self.perform(indices)
-    return if skip?(indices)
-
-    FlyingSphinx::IndexRequest.new([indices]).perform
-  end
-end

data/lib/flying_sphinx/resque_delta/flag_as_deleted_job.rb

@@ -1,7 +0,0 @@
-class FlyingSphinx::ResqueDelta::FlagAsDeletedJob
-  @queue = :fs_delta
-  
-  def self.perform(indices, document_id)
-    FlyingSphinx::FlagAsDeletedJob.new(indices, document_id).perform
-  end
-end

data/lib/thinking_sphinx/deltas/resque_delta/core_index.rb

@@ -1,116 +0,0 @@
-class ThinkingSphinx::Deltas::ResqueDelta::CoreIndex
-
-  def sphinx_indices
-    unless @sphinx_indices
-      @ts_config ||= ThinkingSphinx::Configuration.instance
-      @ts_config.generate
-      @sphinx_indices = @ts_config.configuration.indices.collect { |i| i.name }
-      # The collected indices look like:
-      # ["foo_core", "foo_delta", "foo", "bar_core", "bar_delta", "bar"]
-      @sphinx_indices.reject! { |i| i =~ /_(core|delta)$/}
-      # Now we have:
-      # ["foo", "bar"]
-    end
-    @sphinx_indices
-  end
-
-  # Public: Lock a delta index against indexing or new index jobs.
-  #
-  # index_name - The String index prefix.
-  #
-  # Examples
-  #
-  #   lock_delta('foo')
-  #
-  # Returns nothing.
-  def lock_delta(index_name)
-    ThinkingSphinx::Deltas::ResqueDelta.lock("#{index_name}_delta")
-  end
-
-  # Public: Unlock a delta index for indexing or new index jobs.
-  #
-  # index_name - The String index prefix.
-  #
-  # Examples
-  #
-  #   unlock_delta('foo')
-  #
-  # Returns nothing.
-  def unlock_delta(index_name)
-    ThinkingSphinx::Deltas::ResqueDelta.unlock("#{index_name}_delta")
-  end
-
-  # Public: Lock all delta indexes against indexing or new index jobs.
-  #
-  # Returns nothing.
-  def lock_deltas
-    sphinx_indices.each { |index_name| lock_delta(index_name) }
-  end
-
-  # Public: Unlock all delta indexes for indexing or new index jobs.
-  #
-  # Returns nothing.
-  def unlock_deltas
-    sphinx_indices.each { |index_name| unlock_delta(index_name) }
-  end
-
-  # Public: Index all indices while locking each delta as we index the corresponding core index.
-  #
-  # Returns true on success; false on failure.
-  def smart_index(opts = {})
-    verbose = opts.fetch(:verbose, true)
-    verbose = false if ENV['SILENT'] == 'true'
-
-    # Load config like ts:in.
-    unless ENV['INDEX_ONLY'] == 'true'
-      puts "Generating Configuration to #{ts_config.config_file}" if verbose
-      ts_config.build
-    end
-    FileUtils.mkdir_p(ts_config.searchd_file_path)
-
-    # Index each core, one at a time. Wrap with delta locking logic.
-    index_prefixes.each do |index_name|
-      ret = nil
-
-      with_delta_index_lock(index_name) do
-        ThinkingSphinx::Deltas::ResqueDelta.prepare_for_core_index(index_name)
-        ts_config.controller.index("#{index_name}_core", :verbose => verbose)
-        ret = $?
-      end
-
-      return false if ret.to_i != 0
-
-      Resque.enqueue(
-        ThinkingSphinx::Deltas::ResqueDelta::DeltaJob,
-        "#{index_name}_delta"
-      )
-    end
-
-    true
-  end
-
-  # Public: Wraps the passed block with a delta index lock
-  #
-  # index_name - The String index prefix.
-  #
-  # Examples
-  #
-  #   with_delta_index_lock('foo')
-  #
-  # Returns nothing.
-  def with_delta_index_lock(index_name)
-    lock_delta(index_name)
-    yield
-    unlock_delta(index_name)
-  end
-
-  private
-
-  def ts_config
-    ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.ts_config
-  end
-
-  def index_prefixes
-    ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.index_prefixes
-  end
-end

data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set.rb

@@ -1,59 +0,0 @@
-class ThinkingSphinx::Deltas::ResqueDelta < ThinkingSphinx::Deltas::DefaultDelta
-  module FlagAsDeletedSet
-    extend self
-
-    def set_name(core_name)
-      "#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:flag.deleted:#{core_name}:set"
-    end
-
-    def temp_name(core_name)
-      "#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:flag.deleted:#{core_name}:temp"
-    end
-
-    def processing_name(core_name)
-      "#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:flag.deleted:#{core_name}:processing"
-    end
-
-    def add(core_name, document_id)
-      Resque.redis.sadd(set_name(core_name), document_id)
-    end
-
-    def clear!(core_name)
-      Resque.redis.del(set_name(core_name))
-
-      #Clear processing set as well
-      delta_name = ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.core_to_delta(core_name)
-      ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.around_perform_lock(delta_name) do
-        Resque.redis.del(processing_name(core_name))
-      end
-    end
-
-    def clear_all!
-      ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.core_indices.each do |core_index|
-        clear!(core_index)
-      end
-    end
-
-    def get_subset_for_processing(core_name)
-      # Use a transaction to keep from losing set members if interrupted
-      Resque.redis.multi do
-        # Copy set to temp
-        Resque.redis.sunionstore temp_name(core_name), set_name(core_name)
-        # Store (set - temp) into set.  This removes all items we copied into temp from set.
-        Resque.redis.sdiffstore set_name(core_name), set_name(core_name), temp_name(core_name)
-        # Merge processing and temp together and store into processing.
-        Resque.redis.sunionstore processing_name(core_name), processing_name(core_name), temp_name(core_name)
-
-        Resque.redis.del temp_name(core_name)
-      end
-    end
-
-    def processing_members(core_name)
-      Resque.redis.smembers(processing_name(core_name)).collect(&:to_i)
-    end
-
-    def clear_processing(core_name)
-      Resque.redis.del(processing_name(core_name))
-    end
-  end
-end

data/lib/thinking_sphinx/deltas/resque_delta/index_utils.rb

@@ -1,47 +0,0 @@
-module ThinkingSphinx::Deltas::ResqueDelta::IndexUtils
-  extend self
-
-  # Public: Return a list of index prefixes (i.e. without "_core"/"_delta").
-  #
-  # Examples
-  #
-  #   sphinx_indices
-  #   # => ['foo', 'bar']
-  #
-  # Returns an Array of index prefixes.
-  def index_prefixes
-    @prefixes ||= indices.reject { |i| i =~ /_(core|delta)$/ }
-  end
-
-  def core_indices
-    @core_indices ||= indices.select { |i| i =~ /_core$/ }
-  end
-
-  def delta_indices
-    @delta_indices ||= indices.select { |i| i =~ /_delta$/ }
-  end
-
-  def reload!
-    @ts_config = @indices = @prefixes = @core_indices = @delta_indices = nil
-  end
-
-  def delta_to_core(delta_name)
-    delta_name.sub(/_delta$/, '_core')
-  end
-
-  def core_to_delta(core_name)
-    core_name.sub(/_core$/, '_delta')
-  end
-
-  def ts_config
-    @ts_config ||= ThinkingSphinx::Configuration.instance.tap do |config|
-      config.generate
-    end
-  end
-
-  private
-  def indices
-    @indices ||= ts_config.configuration.indices.collect { |i| i.name }
-  end
-
-end

data/lib/thinking_sphinx/deltas/resque_delta/railtie.rb

@@ -1,8 +0,0 @@
-require 'rails'
-
-class ThinkingSphinx::Deltas::ResqueDelta::Railtie < Rails::Railtie
-
-  rake_tasks do
-    load File.expand_path('../tasks.rb', __FILE__)
-  end
-end

data/lib/thinking_sphinx/deltas/resque_delta/tasks.rb

@@ -1,38 +0,0 @@
-require 'thinking_sphinx/deltas/resque_delta'
-
-namespace :thinking_sphinx do
-  desc 'Lock all delta indices (Resque will not run indexer or place new jobs on the :ts_delta queue).'
-  task :lock_deltas do
-    ThinkingSphinx::Deltas::ResqueDelta::CoreIndex.new.lock_deltas
-  end
-
-  desc 'Unlock all delta indices.'
-  task :unlock_deltas do
-    ThinkingSphinx::Deltas::ResqueDelta::CoreIndex.new.unlock_deltas
-  end
-
-  desc 'Like `rake thinking_sphinx:index`, but locks one index at a time.'
-  task :smart_index => :app_env do
-    ret = ThinkingSphinx::Deltas::ResqueDelta::CoreIndex.new.smart_index
-
-    abort("Indexing failed.") if ret != true
-  end
-end
-
-namespace :ts do
-  desc 'Like `rake thinking_sphinx:index`, but locks one index at a time.'
-  task :si => 'thinking_sphinx:smart_index'
-end
-
-unless Rake::Task.task_defined?('thinking_sphinx:index')
-  require 'thinking_sphinx/tasks'
-end
-
-# Ensure that indexing does not conflict with ts-resque-delta delta jobs.
-Rake::Task['thinking_sphinx:index'].enhance ['thinking_sphinx:lock_deltas'] do
-  Rake::Task['thinking_sphinx:unlock_deltas'].invoke
-end
-
-Rake::Task['thinking_sphinx:reindex'].enhance ['thinking_sphinx:lock_deltas'] do
-  Rake::Task['thinking_sphinx:unlock_deltas'].invoke
-end

data/lib/thinking_sphinx/deltas/resque_delta/version.rb

@@ -1,7 +0,0 @@
-module ThinkingSphinx
-  module Deltas
-    class ResqueDeltaInfo
-      VERSION = '1.2.4'
-    end
-  end
-end

data/spec/flying_sphinx/resque_delta/delta_job_spec.rb

@@ -1,32 +0,0 @@
-require 'spec_helper'
-
-describe FlyingSphinx::ResqueDelta::DeltaJob do
-  describe '@queue' do
-    it "uses the fs_delta queue" do
-      FlyingSphinx::ResqueDelta::DeltaJob.instance_variable_get(:@queue).
-        should == :fs_delta
-    end
-  end
-
-  describe '.perform' do
-    it "doesn't create an index request when skipping" do
-      FlyingSphinx::ResqueDelta::DeltaJob.stub!(:skip? => true)
-
-      FlyingSphinx::IndexRequest.should_not_receive(:new)
-
-      FlyingSphinx::ResqueDelta::DeltaJob.perform 'foo_delta'
-    end
-
-    it "performs an index request when not skipping" do
-      request = double('index request', :perform => true)
-      FlyingSphinx::ResqueDelta::DeltaJob.stub!(:skip? => false)
-
-      FlyingSphinx::IndexRequest.should_receive(:new).
-        with(['foo_delta']).
-        and_return(request)
-      request.should_receive(:perform)
-
-      FlyingSphinx::ResqueDelta::DeltaJob.perform 'foo_delta'
-    end
-  end
-end

data/spec/flying_sphinx/resque_delta/flag_as_deleted_job_spec.rb

@@ -1,23 +0,0 @@
-require 'spec_helper'
-
-describe FlyingSphinx::ResqueDelta::FlagAsDeletedJob do
-  describe '@queue' do
-    it "uses the fs_delta queue" do
-      FlyingSphinx::ResqueDelta::FlagAsDeletedJob.
-        instance_variable_get(:@queue).should == :fs_delta
-    end
-  end
-  
-  describe '.perform' do
-    it "performs a flag-as-deleted job" do
-      job = double('flag as deleted job', :perform => true)
-      
-      FlyingSphinx::FlagAsDeletedJob.should_receive(:new).
-        with(['foo_core'], 5).
-        and_return(job)
-      job.should_receive(:perform)
-      
-      FlyingSphinx::ResqueDelta::FlagAsDeletedJob.perform ['foo_core'], 5
-    end
-  end
-end

data/spec/flying_sphinx/resque_delta_spec.rb

@@ -1,130 +0,0 @@
-require 'spec_helper'
-
-describe FlyingSphinx::ResqueDelta do
-  describe '.job_types' do
-    it "contains just the Flying Sphinx delta and delete jobs" do
-      FlyingSphinx::ResqueDelta.job_types.should == [
-        FlyingSphinx::ResqueDelta::DeltaJob,
-        FlyingSphinx::ResqueDelta::FlagAsDeletedJob
-      ]
-    end
-  end
-
-  describe '.job_prefix' do
-    it "is fs-delta" do
-      FlyingSphinx::ResqueDelta.job_prefix.should == 'fs-delta'
-    end
-  end
-
-  describe '#index' do
-    before :each do
-      ThinkingSphinx.updates_enabled = true
-      ThinkingSphinx.deltas_enabled  = true
-
-      Resque.stub(:enqueue => true)
-
-      @delayed_delta = FlyingSphinx::ResqueDelta.new(
-        stub('instance'), {}
-      )
-      @delayed_delta.stub(:toggled).and_return(true)
-
-      FlyingSphinx::ResqueDelta.stub(:lock)
-      FlyingSphinx::ResqueDelta.stub(:unlock)
-      FlyingSphinx::ResqueDelta.stub(:locked?).and_return(false)
-
-      @model = stub('foo')
-      @model.stub(:core_index_names  => ['foo_core'])
-      @model.stub(:delta_index_names => ['foo_delta'])
-
-      @instance = stub('instance')
-      @instance.stub(:sphinx_document_id => 42)
-    end
-
-    context 'updates disabled' do
-      before :each do
-        ThinkingSphinx.updates_enabled = false
-      end
-
-      it "should not enqueue a delta job" do
-        Resque.should_not_receive(:enqueue)
-        @delayed_delta.index(@model)
-      end
-
-      it "should not enqueue a flag as deleted job" do
-        Resque.should_not_receive(:enqueue)
-        @delayed_delta.index(@model)
-      end
-    end
-
-    context 'deltas disabled' do
-      before :each do
-        ThinkingSphinx.deltas_enabled = false
-      end
-
-      it "should not enqueue a delta job" do
-        Resque.should_not_receive(:enqueue)
-        @delayed_delta.index(@model)
-      end
-
-      it "should not enqueue a flag as deleted job" do
-        Resque.should_not_receive(:enqueue)
-        @delayed_delta.index(@model)
-      end
-    end
-
-    context "instance isn't toggled" do
-      before :each do
-        @delayed_delta.stub(:toggled => false)
-      end
-
-      it "should not enqueue a delta job" do
-        Resque.should_not_receive(:enqueue)
-        @delayed_delta.index(@model, @instance)
-      end
-
-      it "should not enqueue a flag as deleted job" do
-        Resque.should_not_receive(:enqueue)
-        @delayed_delta.index(@model, @instance)
-      end
-    end
-
-    it "should enqueue a delta job" do
-      Resque.should_receive(:enqueue).at_least(:once).with(
-        FlyingSphinx::ResqueDelta::DeltaJob, 'foo_delta'
-      )
-      @delayed_delta.index(@model)
-    end
-
-    it "should enqueue a flag-as-deleted job" do
-      Resque.should_receive(:enqueue).at_least(:once).with(
-        FlyingSphinx::ResqueDelta::FlagAsDeletedJob,
-        ['foo_core'],
-        42
-      )
-      @delayed_delta.index(@model, @instance)
-    end
-
-    context "delta index is locked" do
-      before :each do
-        FlyingSphinx::ResqueDelta.stub(:locked?).and_return(true)
-      end
-
-      it "should not enqueue a delta job" do
-        Resque.should_not_receive(:enqueue).with(
-          FlyingSphinx::ResqueDelta::DeltaJob,
-          ['foo_delta']
-        )
-        @delayed_delta.index(@model, @instance)
-      end
-
-      it "should enqueue a flag-as-deleted job" do
-        Resque.should_receive(:enqueue).at_least(:once).with(
-          FlyingSphinx::ResqueDelta::FlagAsDeletedJob,
-          ['foo_core'],
-          42
-        )
-        @delayed_delta.index(@model, @instance)
-      end
-    end
-  end
-end