flying-sphinx 0.8.5 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Gemfile CHANGED
@@ -1,3 +1,3 @@
1
- source :rubygems
1
+ source 'https://rubygems.org'
2
2
 
3
3
  gemspec
data/HISTORY CHANGED
@@ -1,3 +1,10 @@
1
+ 1.0.0 - 7th May 2012
2
+ * Updating Riddle dependency to >= 1.5.6.
3
+ * Support for Thinking Sphinx v1/2/3.
4
+ * Delayed Job support pushed back to ts-delayed-delta.
5
+ * Updating MultiJson dependency to ensure MultiJson.load is always available.
6
+ * All actions are now tracked through Pusher, instead of polling or using slow HTTP requests.
7
+
1
8
  0.8.5 - 10th December 2012
2
9
  * Daemon actions (start/stop) are now asynchronous.
3
10
  * More forgiving when environment variables aren't around. Particularly helpful for Padrino and Sinatra.
@@ -17,11 +17,12 @@ Gem::Specification.new do |s|
17
17
  s.require_paths = ['lib']
18
18
  s.executables = ['flying-sphinx']
19
19
 
20
- s.add_runtime_dependency 'thinking-sphinx', ['>= 0']
21
- s.add_runtime_dependency 'riddle', ['>= 1.5.0']
22
- s.add_runtime_dependency 'multi_json', ['>= 1.0.1']
20
+ s.add_runtime_dependency 'thinking-sphinx'
21
+ s.add_runtime_dependency 'riddle', ['>= 1.5.6']
22
+ s.add_runtime_dependency 'multi_json', ['>= 1.3.0']
23
23
  s.add_runtime_dependency 'faraday_middleware', ['~> 0.7']
24
24
  s.add_runtime_dependency 'rash', ['~> 0.3.0']
25
+ s.add_runtime_dependency 'pusher-client', ['~> 0.3']
25
26
 
26
27
  s.add_development_dependency 'rake', ['~> 0.9.2']
27
28
  s.add_development_dependency 'rspec', ['~> 2.11']
@@ -29,7 +30,6 @@ Gem::Specification.new do |s|
29
30
  s.add_development_dependency 'yajl-ruby', ['~> 0.8.2']
30
31
  s.add_development_dependency 'fakeweb', ['~> 1.3.0']
31
32
  s.add_development_dependency 'fakeweb-matcher', ['~> 1.2.2']
32
- s.add_development_dependency 'delayed_job', ['~> 2.1.4']
33
33
 
34
34
  s.post_install_message = <<-MESSAGE
35
35
  If you're upgrading, you should rebuild your Sphinx setup when deploying:
data/lib/flying_sphinx.rb CHANGED
@@ -1,28 +1,54 @@
1
+ require 'logger'
2
+
1
3
  module FlyingSphinx
2
- #
4
+ module Translators
5
+ #
6
+ end
7
+
8
+ @logger = Logger.new(STDOUT)
9
+ @logger.level = Logger::INFO
10
+ if ENV['VERBOSE_LOGGING'] && ENV['VERBOSE_LOGGING'] == 'true'
11
+ @logger.level = Logger::DEBUG
12
+ end
13
+
14
+ def self.logger
15
+ @logger
16
+ end
17
+
18
+ def self.logger=(logger)
19
+ @logger = logger
20
+ end
21
+
22
+ def self.translator
23
+ @translator
24
+ end
25
+
26
+ def self.translator=(translator)
27
+ @translator = translator
28
+ end
3
29
  end
4
30
 
31
+ require 'multi_json'
5
32
  require 'faraday'
6
33
  require 'faraday_middleware'
7
34
  require 'riddle'
8
35
  require 'riddle/0.9.9'
36
+ require 'pusher-client'
37
+
38
+ PusherClient.logger = FlyingSphinx.logger
9
39
 
40
+ require 'flying_sphinx/action'
10
41
  require 'flying_sphinx/api'
42
+ require 'flying_sphinx/binary'
11
43
  require 'flying_sphinx/cli'
12
44
  require 'flying_sphinx/configuration'
13
- require 'flying_sphinx/flag_as_deleted_job'
14
- require 'flying_sphinx/index_request'
45
+ require 'flying_sphinx/controller'
15
46
  require 'flying_sphinx/setting_files'
16
- require 'flying_sphinx/sphinx_configuration'
47
+ require 'flying_sphinx/sphinxql'
17
48
  require 'flying_sphinx/version'
18
49
 
19
- if defined?(ThinkingSphinx)
20
- require 'flying_sphinx/delayed_delta'
21
- require 'flying_sphinx/heroku_shared_adapter'
22
- end
23
-
24
50
  if defined?(Rails) && defined?(Rails::Railtie)
25
51
  require 'flying_sphinx/railtie'
26
- elsif defined?(Rails)
52
+ elsif defined?(Rails) && defined?(Rails::Plugin)
27
53
  require 'flying_sphinx/rails'
28
54
  end
@@ -0,0 +1,90 @@
1
+ require 'timeout'
2
+
3
+ class FlyingSphinx::Action
4
+ def self.perform(identifier, timeout = 60, &block)
5
+ new(identifier, timeout, &block).perform
6
+ end
7
+
8
+ def initialize(identifier, timeout, &block)
9
+ @identifier, @timeout, @block = identifier, timeout, block
10
+ @action_id = 0
11
+ @finished = false
12
+ end
13
+
14
+ def perform
15
+ Timeout.timeout(timeout) do
16
+ socket.connect true
17
+
18
+ subscribe_to_events
19
+
20
+ sleep 0.5 until socket.connected
21
+ start
22
+ sleep 0.5 until finished
23
+ end
24
+
25
+ true
26
+ rescue Timeout::Error => error
27
+ FlyingSphinx.logger.warn "Action timed out. If this is happening regularly, please contact Flying Sphinx support: http://support.flying-sphinx.com"
28
+
29
+ return false
30
+ ensure
31
+ socket.disconnect
32
+ end
33
+
34
+ private
35
+
36
+ attr_reader :identifier, :block, :action_id, :finished, :timeout
37
+
38
+ def completion(data)
39
+ FlyingSphinx.logger.debug "Completion: #{data}"
40
+ data = MultiJson.load(data)
41
+
42
+ @finished = (data['id'] == action_id)
43
+ end
44
+
45
+ def debug(data)
46
+ FlyingSphinx.logger.debug "Progress: #{data}"
47
+ data = MultiJson.load(data)
48
+
49
+ puts data['data'] if data['data'] && data['data'].length > 0
50
+ end
51
+
52
+ def failure(data)
53
+ FlyingSphinx.logger.debug "Failure: #{data}"
54
+ data = MultiJson.load(data)
55
+
56
+ if data['id'] == action_id
57
+ FlyingSphinx.logger.warn 'Action failed.'
58
+ @finished = true
59
+ end
60
+ end
61
+
62
+ def response
63
+ attempts = 0
64
+ @response ||= begin
65
+ block.call
66
+ rescue
67
+ attempts += 1
68
+ retry if attempts <= 3
69
+ raise
70
+ end
71
+ end
72
+
73
+ def socket
74
+ @socket ||= PusherClient::Socket.new FlyingSphinx::API::PUSHER_KEY,
75
+ :encrypted => true
76
+ end
77
+
78
+ def start
79
+ raise "Action blocked" if response.body.status == 'BLOCKED'
80
+
81
+ @action_id = response.body.id
82
+ end
83
+
84
+ def subscribe_to_events
85
+ socket.subscribe identifier
86
+ socket[identifier].bind 'debug', &method(:debug)
87
+ socket[identifier].bind 'completion', &method(:completion)
88
+ socket[identifier].bind 'failure', &method(:failure)
89
+ end
90
+ end
@@ -1,8 +1,16 @@
1
1
  class FlyingSphinx::API
2
- SERVER = 'https://flying-sphinx.com'
3
- STAGING_SERVER = 'https://staging.flying-sphinx.com'
4
- PATH = "/api/my/app"
5
- VERSION = 3
2
+ unless ENV['STAGED_SPHINX_API_KEY']
3
+ SERVER = 'https://flying-sphinx.com'
4
+ PUSHER_KEY = 'a8518107ea8a18fe5559'
5
+ else
6
+ SERVER = 'https://staging.flying-sphinx.com'
7
+ PUSHER_KEY = 'c5602d4909b5144321ce'
8
+ end
9
+
10
+ PATH = '/api/my/app'
11
+ VERSION = 4
12
+
13
+ attr_reader :api_key, :identifier
6
14
 
7
15
  def initialize(identifier, api_key, adapter = Faraday.default_adapter)
8
16
  @api_key = api_key
@@ -32,7 +40,7 @@ class FlyingSphinx::API
32
40
 
33
41
  private
34
42
 
35
- attr_reader :api_key, :identifier, :adapter
43
+ attr_reader :adapter
36
44
 
37
45
  def normalize_path(path)
38
46
  path = (path == '/' ? '' : "/#{path.gsub(/^\//, '')}")
@@ -50,7 +58,7 @@ class FlyingSphinx::API
50
58
  def connection(connection_options = {})
51
59
  options = {
52
60
  :ssl => {:verify => false},
53
- :url => (ENV['STAGED_SPHINX_API_KEY'] ? STAGING_SERVER : SERVER),
61
+ :url => SERVER,
54
62
  :headers => api_headers
55
63
  }
56
64
 
@@ -63,20 +71,9 @@ class FlyingSphinx::API
63
71
  end
64
72
 
65
73
  def log(method, path, data = {}, option = {}, &block)
66
- return block.call unless log?
67
-
68
- log_message "API Request: #{method} '#{path}'; params: #{data.inspect}"
74
+ FlyingSphinx.logger.debug "API Request: #{method} '#{path}'; params: #{data.inspect}"
69
75
  response = block.call
70
- log_message "API Response: #{response.body.inspect}"
76
+ FlyingSphinx.logger.debug "API Response: #{response.body.inspect}"
71
77
  return response
72
78
  end
73
-
74
- def log_message(message)
75
- time = (Time.respond_to?(:zone) && Time.zone) ? Time.zone.now : Time.now.utc
76
- puts "[#{time.to_s}] #{message}"
77
- end
78
-
79
- def log?
80
- ENV['VERBOSE_LOGGING'] && ENV['VERBOSE_LOGGING'].length > 0
81
- end
82
79
  end
@@ -0,0 +1,7 @@
1
+ module FlyingSphinx::Binary
2
+ def self.load
3
+ require 'flying_sphinx/binary/translator'
4
+
5
+ FlyingSphinx.translator = Translator.new FlyingSphinx::Configuration.new
6
+ end
7
+ end
@@ -0,0 +1,48 @@
1
+ class FlyingSphinx::Binary::Translator
2
+ def initialize(configuration)
3
+ ThinkingSphinx.remote_sphinx = true
4
+
5
+ thinking_sphinx.controller = FlyingSphinx::Controller.new configuration.api
6
+ thinking_sphinx.address = configuration.host
7
+ thinking_sphinx.port = configuration.port
8
+ thinking_sphinx.configuration.searchd.client_key =
9
+ configuration.client_key
10
+
11
+ if ENV['DATABASE_URL'] && ENV['DATABASE_URL'][/^mysql/].nil?
12
+ ThinkingSphinx.database_adapter = Class.new(ThinkingSphinx::PostgreSQLAdapter) do
13
+ def setup
14
+ create_array_accum_function
15
+ end
16
+ end
17
+ end
18
+ end
19
+
20
+ def sphinx_configuration
21
+ @sphinx_configuration ||= begin
22
+ generate_configuration
23
+ thinking_sphinx.configuration.searchd.client_key =
24
+ FlyingSphinx::Configuration.new.client_key
25
+ thinking_sphinx.configuration.render
26
+ end
27
+ end
28
+
29
+ def sphinx_indices
30
+ @sphinx_indices ||= begin
31
+ generate_configuration
32
+ thinking_sphinx.configuration.indices
33
+ end
34
+ end
35
+
36
+ private
37
+
38
+ def thinking_sphinx
39
+ ThinkingSphinx::Configuration.instance
40
+ end
41
+
42
+ def generate_configuration
43
+ return if @generated_configuration
44
+
45
+ thinking_sphinx.generate
46
+ @generated_configuration = true
47
+ end
48
+ end
@@ -1,14 +1,20 @@
1
+ require 'forwardable'
2
+
1
3
  class FlyingSphinx::CLI
4
+ extend Forwardable
5
+
2
6
  COMMANDS = {
3
7
  'configure' => [:configure],
4
8
  'index' => [:index],
5
9
  'setup' => [:configure, :index],
6
10
  'start' => [:start],
7
11
  'stop' => [:stop],
8
- 'restart' => [:stop, :start],
9
- 'rebuild' => [:stop, :configure, :index, :start]
12
+ 'restart' => [:restart],
13
+ 'rebuild' => [:rebuild]
10
14
  }
11
15
 
16
+ def_delegators :controller, :start, :stop, :restart
17
+
12
18
  def initialize(command, arguments = [])
13
19
  @command, @arguments = command, arguments
14
20
  end
@@ -17,7 +23,13 @@ class FlyingSphinx::CLI
17
23
  methods = COMMANDS[@command]
18
24
  raise "Unknown command #{@command}" if methods.nil?
19
25
 
20
- methods.all? { |method| send method }
26
+ methods.all? do |method|
27
+ FlyingSphinx.logger.info "Executing Action: #{method}"
28
+ result = send method
29
+ FlyingSphinx.logger.info "Action Finished: #{method}"
30
+
31
+ result
32
+ end
21
33
  end
22
34
 
23
35
  private
@@ -29,50 +41,45 @@ class FlyingSphinx::CLI
29
41
  def configure
30
42
  if @arguments.empty?
31
43
  load_rails
32
- FlyingSphinx::SphinxConfiguration.new.upload_to configuration.api
33
- FlyingSphinx::SettingFiles.new.upload_to configuration.api
44
+
45
+ controller.configure
34
46
  else
35
- FlyingSphinx::SphinxConfiguration.new.upload_file_to configuration.api,
36
- @arguments.first
47
+ controller.configure File.read(@arguments.first)
37
48
  end
38
49
 
39
- puts "Sent configuration to Sphinx"
40
50
  true
41
51
  end
42
52
 
43
- def index
44
- FlyingSphinx::IndexRequest.cancel_jobs
45
-
46
- request = FlyingSphinx::IndexRequest.new @arguments
47
- request.index
48
- puts request.status_message
53
+ def controller
54
+ @controller ||= FlyingSphinx::Controller.new configuration.api
55
+ end
49
56
 
50
- true
57
+ def index
58
+ indices = @arguments + [{:verbose => true}]
59
+ controller.index *indices
51
60
  end
52
61
 
53
62
  def load_rails
54
63
  return unless ENV['RAILS_ENV']
55
64
 
56
65
  require File.expand_path('config/boot', Dir.pwd)
57
- require File.expand_path('config/application', Dir.pwd)
58
- Rails.application.require_environment!
59
66
 
60
- require 'flying_sphinx/delayed_delta'
61
- end
67
+ if defined?(Rails) && !defined?(Rails::Railtie)
68
+ require File.expand_path('config/environment', Dir.pwd)
69
+ require 'flying_sphinx/rails'
62
70
 
63
- def start
64
- if configuration.start_sphinx
65
- puts "Started Sphinx"
66
- true
71
+ FlyingSphinx::Binary.load
67
72
  else
68
- puts "Sphinx failed to start... have you indexed first?"
69
- false
73
+ require File.expand_path('config/application', Dir.pwd)
74
+ require 'flying_sphinx/railtie'
75
+
76
+ Rails.application.require_environment!
70
77
  end
71
78
  end
72
79
 
73
- def stop
74
- configuration.stop_sphinx
75
- puts "Stopped Sphinx"
76
- true
80
+ def rebuild
81
+ load_rails
82
+
83
+ controller.rebuild
77
84
  end
78
85
  end
@@ -1,39 +1,33 @@
1
1
  class FlyingSphinx::Configuration
2
- attr_reader :host, :port, :ssh_server, :database_port
3
-
4
2
  def initialize(identifier = nil, api_key = nil)
5
3
  @identifier = identifier || identifier_from_env
6
4
  @api_key = api_key || api_key_from_env
7
-
8
- set_from_server
9
5
  end
10
6
 
11
7
  def api
12
8
  @api ||= FlyingSphinx::API.new(identifier, api_key)
13
9
  end
14
10
 
15
- def start_sphinx
16
- change 'starting', 'started'
11
+ def client_key
12
+ "#{identifier}:#{api_key}"
17
13
  end
18
14
 
19
- def stop_sphinx
20
- change 'stopping', 'stopped'
15
+ def host
16
+ @host ||= response_body.server rescue host_from_env
21
17
  end
22
18
 
23
- def client_key
24
- "#{identifier}:#{api_key}"
19
+ def port
20
+ @port ||= response_body.port rescue port_from_env
25
21
  end
26
22
 
27
- def output_recent_actions
28
- api.get('actions').body.each do |action|
29
- puts "#{action.created_at} #{action.name}"
30
- end
23
+ def username
24
+ "#{identifier}#{api_key}"
31
25
  end
32
26
 
33
27
  private
34
28
 
35
29
  attr_reader :identifier, :api_key
36
-
30
+
37
31
  def change(initial, expected)
38
32
  api.post(initial)
39
33
 
@@ -46,19 +40,12 @@ class FlyingSphinx::Configuration
46
40
  response.body.status == expected
47
41
  end
48
42
 
49
- def set_from_server
50
- response = api.get '/'
51
- raise 'Invalid Flying Sphinx credentials' if response.status == 403
52
-
53
- @host = response.body.server
54
- @port = response.body.port
55
- @ssh_server = response.body.ssh_server
56
- @database_port = response.body.database_port
57
- rescue
58
- # If the central Flying Sphinx server is down, let's use the environment
59
- # variables so searching is still going to work.
60
- @host = host_from_env
61
- @port = port_from_env
43
+ def response_body
44
+ @response_body ||= begin
45
+ response = api.get '/'
46
+ raise 'Invalid Flying Sphinx credentials' if response.status == 403
47
+ response.body
48
+ end
62
49
  end
63
50
 
64
51
  def identifier_from_env
@@ -74,6 +61,6 @@ class FlyingSphinx::Configuration
74
61
  end
75
62
 
76
63
  def port_from_env
77
- (ENV['STAGED_SPHINX_PORT'] || ENV['FLYING_SPHINX_PORT'] || 9306).dup
64
+ (ENV['STAGED_SPHINX_PORT'] || ENV['FLYING_SPHINX_PORT'] || '9306').dup
78
65
  end
79
66
  end