flying-sphinx 0.8.5 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +1 -1
- data/HISTORY +7 -0
- data/flying-sphinx.gemspec +4 -4
- data/lib/flying_sphinx.rb +36 -10
- data/lib/flying_sphinx/action.rb +90 -0
- data/lib/flying_sphinx/api.rb +16 -19
- data/lib/flying_sphinx/binary.rb +7 -0
- data/lib/flying_sphinx/binary/translator.rb +48 -0
- data/lib/flying_sphinx/cli.rb +36 -29
- data/lib/flying_sphinx/configuration.rb +16 -29
- data/lib/flying_sphinx/controller.rb +91 -0
- data/lib/flying_sphinx/rails.rb +1 -11
- data/lib/flying_sphinx/railtie.rb +4 -10
- data/lib/flying_sphinx/setting_files.rb +8 -7
- data/lib/flying_sphinx/sinatra.rb +4 -9
- data/lib/flying_sphinx/sphinxql.rb +7 -0
- data/lib/flying_sphinx/sphinxql/translator.rb +26 -0
- data/lib/flying_sphinx/tasks.rb +5 -8
- data/lib/flying_sphinx/version.rb +1 -1
- data/spec/specs/action_spec.rb +101 -0
- data/spec/specs/api_spec.rb +13 -7
- data/spec/specs/configuration_spec.rb +10 -18
- data/spec/specs/setting_files_spec.rb +27 -31
- metadata +198 -187
- data/lib/flying_sphinx/delayed_delta.rb +0 -89
- data/lib/flying_sphinx/flag_as_deleted_job.rb +0 -41
- data/lib/flying_sphinx/heroku_shared_adapter.rb +0 -5
- data/lib/flying_sphinx/index_request.rb +0 -110
- data/lib/flying_sphinx/sphinx_configuration.rb +0 -30
- data/spec/specs/delayed_delta_spec.rb +0 -147
- data/spec/specs/flag_as_deleted_job_spec.rb +0 -45
- data/spec/specs/index_request_spec.rb +0 -172
- data/spec/specs/sphinx_configuration_spec.rb +0 -42
data/Gemfile
CHANGED
data/HISTORY
CHANGED
@@ -1,3 +1,10 @@
|
|
1
|
+
1.0.0 - 7th May 2012
|
2
|
+
* Updating Riddle dependency to >= 1.5.6.
|
3
|
+
* Support for Thinking Sphinx v1/2/3.
|
4
|
+
* Delayed Job support pushed back to ts-delayed-delta.
|
5
|
+
* Updating MultiJson dependency to ensure MultiJson.load is always available.
|
6
|
+
* All actions are now tracked through Pusher, instead of polling or using slow HTTP requests.
|
7
|
+
|
1
8
|
0.8.5 - 10th December 2012
|
2
9
|
* Daemon actions (start/stop) are now asynchronous.
|
3
10
|
* More forgiving when environment variables aren't around. Particularly helpful for Padrino and Sinatra.
|
data/flying-sphinx.gemspec
CHANGED
@@ -17,11 +17,12 @@ Gem::Specification.new do |s|
|
|
17
17
|
s.require_paths = ['lib']
|
18
18
|
s.executables = ['flying-sphinx']
|
19
19
|
|
20
|
-
s.add_runtime_dependency 'thinking-sphinx'
|
21
|
-
s.add_runtime_dependency 'riddle', ['>= 1.5.
|
22
|
-
s.add_runtime_dependency 'multi_json', ['>= 1.0
|
20
|
+
s.add_runtime_dependency 'thinking-sphinx'
|
21
|
+
s.add_runtime_dependency 'riddle', ['>= 1.5.6']
|
22
|
+
s.add_runtime_dependency 'multi_json', ['>= 1.3.0']
|
23
23
|
s.add_runtime_dependency 'faraday_middleware', ['~> 0.7']
|
24
24
|
s.add_runtime_dependency 'rash', ['~> 0.3.0']
|
25
|
+
s.add_runtime_dependency 'pusher-client', ['~> 0.3']
|
25
26
|
|
26
27
|
s.add_development_dependency 'rake', ['~> 0.9.2']
|
27
28
|
s.add_development_dependency 'rspec', ['~> 2.11']
|
@@ -29,7 +30,6 @@ Gem::Specification.new do |s|
|
|
29
30
|
s.add_development_dependency 'yajl-ruby', ['~> 0.8.2']
|
30
31
|
s.add_development_dependency 'fakeweb', ['~> 1.3.0']
|
31
32
|
s.add_development_dependency 'fakeweb-matcher', ['~> 1.2.2']
|
32
|
-
s.add_development_dependency 'delayed_job', ['~> 2.1.4']
|
33
33
|
|
34
34
|
s.post_install_message = <<-MESSAGE
|
35
35
|
If you're upgrading, you should rebuild your Sphinx setup when deploying:
|
data/lib/flying_sphinx.rb
CHANGED
@@ -1,28 +1,54 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
1
3
|
module FlyingSphinx
|
2
|
-
|
4
|
+
module Translators
|
5
|
+
#
|
6
|
+
end
|
7
|
+
|
8
|
+
@logger = Logger.new(STDOUT)
|
9
|
+
@logger.level = Logger::INFO
|
10
|
+
if ENV['VERBOSE_LOGGING'] && ENV['VERBOSE_LOGGING'] == 'true'
|
11
|
+
@logger.level = Logger::DEBUG
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.logger
|
15
|
+
@logger
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.logger=(logger)
|
19
|
+
@logger = logger
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.translator
|
23
|
+
@translator
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.translator=(translator)
|
27
|
+
@translator = translator
|
28
|
+
end
|
3
29
|
end
|
4
30
|
|
31
|
+
require 'multi_json'
|
5
32
|
require 'faraday'
|
6
33
|
require 'faraday_middleware'
|
7
34
|
require 'riddle'
|
8
35
|
require 'riddle/0.9.9'
|
36
|
+
require 'pusher-client'
|
37
|
+
|
38
|
+
PusherClient.logger = FlyingSphinx.logger
|
9
39
|
|
40
|
+
require 'flying_sphinx/action'
|
10
41
|
require 'flying_sphinx/api'
|
42
|
+
require 'flying_sphinx/binary'
|
11
43
|
require 'flying_sphinx/cli'
|
12
44
|
require 'flying_sphinx/configuration'
|
13
|
-
require 'flying_sphinx/
|
14
|
-
require 'flying_sphinx/index_request'
|
45
|
+
require 'flying_sphinx/controller'
|
15
46
|
require 'flying_sphinx/setting_files'
|
16
|
-
require 'flying_sphinx/
|
47
|
+
require 'flying_sphinx/sphinxql'
|
17
48
|
require 'flying_sphinx/version'
|
18
49
|
|
19
|
-
if defined?(ThinkingSphinx)
|
20
|
-
require 'flying_sphinx/delayed_delta'
|
21
|
-
require 'flying_sphinx/heroku_shared_adapter'
|
22
|
-
end
|
23
|
-
|
24
50
|
if defined?(Rails) && defined?(Rails::Railtie)
|
25
51
|
require 'flying_sphinx/railtie'
|
26
|
-
elsif defined?(Rails)
|
52
|
+
elsif defined?(Rails) && defined?(Rails::Plugin)
|
27
53
|
require 'flying_sphinx/rails'
|
28
54
|
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
require 'timeout'
|
2
|
+
|
3
|
+
class FlyingSphinx::Action
|
4
|
+
def self.perform(identifier, timeout = 60, &block)
|
5
|
+
new(identifier, timeout, &block).perform
|
6
|
+
end
|
7
|
+
|
8
|
+
def initialize(identifier, timeout, &block)
|
9
|
+
@identifier, @timeout, @block = identifier, timeout, block
|
10
|
+
@action_id = 0
|
11
|
+
@finished = false
|
12
|
+
end
|
13
|
+
|
14
|
+
def perform
|
15
|
+
Timeout.timeout(timeout) do
|
16
|
+
socket.connect true
|
17
|
+
|
18
|
+
subscribe_to_events
|
19
|
+
|
20
|
+
sleep 0.5 until socket.connected
|
21
|
+
start
|
22
|
+
sleep 0.5 until finished
|
23
|
+
end
|
24
|
+
|
25
|
+
true
|
26
|
+
rescue Timeout::Error => error
|
27
|
+
FlyingSphinx.logger.warn "Action timed out. If this is happening regularly, please contact Flying Sphinx support: http://support.flying-sphinx.com"
|
28
|
+
|
29
|
+
return false
|
30
|
+
ensure
|
31
|
+
socket.disconnect
|
32
|
+
end
|
33
|
+
|
34
|
+
private
|
35
|
+
|
36
|
+
attr_reader :identifier, :block, :action_id, :finished, :timeout
|
37
|
+
|
38
|
+
def completion(data)
|
39
|
+
FlyingSphinx.logger.debug "Completion: #{data}"
|
40
|
+
data = MultiJson.load(data)
|
41
|
+
|
42
|
+
@finished = (data['id'] == action_id)
|
43
|
+
end
|
44
|
+
|
45
|
+
def debug(data)
|
46
|
+
FlyingSphinx.logger.debug "Progress: #{data}"
|
47
|
+
data = MultiJson.load(data)
|
48
|
+
|
49
|
+
puts data['data'] if data['data'] && data['data'].length > 0
|
50
|
+
end
|
51
|
+
|
52
|
+
def failure(data)
|
53
|
+
FlyingSphinx.logger.debug "Failure: #{data}"
|
54
|
+
data = MultiJson.load(data)
|
55
|
+
|
56
|
+
if data['id'] == action_id
|
57
|
+
FlyingSphinx.logger.warn 'Action failed.'
|
58
|
+
@finished = true
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def response
|
63
|
+
attempts = 0
|
64
|
+
@response ||= begin
|
65
|
+
block.call
|
66
|
+
rescue
|
67
|
+
attempts += 1
|
68
|
+
retry if attempts <= 3
|
69
|
+
raise
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def socket
|
74
|
+
@socket ||= PusherClient::Socket.new FlyingSphinx::API::PUSHER_KEY,
|
75
|
+
:encrypted => true
|
76
|
+
end
|
77
|
+
|
78
|
+
def start
|
79
|
+
raise "Action blocked" if response.body.status == 'BLOCKED'
|
80
|
+
|
81
|
+
@action_id = response.body.id
|
82
|
+
end
|
83
|
+
|
84
|
+
def subscribe_to_events
|
85
|
+
socket.subscribe identifier
|
86
|
+
socket[identifier].bind 'debug', &method(:debug)
|
87
|
+
socket[identifier].bind 'completion', &method(:completion)
|
88
|
+
socket[identifier].bind 'failure', &method(:failure)
|
89
|
+
end
|
90
|
+
end
|
data/lib/flying_sphinx/api.rb
CHANGED
@@ -1,8 +1,16 @@
|
|
1
1
|
class FlyingSphinx::API
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
2
|
+
unless ENV['STAGED_SPHINX_API_KEY']
|
3
|
+
SERVER = 'https://flying-sphinx.com'
|
4
|
+
PUSHER_KEY = 'a8518107ea8a18fe5559'
|
5
|
+
else
|
6
|
+
SERVER = 'https://staging.flying-sphinx.com'
|
7
|
+
PUSHER_KEY = 'c5602d4909b5144321ce'
|
8
|
+
end
|
9
|
+
|
10
|
+
PATH = '/api/my/app'
|
11
|
+
VERSION = 4
|
12
|
+
|
13
|
+
attr_reader :api_key, :identifier
|
6
14
|
|
7
15
|
def initialize(identifier, api_key, adapter = Faraday.default_adapter)
|
8
16
|
@api_key = api_key
|
@@ -32,7 +40,7 @@ class FlyingSphinx::API
|
|
32
40
|
|
33
41
|
private
|
34
42
|
|
35
|
-
attr_reader :
|
43
|
+
attr_reader :adapter
|
36
44
|
|
37
45
|
def normalize_path(path)
|
38
46
|
path = (path == '/' ? '' : "/#{path.gsub(/^\//, '')}")
|
@@ -50,7 +58,7 @@ class FlyingSphinx::API
|
|
50
58
|
def connection(connection_options = {})
|
51
59
|
options = {
|
52
60
|
:ssl => {:verify => false},
|
53
|
-
:url =>
|
61
|
+
:url => SERVER,
|
54
62
|
:headers => api_headers
|
55
63
|
}
|
56
64
|
|
@@ -63,20 +71,9 @@ class FlyingSphinx::API
|
|
63
71
|
end
|
64
72
|
|
65
73
|
def log(method, path, data = {}, option = {}, &block)
|
66
|
-
|
67
|
-
|
68
|
-
log_message "API Request: #{method} '#{path}'; params: #{data.inspect}"
|
74
|
+
FlyingSphinx.logger.debug "API Request: #{method} '#{path}'; params: #{data.inspect}"
|
69
75
|
response = block.call
|
70
|
-
|
76
|
+
FlyingSphinx.logger.debug "API Response: #{response.body.inspect}"
|
71
77
|
return response
|
72
78
|
end
|
73
|
-
|
74
|
-
def log_message(message)
|
75
|
-
time = (Time.respond_to?(:zone) && Time.zone) ? Time.zone.now : Time.now.utc
|
76
|
-
puts "[#{time.to_s}] #{message}"
|
77
|
-
end
|
78
|
-
|
79
|
-
def log?
|
80
|
-
ENV['VERBOSE_LOGGING'] && ENV['VERBOSE_LOGGING'].length > 0
|
81
|
-
end
|
82
79
|
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
class FlyingSphinx::Binary::Translator
|
2
|
+
def initialize(configuration)
|
3
|
+
ThinkingSphinx.remote_sphinx = true
|
4
|
+
|
5
|
+
thinking_sphinx.controller = FlyingSphinx::Controller.new configuration.api
|
6
|
+
thinking_sphinx.address = configuration.host
|
7
|
+
thinking_sphinx.port = configuration.port
|
8
|
+
thinking_sphinx.configuration.searchd.client_key =
|
9
|
+
configuration.client_key
|
10
|
+
|
11
|
+
if ENV['DATABASE_URL'] && ENV['DATABASE_URL'][/^mysql/].nil?
|
12
|
+
ThinkingSphinx.database_adapter = Class.new(ThinkingSphinx::PostgreSQLAdapter) do
|
13
|
+
def setup
|
14
|
+
create_array_accum_function
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def sphinx_configuration
|
21
|
+
@sphinx_configuration ||= begin
|
22
|
+
generate_configuration
|
23
|
+
thinking_sphinx.configuration.searchd.client_key =
|
24
|
+
FlyingSphinx::Configuration.new.client_key
|
25
|
+
thinking_sphinx.configuration.render
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def sphinx_indices
|
30
|
+
@sphinx_indices ||= begin
|
31
|
+
generate_configuration
|
32
|
+
thinking_sphinx.configuration.indices
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def thinking_sphinx
|
39
|
+
ThinkingSphinx::Configuration.instance
|
40
|
+
end
|
41
|
+
|
42
|
+
def generate_configuration
|
43
|
+
return if @generated_configuration
|
44
|
+
|
45
|
+
thinking_sphinx.generate
|
46
|
+
@generated_configuration = true
|
47
|
+
end
|
48
|
+
end
|
data/lib/flying_sphinx/cli.rb
CHANGED
@@ -1,14 +1,20 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
|
1
3
|
class FlyingSphinx::CLI
|
4
|
+
extend Forwardable
|
5
|
+
|
2
6
|
COMMANDS = {
|
3
7
|
'configure' => [:configure],
|
4
8
|
'index' => [:index],
|
5
9
|
'setup' => [:configure, :index],
|
6
10
|
'start' => [:start],
|
7
11
|
'stop' => [:stop],
|
8
|
-
'restart' => [:
|
9
|
-
'rebuild' => [:
|
12
|
+
'restart' => [:restart],
|
13
|
+
'rebuild' => [:rebuild]
|
10
14
|
}
|
11
15
|
|
16
|
+
def_delegators :controller, :start, :stop, :restart
|
17
|
+
|
12
18
|
def initialize(command, arguments = [])
|
13
19
|
@command, @arguments = command, arguments
|
14
20
|
end
|
@@ -17,7 +23,13 @@ class FlyingSphinx::CLI
|
|
17
23
|
methods = COMMANDS[@command]
|
18
24
|
raise "Unknown command #{@command}" if methods.nil?
|
19
25
|
|
20
|
-
methods.all?
|
26
|
+
methods.all? do |method|
|
27
|
+
FlyingSphinx.logger.info "Executing Action: #{method}"
|
28
|
+
result = send method
|
29
|
+
FlyingSphinx.logger.info "Action Finished: #{method}"
|
30
|
+
|
31
|
+
result
|
32
|
+
end
|
21
33
|
end
|
22
34
|
|
23
35
|
private
|
@@ -29,50 +41,45 @@ class FlyingSphinx::CLI
|
|
29
41
|
def configure
|
30
42
|
if @arguments.empty?
|
31
43
|
load_rails
|
32
|
-
|
33
|
-
|
44
|
+
|
45
|
+
controller.configure
|
34
46
|
else
|
35
|
-
|
36
|
-
@arguments.first
|
47
|
+
controller.configure File.read(@arguments.first)
|
37
48
|
end
|
38
49
|
|
39
|
-
puts "Sent configuration to Sphinx"
|
40
50
|
true
|
41
51
|
end
|
42
52
|
|
43
|
-
def
|
44
|
-
FlyingSphinx::
|
45
|
-
|
46
|
-
request = FlyingSphinx::IndexRequest.new @arguments
|
47
|
-
request.index
|
48
|
-
puts request.status_message
|
53
|
+
def controller
|
54
|
+
@controller ||= FlyingSphinx::Controller.new configuration.api
|
55
|
+
end
|
49
56
|
|
50
|
-
|
57
|
+
def index
|
58
|
+
indices = @arguments + [{:verbose => true}]
|
59
|
+
controller.index *indices
|
51
60
|
end
|
52
61
|
|
53
62
|
def load_rails
|
54
63
|
return unless ENV['RAILS_ENV']
|
55
64
|
|
56
65
|
require File.expand_path('config/boot', Dir.pwd)
|
57
|
-
require File.expand_path('config/application', Dir.pwd)
|
58
|
-
Rails.application.require_environment!
|
59
66
|
|
60
|
-
|
61
|
-
|
67
|
+
if defined?(Rails) && !defined?(Rails::Railtie)
|
68
|
+
require File.expand_path('config/environment', Dir.pwd)
|
69
|
+
require 'flying_sphinx/rails'
|
62
70
|
|
63
|
-
|
64
|
-
if configuration.start_sphinx
|
65
|
-
puts "Started Sphinx"
|
66
|
-
true
|
71
|
+
FlyingSphinx::Binary.load
|
67
72
|
else
|
68
|
-
|
69
|
-
|
73
|
+
require File.expand_path('config/application', Dir.pwd)
|
74
|
+
require 'flying_sphinx/railtie'
|
75
|
+
|
76
|
+
Rails.application.require_environment!
|
70
77
|
end
|
71
78
|
end
|
72
79
|
|
73
|
-
def
|
74
|
-
|
75
|
-
|
76
|
-
|
80
|
+
def rebuild
|
81
|
+
load_rails
|
82
|
+
|
83
|
+
controller.rebuild
|
77
84
|
end
|
78
85
|
end
|
@@ -1,39 +1,33 @@
|
|
1
1
|
class FlyingSphinx::Configuration
|
2
|
-
attr_reader :host, :port, :ssh_server, :database_port
|
3
|
-
|
4
2
|
def initialize(identifier = nil, api_key = nil)
|
5
3
|
@identifier = identifier || identifier_from_env
|
6
4
|
@api_key = api_key || api_key_from_env
|
7
|
-
|
8
|
-
set_from_server
|
9
5
|
end
|
10
6
|
|
11
7
|
def api
|
12
8
|
@api ||= FlyingSphinx::API.new(identifier, api_key)
|
13
9
|
end
|
14
10
|
|
15
|
-
def
|
16
|
-
|
11
|
+
def client_key
|
12
|
+
"#{identifier}:#{api_key}"
|
17
13
|
end
|
18
14
|
|
19
|
-
def
|
20
|
-
|
15
|
+
def host
|
16
|
+
@host ||= response_body.server rescue host_from_env
|
21
17
|
end
|
22
18
|
|
23
|
-
def
|
24
|
-
|
19
|
+
def port
|
20
|
+
@port ||= response_body.port rescue port_from_env
|
25
21
|
end
|
26
22
|
|
27
|
-
def
|
28
|
-
|
29
|
-
puts "#{action.created_at} #{action.name}"
|
30
|
-
end
|
23
|
+
def username
|
24
|
+
"#{identifier}#{api_key}"
|
31
25
|
end
|
32
26
|
|
33
27
|
private
|
34
28
|
|
35
29
|
attr_reader :identifier, :api_key
|
36
|
-
|
30
|
+
|
37
31
|
def change(initial, expected)
|
38
32
|
api.post(initial)
|
39
33
|
|
@@ -46,19 +40,12 @@ class FlyingSphinx::Configuration
|
|
46
40
|
response.body.status == expected
|
47
41
|
end
|
48
42
|
|
49
|
-
def
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
@ssh_server = response.body.ssh_server
|
56
|
-
@database_port = response.body.database_port
|
57
|
-
rescue
|
58
|
-
# If the central Flying Sphinx server is down, let's use the environment
|
59
|
-
# variables so searching is still going to work.
|
60
|
-
@host = host_from_env
|
61
|
-
@port = port_from_env
|
43
|
+
def response_body
|
44
|
+
@response_body ||= begin
|
45
|
+
response = api.get '/'
|
46
|
+
raise 'Invalid Flying Sphinx credentials' if response.status == 403
|
47
|
+
response.body
|
48
|
+
end
|
62
49
|
end
|
63
50
|
|
64
51
|
def identifier_from_env
|
@@ -74,6 +61,6 @@ class FlyingSphinx::Configuration
|
|
74
61
|
end
|
75
62
|
|
76
63
|
def port_from_env
|
77
|
-
(ENV['STAGED_SPHINX_PORT'] || ENV['FLYING_SPHINX_PORT'] || 9306).dup
|
64
|
+
(ENV['STAGED_SPHINX_PORT'] || ENV['FLYING_SPHINX_PORT'] || '9306').dup
|
78
65
|
end
|
79
66
|
end
|