supernova-core 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +17 -0
- data/Gemfile +4 -0
- data/LICENSE +22 -0
- data/README.md +29 -0
- data/Rakefile +10 -0
- data/bin/solr_admin +5 -0
- data/lib/functional_delegator.rb +44 -0
- data/lib/supernova-core.rb +7 -0
- data/lib/supernova-core/version.rb +5 -0
- data/lib/supernova.rb +18 -0
- data/lib/supernova/batch_indexer.rb +40 -0
- data/lib/supernova/solr/cli.rb +169 -0
- data/lib/supernova/solr/core.rb +41 -0
- data/lib/supernova/solr/mapper.rb +31 -0
- data/lib/supernova/solr/server.rb +108 -0
- data/solr/conf/admin-extra.html +31 -0
- data/solr/conf/elevate.xml +36 -0
- data/solr/conf/mapping-FoldToASCII.txt +3813 -0
- data/solr/conf/mapping-ISOLatin1Accent.txt +246 -0
- data/solr/conf/protwords.txt +21 -0
- data/solr/conf/schema.xml +477 -0
- data/solr/conf/scripts.conf +24 -0
- data/solr/conf/solrconfig.xml +1508 -0
- data/solr/conf/spellings.txt +2 -0
- data/solr/conf/stopwords.txt +58 -0
- data/solr/conf/synonyms.txt +29 -0
- data/solr/conf/velocity/VM_global_library.vm +184 -0
- data/solr/conf/velocity/browse.vm +45 -0
- data/solr/conf/velocity/cluster.vm +26 -0
- data/solr/conf/velocity/clusterResults.vm +29 -0
- data/solr/conf/velocity/doc.vm +29 -0
- data/solr/conf/velocity/facet_dates.vm +0 -0
- data/solr/conf/velocity/facet_fields.vm +12 -0
- data/solr/conf/velocity/facet_queries.vm +3 -0
- data/solr/conf/velocity/facet_ranges.vm +30 -0
- data/solr/conf/velocity/facets.vm +7 -0
- data/solr/conf/velocity/footer.vm +17 -0
- data/solr/conf/velocity/head.vm +45 -0
- data/solr/conf/velocity/header.vm +3 -0
- data/solr/conf/velocity/hit.vm +5 -0
- data/solr/conf/velocity/jquery.autocomplete.css +48 -0
- data/solr/conf/velocity/jquery.autocomplete.js +762 -0
- data/solr/conf/velocity/layout.vm +20 -0
- data/solr/conf/velocity/main.css +184 -0
- data/solr/conf/velocity/query.vm +56 -0
- data/solr/conf/velocity/querySpatial.vm +40 -0
- data/solr/conf/velocity/suggest.vm +3 -0
- data/solr/conf/velocity/tabs.vm +22 -0
- data/solr/conf/xslt/example.xsl +132 -0
- data/solr/conf/xslt/example_atom.xsl +67 -0
- data/solr/conf/xslt/example_rss.xsl +66 -0
- data/solr/conf/xslt/luke.xsl +337 -0
- data/spec/fixtures/cores_create.json +1 -0
- data/spec/fixtures/cores_status.json +1 -0
- data/spec/integration/supernova_core_spec.rb +95 -0
- data/spec/spec_helper.rb +30 -0
- data/spec/supernova/batch_indexer_spec.rb +69 -0
- data/spec/supernova/solr/cli_spec.rb +26 -0
- data/spec/supernova/solr/core_spec.rb +81 -0
- data/spec/supernova/solr/mapper_spec.rb +25 -0
- data/spec/supernova/solr/server_spec.rb +144 -0
- data/spec/supernova_spec.rb +31 -0
- data/supernova-core.gemspec +25 -0
- metadata +237 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 Tobias Schwab
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# Supernova::Core
|
2
|
+
|
3
|
+
TODO: Write a gem description
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'supernova-core'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install supernova-core
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
TODO: Write usage instructions here
|
22
|
+
|
23
|
+
## Contributing
|
24
|
+
|
25
|
+
1. Fork it
|
26
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
27
|
+
3. Commit your changes (`git commit -am 'Added some feature'`)
|
28
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
29
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
data/bin/solr_admin
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
module FunctionalDelegator
|
2
|
+
module ClassMethods
|
3
|
+
# class Page
|
4
|
+
# attr_reader :url, :protocol
|
5
|
+
#
|
6
|
+
# def initialize(url, protocol)
|
7
|
+
# @url = url
|
8
|
+
# @protocol = protocol
|
9
|
+
# end
|
10
|
+
#
|
11
|
+
# def self.get(url)
|
12
|
+
# Net::HTTP.get(URI.parse(url))
|
13
|
+
# end
|
14
|
+
#
|
15
|
+
# def self.post(url, body)
|
16
|
+
# Net::HTTP.post(URI.parse(url))
|
17
|
+
# end
|
18
|
+
#
|
19
|
+
# def self.head(url, protocol)
|
20
|
+
# Net::HTTP.head(URI.parse(url), protocol)
|
21
|
+
# end
|
22
|
+
#
|
23
|
+
# functional_delegate :get, :post, :attributes => :url
|
24
|
+
# functional_delegate :head, :attributes => [:url, :protocol]
|
25
|
+
# end
|
26
|
+
#
|
27
|
+
# Page.new("http://www.dynport.de").get # delegates to Page.get("http://www.dynport.de")
|
28
|
+
# Page.new("http://www.dynport.de").post("the body") # delegates to Page.post("http://www.dynport.de", "the body")
|
29
|
+
# Page.new("http://www.dynport.de", "HTTP/1.1").head # delegates to Page.head("http://www.dynport.de", "HTTP/1.1")
|
30
|
+
def functional_delegate(*keys)
|
31
|
+
raise "ERROR: last argument must be e.g. :attributes => [:url]" if !keys.last.is_a?(Hash) || !keys.last.has_key?(:attributes)
|
32
|
+
keys[0..-2].each do |method|
|
33
|
+
define_method(method) do |*args|
|
34
|
+
all_args = [keys.last[:attributes]].flatten.map { |att| self.send(att) } + args
|
35
|
+
self.class.send(method, *all_args)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.included(base)
|
42
|
+
base.extend(ClassMethods)
|
43
|
+
end
|
44
|
+
end
|
data/lib/supernova.rb
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
module Supernova
|
2
|
+
class << self
|
3
|
+
attr_accessor :logger
|
4
|
+
private :logger
|
5
|
+
|
6
|
+
def log(message, method = :info)
|
7
|
+
logger.send(method, message) if logger
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
module Solr
|
12
|
+
class << self
|
13
|
+
def remove_trailing_slash(url)
|
14
|
+
url.gsub(/[\/]+$/, "")
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
class Supernova::BatchIndexer
|
2
|
+
def initialize(index, batch_size = 1_000)
|
3
|
+
@index = index
|
4
|
+
@batch_size = batch_size
|
5
|
+
@docs = []
|
6
|
+
if block_given?
|
7
|
+
yield(self)
|
8
|
+
finish
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def index_doc(doc)
|
13
|
+
@docs << doc
|
14
|
+
if should_index?
|
15
|
+
do_index
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def finish
|
20
|
+
do_index
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def should_index?
|
26
|
+
@docs.count >= @batch_size
|
27
|
+
end
|
28
|
+
|
29
|
+
def do_index
|
30
|
+
started = Time.now
|
31
|
+
@index.index_docs(@docs) if @docs.any?
|
32
|
+
Supernova.log(do_index_log_message(started, Time.now, @docs))
|
33
|
+
@docs = []
|
34
|
+
end
|
35
|
+
|
36
|
+
def do_index_log_message(started, finished, docs)
|
37
|
+
diff = finished - started
|
38
|
+
"indexed #{docs.count} docs in %.3f seconds (%.1f docs/second)" % [diff, docs.count / diff]
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,169 @@
|
|
1
|
+
require "optparse"
|
2
|
+
require "supernova"
|
3
|
+
require "supernova/solr/server"
|
4
|
+
require "supernova/solr/core"
|
5
|
+
|
6
|
+
class Supernova::Solr::CLI
|
7
|
+
SOLR_URL = "http://localhost:8983/solr"
|
8
|
+
SOLR_VERSION = "3.5.0"
|
9
|
+
|
10
|
+
attr_accessor :options
|
11
|
+
|
12
|
+
def initialize(args)
|
13
|
+
self.options = {}
|
14
|
+
opts.parse(args)
|
15
|
+
end
|
16
|
+
|
17
|
+
def opts
|
18
|
+
OptionParser.new do |o|
|
19
|
+
o.on("--core CORE", String) do |core_name|
|
20
|
+
self.options[:core] = core_name
|
21
|
+
end
|
22
|
+
|
23
|
+
o.on("--server SERVER", String) do |server_name|
|
24
|
+
self.options[:server] = server_name
|
25
|
+
end
|
26
|
+
|
27
|
+
o.separator ""
|
28
|
+
o.separator "Server ACTIONS"
|
29
|
+
|
30
|
+
|
31
|
+
o.on("--start-server", "Start SOLR server") do |solr_bin|
|
32
|
+
self.options[:action] = :start
|
33
|
+
end
|
34
|
+
|
35
|
+
o.on("--server-status", "SOLR server status") do |solr_bin|
|
36
|
+
self.options[:action] = :server_status
|
37
|
+
end
|
38
|
+
|
39
|
+
o.on("--stop-server", "Stop SOLR server") do |solr_bin|
|
40
|
+
self.options[:action] = :stop
|
41
|
+
end
|
42
|
+
|
43
|
+
o.separator ""
|
44
|
+
o.separator "Core ACTIONS"
|
45
|
+
|
46
|
+
o.on("--list-cores", "List Cores") do
|
47
|
+
self.options[:action] = :list
|
48
|
+
end
|
49
|
+
|
50
|
+
o.on("--create", "Create Core") do
|
51
|
+
self.options[:action] = :create
|
52
|
+
end
|
53
|
+
|
54
|
+
o.on("--commit", "Commit Core") do
|
55
|
+
self.options[:action] = :commit
|
56
|
+
end
|
57
|
+
|
58
|
+
o.on("--optimize", "Optimize Core") do
|
59
|
+
self.options[:action] = :optimize
|
60
|
+
end
|
61
|
+
|
62
|
+
o.on("--unload", "Unload Core") do
|
63
|
+
self.options[:action] = :unload
|
64
|
+
end
|
65
|
+
|
66
|
+
o.separator ""
|
67
|
+
o.separator "OPTIONS"
|
68
|
+
|
69
|
+
o.on("--solr-bin SOLR_BIN", "Path to SOLR jarfile") do |solr_bin|
|
70
|
+
self.options[:solr_bin] = solr_bin
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def core
|
76
|
+
Supernova::Solr::Core.new(solr_url, core_name)
|
77
|
+
end
|
78
|
+
|
79
|
+
def server
|
80
|
+
Supernova::Solr::Server.new(solr_url)
|
81
|
+
end
|
82
|
+
|
83
|
+
def execute
|
84
|
+
if action = options[:action]
|
85
|
+
send(action)
|
86
|
+
else
|
87
|
+
puts opts
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def commit
|
92
|
+
handle_response core.commit
|
93
|
+
end
|
94
|
+
|
95
|
+
def optimize
|
96
|
+
handle_response core.optimize
|
97
|
+
end
|
98
|
+
|
99
|
+
def unload
|
100
|
+
handle_response core
|
101
|
+
end
|
102
|
+
|
103
|
+
def create
|
104
|
+
handle_response core.create(options[:data], options[:instance])
|
105
|
+
end
|
106
|
+
|
107
|
+
def list
|
108
|
+
server.core_names.sort.each do |name|
|
109
|
+
puts name
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def start
|
114
|
+
puts "starting SOLR"
|
115
|
+
exec start_solr_cmd
|
116
|
+
end
|
117
|
+
|
118
|
+
def server_status
|
119
|
+
if solr_pid
|
120
|
+
puts "SOLR running with pid #{solr_pid}"
|
121
|
+
else
|
122
|
+
puts "SOLR NOT running"
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
def stop
|
127
|
+
if solr_pid
|
128
|
+
system("kill #{solr_pid}")
|
129
|
+
puts "solr killed"
|
130
|
+
else
|
131
|
+
puts "solr not running"
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
def handle_response(response)
|
136
|
+
puts response.to_json
|
137
|
+
end
|
138
|
+
|
139
|
+
def start_solr_cmd
|
140
|
+
"cd #{File.dirname(solr_bin_path)} && /usr/bin/env java -Dsolr.solr.home=#{solr_home} -jar #{File.basename(solr_bin_path)} > /opt/solr/solr.log 2>&1 &"
|
141
|
+
end
|
142
|
+
|
143
|
+
def solr_pid
|
144
|
+
@solr_pid ||= get_solr_pid
|
145
|
+
end
|
146
|
+
|
147
|
+
def get_solr_pid
|
148
|
+
cmd = %(ps ax | grep "\\-Dsolr" | grep -v "grep" | grep -v " cd ")
|
149
|
+
if pid_s = `#{cmd}`[/^(\d+)/, 1]
|
150
|
+
pid_s.to_i
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
def solr_home
|
155
|
+
"/opt/solr"
|
156
|
+
end
|
157
|
+
|
158
|
+
def solr_bin_path
|
159
|
+
options[:solr_bin] || "/usr/local/Cellar/solr/#{SOLR_VERSION}/libexec/example/start.jar"
|
160
|
+
end
|
161
|
+
|
162
|
+
def solr_url
|
163
|
+
options[:server] || SOLR_URL
|
164
|
+
end
|
165
|
+
|
166
|
+
def core_name
|
167
|
+
options[:core]
|
168
|
+
end
|
169
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require "supernova"
|
2
|
+
require "supernova/solr/server"
|
3
|
+
require "typhoeus"
|
4
|
+
require "json"
|
5
|
+
|
6
|
+
class Supernova::Solr::Core < Supernova::Solr::Server
|
7
|
+
attr_reader :solr_url, :name
|
8
|
+
DEFAULT_SOLR_PATH = "/opt/solr"
|
9
|
+
|
10
|
+
def initialize(solr_url, name)
|
11
|
+
@solr_url = Supernova::Solr.remove_trailing_slash(solr_url)
|
12
|
+
@name = name
|
13
|
+
end
|
14
|
+
|
15
|
+
def url
|
16
|
+
"#{solr_url}/#{name}"
|
17
|
+
end
|
18
|
+
|
19
|
+
# add instance methods which call class methods with :attributes + original attributes
|
20
|
+
functional_delegate :create, :unload, :status, :exists?, :attributes => [:solr_url, :name]
|
21
|
+
|
22
|
+
class << self
|
23
|
+
def create(solr_url, core_name, instance_dir = nil, data_dir = nil)
|
24
|
+
instance_dir ||= DEFAULT_SOLR_PATH
|
25
|
+
data_dir ||= "#{DEFAULT_SOLR_PATH}/data/#{core_name}"
|
26
|
+
admin_action(solr_url, "CREATE", "name=#{core_name}&instanceDir=#{instance_dir}&dataDir=#{data_dir}")
|
27
|
+
end
|
28
|
+
|
29
|
+
def unload(solr_url, core_name)
|
30
|
+
admin_action(solr_url, "UNLOAD", "core=#{core_name}&deleteIndex=true")
|
31
|
+
end
|
32
|
+
|
33
|
+
def status(solr_url, core_name)
|
34
|
+
admin_action(solr_url, "STATUS", "core=#{core_name}")
|
35
|
+
end
|
36
|
+
|
37
|
+
def exists?(solr_url, core_name)
|
38
|
+
!status(solr_url, core_name)["status"][core_name].empty?
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
class Supernova::Solr::Mapper
|
2
|
+
|
3
|
+
DEFAULT_MAPPINGS = { :id => :id }
|
4
|
+
SUFFIXES = {
|
5
|
+
:string => :s,
|
6
|
+
:integer => :i,
|
7
|
+
:float => :f,
|
8
|
+
:boolean => :b,
|
9
|
+
:multi_string => :ms,
|
10
|
+
:time => :dt,
|
11
|
+
:date => :dt,
|
12
|
+
}
|
13
|
+
|
14
|
+
def initialize(mapping = {})
|
15
|
+
@mapping = DEFAULT_MAPPINGS.merge(mapping).inject({}) do |hash, (from, to)|
|
16
|
+
hash[from] = SUFFIXES[to] ? :"#{from}_#{SUFFIXES[to]}" : to
|
17
|
+
hash
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def map(hash)
|
22
|
+
mapping.inject({}) do |solr_hash, (from, to)|
|
23
|
+
solr_hash[to] = hash[from]
|
24
|
+
solr_hash
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
attr_reader :mapping
|
31
|
+
end
|