jberkel-solrium 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (66) hide show
  1. data/LICENSE.txt +712 -0
  2. data/README.markdown +51 -0
  3. data/Rakefile +15 -0
  4. data/VERSION.yml +4 -0
  5. data/lib/jars/jetty/ant-1.6.5.jar +0 -0
  6. data/lib/jars/jetty/commons-codec-1.3.jar +0 -0
  7. data/lib/jars/jetty/commons-fileupload-1.2.jar +0 -0
  8. data/lib/jars/jetty/jetty-7.0.0.pre5.jar +0 -0
  9. data/lib/jars/jetty/jetty-util-7.0.0.pre5.jar +0 -0
  10. data/lib/jars/jetty/jsp-2.1.jar +0 -0
  11. data/lib/jars/jetty/jsp-api-2.1.jar +0 -0
  12. data/lib/jars/jetty/servlet-api-3.0.pre4.jar +0 -0
  13. data/lib/jars/lucene/lucene-analyzers-2.4-dev.jar +0 -0
  14. data/lib/jars/lucene/lucene-core-2.4-dev.jar +0 -0
  15. data/lib/jars/lucene/lucene-highlighter-2.4-dev.jar +0 -0
  16. data/lib/jars/lucene/lucene-memory-2.4-dev.jar +0 -0
  17. data/lib/jars/lucene/lucene-queries-2.4-dev.jar +0 -0
  18. data/lib/jars/lucene/lucene-snowball-2.4-dev.jar +0 -0
  19. data/lib/jars/lucene/lucene-spellchecker-2.4-dev.jar +0 -0
  20. data/lib/jars/solr/apache-solr-common-1.3.0.jar +0 -0
  21. data/lib/jars/solr/apache-solr-core-1.3.0.jar +0 -0
  22. data/lib/jars/solr/commons-io-1.2.jar +0 -0
  23. data/lib/jars/solr/stax-1.2.0-dev.jar +0 -0
  24. data/lib/jars/solr/stax-api-1.0.jar +0 -0
  25. data/lib/jars/solr/stax-utils.jar +0 -0
  26. data/lib/jars/solr/xpp3-1.1.3.4.O.jar +0 -0
  27. data/lib/jetty.rb +83 -0
  28. data/lib/lucene.rb +33 -0
  29. data/lib/queries.rb +60 -0
  30. data/lib/remote.rb +22 -0
  31. data/lib/solr.rb +48 -0
  32. data/lib/solr_ext.rb +59 -0
  33. data/lib/webapp/admin/_info.jsp +110 -0
  34. data/lib/webapp/admin/action.jsp +116 -0
  35. data/lib/webapp/admin/analysis.jsp +456 -0
  36. data/lib/webapp/admin/analysis.xsl +179 -0
  37. data/lib/webapp/admin/distributiondump.jsp +158 -0
  38. data/lib/webapp/admin/favicon.ico +0 -0
  39. data/lib/webapp/admin/form.jsp +129 -0
  40. data/lib/webapp/admin/get-file.jsp +72 -0
  41. data/lib/webapp/admin/get-properties.jsp +24 -0
  42. data/lib/webapp/admin/header.jsp +41 -0
  43. data/lib/webapp/admin/index.jsp +154 -0
  44. data/lib/webapp/admin/jquery-1.2.3.min.js +32 -0
  45. data/lib/webapp/admin/logging.jsp +50 -0
  46. data/lib/webapp/admin/logging.xsl +91 -0
  47. data/lib/webapp/admin/meta.xsl +34 -0
  48. data/lib/webapp/admin/ping.jsp +65 -0
  49. data/lib/webapp/admin/ping.xsl +71 -0
  50. data/lib/webapp/admin/raw-schema.jsp +38 -0
  51. data/lib/webapp/admin/registry.jsp +107 -0
  52. data/lib/webapp/admin/registry.xsl +287 -0
  53. data/lib/webapp/admin/schema.jsp +661 -0
  54. data/lib/webapp/admin/solr-admin.css +206 -0
  55. data/lib/webapp/admin/solr-head.gif +0 -0
  56. data/lib/webapp/admin/solr-head.png +0 -0
  57. data/lib/webapp/admin/solr-lowercase.gif +0 -0
  58. data/lib/webapp/admin/solr-lowercase.png +0 -0
  59. data/lib/webapp/admin/stats.jsp +94 -0
  60. data/lib/webapp/admin/stats.xsl +220 -0
  61. data/lib/webapp/admin/tabular.xsl +141 -0
  62. data/lib/webapp/admin/threaddump.jsp +110 -0
  63. data/lib/webapp/admin/threaddump.xsl +103 -0
  64. data/lib/webapp/favicon.ico +0 -0
  65. data/lib/webapp/index.jsp +42 -0
  66. metadata +118 -0
@@ -0,0 +1,51 @@
1
+ # Solrium
2
+
3
+ A JRuby integration layer for Apache Solr/Lucene with a focus on usage from within irb.
4
+
5
+ ## Features
6
+
7
+ * embed solr directly into your app
8
+ * convenience methods for searching using ruby idioms
9
+ * jetty integration
10
+ * bundles all necessary jars (solr, lucene & jetty)
11
+
12
+ I mainly wrote it as a tool for quickly prototyping/debugging lucene queries and solr parser plugins.
13
+
14
+ Note: there's limited support for remote solr queries, searching will done in-process using a [SolrIndexSearcher](http://lucene.apache.org/solr/api/org/apache/solr/search/SolrIndexSearcher.html]), bypassing the HTTP layer. If you need to do remote queries in ruby consider using something like [sunspot](http://github.com/outoftime/sunspot) instead.
15
+
16
+ ## Synposis
17
+
18
+ ### Embedding solr into jirb
19
+
20
+ $ jirb -r lib/solr.rb
21
+ irb> solr = Solrium::Solr.new('/path/to/solr_home')
22
+ irb> solr.search("foo")
23
+ => [[Document: {"id"=>"1", "name_t"=>"Foomatic"}, 1.2112]]
24
+
25
+ ### Embedding jetty
26
+
27
+ $ jirb -r lib/solr.rb
28
+ irb> Solrium::Jetty.new('/path/to_solr_home').start(2000).join
29
+ 2009-05-14 13:47:24.444::INFO: jetty-7.0.0.pre5
30
+ ....
31
+ INFO: SolrDispatchFilter.init() done
32
+ 2009-05-14 13:47:25.336::INFO: Started SelectChannelConnector@0.0.0.0:2000
33
+
34
+ ## Credits
35
+
36
+ Lucene, Solr, Jetty are all licensed under Apache License Version 2.0.
37
+
38
+ ## Contact
39
+
40
+ Jan Berkel <jan.berkel@gmail.com>
41
+
42
+ ## License
43
+
44
+ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.
45
+ You may obtain a copy of the License at
46
+
47
+ http://www.apache.org/licenses/LICENSE-2.0
48
+
49
+ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
50
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
51
+ and limitations under the License.
@@ -0,0 +1,15 @@
1
+ require 'rubygems'
2
+
3
+ begin
4
+ require 'jeweler'
5
+ Jeweler::Tasks.new do |gemspec|
6
+ gemspec.name = "solrium"
7
+ gemspec.summary = "TODO"
8
+ gemspec.email = "jan.berkel@gmail.com"
9
+ gemspec.homepage = "http://github.com/jberkel/solrium"
10
+ gemspec.description = "A JRuby integration layer for Apache Solr/Lucene with a focus on irb integration."
11
+ gemspec.authors = ["Jan Berkel"]
12
+ end
13
+ rescue LoadError
14
+ puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
15
+ end
@@ -0,0 +1,4 @@
1
+ ---
2
+ :major: 0
3
+ :minor: 1
4
+ :patch: 0
@@ -0,0 +1,83 @@
1
+ #!/usr/bin/env jruby
2
+
3
+ require 'java'
4
+
5
+ Dir[File.join(File.dirname(__FILE__), 'jars', 'jetty', '*.jar')].each { |jar| require jar }
6
+ Dir[File.join(File.dirname(__FILE__), 'jars', 'solr', '*.jar')].each { |jar| require jar }
7
+ Dir[File.join(File.dirname(__FILE__), 'jars', 'lucene', '*.jar')].each { |jar| require jar }
8
+
9
+ module Solrium
10
+ class Jetty
11
+ include_class 'org.mortbay.jetty.Server'
12
+ include_class 'org.mortbay.jetty.servlet.Context'
13
+ include_class 'org.mortbay.jetty.servlet.ServletHolder'
14
+ include_class 'org.mortbay.jetty.servlet.DefaultServlet'
15
+ include_class 'org.mortbay.jetty.handler.ResourceHandler'
16
+ include_class 'org.mortbay.jetty.handler.MovedContextHandler'
17
+ include_class 'org.mortbay.jetty.handler.HandlerList'
18
+ include_class 'org.apache.solr.servlet.SolrServlet'
19
+ include_class 'org.apache.solr.servlet.SolrUpdateServlet'
20
+ include_class 'org.apache.jasper.servlet.JspServlet'
21
+ include_class 'org.mortbay.thread.QueuedThreadPool'
22
+ include_class 'org.mortbay.jetty.nio.SelectChannelConnector'
23
+ include_class 'org.mortbay.jetty.Handler'
24
+
25
+
26
+ def initialize(dir='solr-instance')
27
+ Java::JavaLang::System.setProperty("solr.solr.home", dir)
28
+ end
29
+
30
+ def start(port=8000, servlet_path='/solr', admin_interface=true)
31
+ server = create_server(port)
32
+ context = create_solr_context(servlet_path, admin_interface)
33
+
34
+ hl = HandlerList.new
35
+ hl.add_handler(context)
36
+ MovedContextHandler.new(hl, '/', "#{servlet_path}/admin") if admin_interface
37
+
38
+ server.set_handler(hl)
39
+ server.start
40
+
41
+ at_exit do
42
+ server.stop rescue nil
43
+ end
44
+ server
45
+ end
46
+
47
+ private
48
+ def create_server(port)
49
+ server = Server.new
50
+
51
+ thread_pool = QueuedThreadPool.new
52
+ thread_pool.min_threads = 5
53
+ thread_pool.max_threads = 50
54
+ server.set_thread_pool(thread_pool)
55
+
56
+ #use the more efficient NIO connector
57
+ connector = SelectChannelConnector.new
58
+ connector.port = port
59
+
60
+ server.add_connector(connector)
61
+ server
62
+ end
63
+
64
+ def create_solr_context(path, admin_interface=true)
65
+ context = Context.new(nil, path, Context::SESSIONS)
66
+ context.add_servlet(ServletHolder.new(DefaultServlet.new), "/")
67
+ context.add_servlet(ServletHolder.new(JspServlet.new), "*.jsp")
68
+ context.add_filter('org.apache.solr.servlet.SolrDispatchFilter', '/*', Handler::DEFAULT)
69
+ context.set_mime_types(Java::org.mortbay.jetty.MimeTypes.new)
70
+ context.mime_types.add_mime_mapping('xsl', 'application/xslt+xml')
71
+
72
+ if admin_interface
73
+ jsp_servlet = ServletHolder.new(JspServlet.new)
74
+ jsp_servlet.set_forced_path('/admin/ping.jsp')
75
+
76
+ context.add_servlet(jsp_servlet, '/admin/ping')
77
+ context.set_resource_base(File.join(File.dirname(__FILE__), 'webapp'))
78
+ end
79
+
80
+ context
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,33 @@
1
+ require 'java'
2
+ require File.join(File.dirname(__FILE__), 'queries')
3
+
4
+ module Solrium
5
+
6
+ # simple lucene integration, bypassing solr completely
7
+ class Lucene
8
+ include Queries
9
+
10
+ include_class 'org.apache.lucene.queryParser.QueryParser'
11
+ include_class 'org.apache.lucene.analysis.standard.StandardAnalyzer'
12
+ include_class 'org.apache.lucene.search.IndexSearcher'
13
+
14
+ attr_accessor :index_path
15
+
16
+ def initialize(solr_home = "solr-instance")
17
+ @index_path = File.join(solr_home, 'data', 'index')
18
+ end
19
+
20
+ def analyzer
21
+ StandardAnalyzer.new
22
+ end
23
+
24
+ def with_searcher(&block)
25
+ searcher = IndexSearcher.new(@index_path)
26
+ begin
27
+ block.call(searcher)
28
+ ensure
29
+ searcher.close
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,60 @@
1
+ require 'java'
2
+
3
+ require File.join(File.dirname(__FILE__), 'solr_ext')
4
+
5
+ module Solrium
6
+ module Queries
7
+
8
+ include_class 'org.apache.lucene.queryParser.QueryParser'
9
+ include_class 'org.apache.lucene.search.MultiPhraseQuery'
10
+ include_class 'org.apache.lucene.search.WildcardTermEnum'
11
+
12
+ include_class 'org.apache.lucene.search.BooleanQuery'
13
+ include_class 'org.apache.lucene.search.WildcardQuery'
14
+ include_class 'org.apache.lucene.search.TermQuery'
15
+
16
+
17
+ # convenience search method
18
+ def search(q, max=10, &block)
19
+ q = q.is_a?(org.apache.lucene.search.Query) ? q : parse(q.to_s)
20
+
21
+ block ||= lambda do |doc, score, searcher|
22
+ [doc, score]
23
+ end
24
+
25
+ with_searcher do |searcher|
26
+ searcher.search(q, nil, max).scoreDocs.map do |sd|
27
+ block.call(searcher.doc(sd.doc), sd.score, searcher)
28
+ end
29
+ end
30
+ end
31
+
32
+ def parse(q, default='default')
33
+ QueryParser.new(default, analyzer).parse(q)
34
+ end
35
+
36
+ #a wildcard phrase query
37
+ def wildcard_pq(*terms)
38
+ return MultiPhraseQuery.new if terms.empty?
39
+
40
+ if terms.size == 1
41
+ #only one term, do simple term+wildcard query
42
+ bq = BooleanQuery.new
43
+ bq.add(TermQuery.new(Term[terms[0]]), org.apache.lucene.search.BooleanClause::Occur::SHOULD)
44
+ bq.add(WildcardQuery.new(Term[terms[0] + "*"]), org.apache.lucene.search.BooleanClause::Occur::SHOULD)
45
+ return bq
46
+ end
47
+
48
+ mpq = MultiPhraseQuery.new
49
+
50
+ terms[0..-2].each { |t| mpq.add(Term[t]) }
51
+
52
+ with_searcher do |s|
53
+ completions = WildcardTermEnum.new(s.reader, Term["#{terms[-1]}*"]).to_java_array
54
+ return MultiPhraseQuery.new if completions.length == 0
55
+ mpq.add(completions)
56
+ end
57
+ mpq
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,22 @@
1
+ require 'net/http'
2
+ require 'uri'
3
+ require 'cgi'
4
+
5
+ module Solrium
6
+ module Remote
7
+ extend self
8
+
9
+ def query(q, params={}, port=8000)
10
+ params['q'] = q
11
+ params['wt'] = 'ruby'
12
+
13
+ url = URI.parse("http://localhost:#{port}")
14
+ res = Net::HTTP.start(url.host, url.port) do |http|
15
+ http.get("/solr/select?" + params.map { |k,v| "#{k}=#{CGI.escape(v)}" }.join("&"))
16
+ end
17
+ res.value
18
+ eval(res.body)
19
+ end
20
+
21
+ end
22
+ end
@@ -0,0 +1,48 @@
1
+ #!/usr/bin/env jruby
2
+ require 'java'
3
+
4
+ Dir[File.join(File.dirname(__FILE__), 'jars', 'lucene', '*.jar')].each { |jar| require jar }
5
+ Dir[File.join(File.dirname(__FILE__), 'jars', 'solr', '*.jar')].each { |jar| require jar }
6
+
7
+ require File.join(File.dirname(__FILE__), 'solr_ext')
8
+ require File.join(File.dirname(__FILE__), 'lucene')
9
+ require File.join(File.dirname(__FILE__), 'queries')
10
+ require File.join(File.dirname(__FILE__), 'remote')
11
+ require File.join(File.dirname(__FILE__), 'jetty')
12
+
13
+ module Solrium
14
+ #wrapper for SolrCore
15
+ class Solr
16
+ include Queries
17
+ include Remote
18
+
19
+ include_class 'org.apache.solr.core.SolrCore'
20
+ include_class 'org.apache.solr.core.SolrConfig'
21
+
22
+ attr_accessor :config, :core
23
+
24
+ def initialize(solr_home='solr-instance')
25
+ Java::JavaLang::System.setProperty("solr.solr.home", solr_home)
26
+ @config = SolrConfig.new
27
+ @core = SolrCore.new('core', nil, @config, nil, nil)
28
+
29
+ at_exit do
30
+ @core.close rescue nil
31
+ end
32
+ end
33
+
34
+ def analyzer
35
+ core.schema.analyzer
36
+ end
37
+
38
+ def with_searcher(&block)
39
+ searcher_ref = core.searcher
40
+ searcher = searcher_ref.get
41
+ begin
42
+ block.call(searcher)
43
+ ensure
44
+ searcher_ref.decref
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,59 @@
1
+ # make lucene/solr a bit more rubyis
2
+ module Solrium
3
+ include_class 'org.apache.lucene.index.TermEnum'
4
+ include_class 'org.apache.lucene.index.Term'
5
+ include_class 'org.apache.lucene.document.Document'
6
+ include_class 'org.apache.lucene.document.Field'
7
+
8
+ class Document
9
+ def [](n)
10
+ self.getField(n)
11
+ end
12
+
13
+ def inspect
14
+ "Document: " + to_hash.inspect
15
+ end
16
+
17
+ def to_hash
18
+ fields.inject({}) do |h, f|
19
+ h[f.name] = f.stringValue()
20
+ h
21
+ end
22
+ end
23
+ end
24
+
25
+ class Field
26
+ def inspect
27
+ "#{name}: #{to_s}"
28
+ end
29
+
30
+ def to_s
31
+ self.stringValue()
32
+ end
33
+ end
34
+
35
+ # lets you use Term['foo'] ==> Term.new('default', foo)
36
+ class Term
37
+ def self.[](t)
38
+ new('default', t)
39
+ end
40
+ end
41
+
42
+ #make TermEnum a bit friendlier to use
43
+ class TermEnum
44
+ include Enumerable
45
+
46
+ def each
47
+ while self.next()
48
+ yield(self.term())
49
+ end
50
+ end
51
+
52
+ def to_java_array
53
+ elements = to_a
54
+ array = java.lang.reflect.Array.newInstance(Term.java_class, elements.length)
55
+ elements.each_with_index { |e,i| array[i] = e }
56
+ array
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,110 @@
1
+ <%--
2
+ Licensed to the Apache Software Foundation (ASF) under one or more
3
+ contributor license agreements. See the NOTICE file distributed with
4
+ this work for additional information regarding copyright ownership.
5
+ The ASF licenses this file to You under the Apache License, Version 2.0
6
+ (the "License"); you may not use this file except in compliance with
7
+ the License. You may obtain a copy of the License at
8
+
9
+ http://www.apache.org/licenses/LICENSE-2.0
10
+
11
+ Unless required by applicable law or agreed to in writing, software
12
+ distributed under the License is distributed on an "AS IS" BASIS,
13
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ See the License for the specific language governing permissions and
15
+ limitations under the License.
16
+ --%>
17
+ <%@ page import="org.apache.solr.core.SolrConfig,
18
+ org.apache.solr.core.SolrCore,
19
+ org.apache.solr.schema.IndexSchema,
20
+ java.io.File"%>
21
+ <%@ page import="java.net.InetAddress"%>
22
+ <%@ page import="java.io.StringWriter"%>
23
+ <%@ page import="org.apache.solr.core.Config"%>
24
+ <%@ page import="org.apache.solr.common.util.XML"%>
25
+ <%@ page import="org.apache.solr.common.SolrException"%>
26
+ <%@ page import="org.apache.lucene.LucenePackage"%>
27
+ <%@ page import="java.net.UnknownHostException" %>
28
+ <%
29
+ //
30
+ SolrCore core = (SolrCore) request.getAttribute("org.apache.solr.SolrCore");
31
+ if (core == null) {
32
+ response.sendError( 404, "missing core name in path" );
33
+ return;
34
+ }
35
+
36
+ SolrConfig solrConfig = core.getSolrConfig();
37
+ int port = request.getServerPort();
38
+ IndexSchema schema = core.getSchema();
39
+
40
+ // enabled/disabled is purely from the point of a load-balancer
41
+ // and has no effect on local server function. If there is no healthcheck
42
+ // configured, don't put any status on the admin pages.
43
+ String enabledStatus = null;
44
+ String enabledFile = solrConfig.get("admin/healthcheck/text()",null);
45
+ boolean isEnabled = false;
46
+ if (enabledFile!=null) {
47
+ isEnabled = new File(enabledFile).exists();
48
+ }
49
+
50
+ String collectionName = schema!=null ? schema.getName():"unknown";
51
+ InetAddress addr = null;
52
+ String hostname = "unknown";
53
+ try {
54
+ addr = InetAddress.getLocalHost();
55
+ hostname = addr.getCanonicalHostName();
56
+ } catch (UnknownHostException e) {
57
+ //default to unknown
58
+ }
59
+
60
+ String defaultSearch = "";
61
+ {
62
+ StringWriter tmp = new StringWriter();
63
+ XML.escapeCharData
64
+ (solrConfig.get("admin/defaultQuery/text()", null), tmp);
65
+ defaultSearch = tmp.toString();
66
+ }
67
+
68
+ String solrImplVersion = "";
69
+ String solrSpecVersion = "";
70
+ String luceneImplVersion = "";
71
+ String luceneSpecVersion = "";
72
+
73
+ {
74
+ Package p;
75
+ StringWriter tmp;
76
+
77
+ p = SolrCore.class.getPackage();
78
+
79
+ tmp = new StringWriter();
80
+ solrImplVersion = p.getImplementationVersion();
81
+ if (null != solrImplVersion) {
82
+ XML.escapeCharData(solrImplVersion, tmp);
83
+ solrImplVersion = tmp.toString();
84
+ }
85
+ tmp = new StringWriter();
86
+ solrSpecVersion = p.getSpecificationVersion() ;
87
+ if (null != solrSpecVersion) {
88
+ XML.escapeCharData(solrSpecVersion, tmp);
89
+ solrSpecVersion = tmp.toString();
90
+ }
91
+
92
+ p = LucenePackage.class.getPackage();
93
+
94
+ tmp = new StringWriter();
95
+ luceneImplVersion = p.getImplementationVersion();
96
+ if (null != luceneImplVersion) {
97
+ XML.escapeCharData(luceneImplVersion, tmp);
98
+ luceneImplVersion = tmp.toString();
99
+ }
100
+ tmp = new StringWriter();
101
+ luceneSpecVersion = p.getSpecificationVersion() ;
102
+ if (null != luceneSpecVersion) {
103
+ XML.escapeCharData(luceneSpecVersion, tmp);
104
+ luceneSpecVersion = tmp.toString();
105
+ }
106
+ }
107
+
108
+ String cwd=System.getProperty("user.dir");
109
+ String solrHome= solrConfig.getInstanceDir();
110
+ %>