simple_solr_client 0.1.2 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGES.md +5 -0
- data/README.md +108 -125
- data/bin/solr_shell +40 -0
- data/lib/simple_solr_client/client/system.rb +39 -0
- data/lib/simple_solr_client/client.rb +61 -17
- data/lib/simple_solr_client/core/admin.rb +0 -13
- data/lib/simple_solr_client/core/core_data.rb +20 -5
- data/lib/simple_solr_client/core.rb +6 -1
- data/lib/simple_solr_client/schema/analysis.rb +22 -1
- data/lib/simple_solr_client/schema/field.rb +14 -3
- data/lib/simple_solr_client/schema/field_or_type.rb +42 -42
- data/lib/simple_solr_client/schema/field_type.rb +0 -15
- data/lib/simple_solr_client/schema.rb +0 -44
- data/lib/simple_solr_client/version.rb +1 -1
- data/simple_solr_client.gemspec +1 -0
- data/spec/minitest_helper.rb +2 -2
- metadata +22 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7bd1347fc6a4bdc8dd5cbf6fc240e8341cc1421c
|
4
|
+
data.tar.gz: 34cf45357e585520d46ebeab5756010eab14c0d0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a10b32a6954ee985251d9d3386e0b9dca42bfb3efd677ef3469f4e6b82226efbfd1c70d0e1ceab691f5ce5f8aa293d3154558680918edd5fbd25b30887387e96
|
7
|
+
data.tar.gz: fd691d29fdbd45b255a024be57d89436a9fc2bd8a14177b23d449ff8dafb4c14613392e06229579e4aba60e235f6c17e90c118223d26dc970ef5a6cce35d4b34
|
data/CHANGES.md
ADDED
data/README.md
CHANGED
@@ -1,55 +1,24 @@
|
|
1
1
|
# SimpleSolrClient
|
2
2
|
|
3
|
-
|
4
|
-
|
3
|
+
A simple client and accompanying shell to help test and give simple
|
4
|
+
commands to a solr instance.
|
5
5
|
|
6
|
-
A Solr client specifically designed to try to help you test what the heck
|
7
|
-
solr is actually doing.
|
8
|
-
|
9
|
-
Most useful when running on the same machine as the solr install, but
|
10
|
-
still useful even when you're not.
|
11
|
-
|
12
|
-
|
13
|
-
## Motivation
|
14
|
-
|
15
|
-
Solr is complex.
|
16
|
-
|
17
|
-
It's complex enough, and fuddles with enough edge cases, that reading
|
18
|
-
the documentation and/or the code doesn't get me the understanding
|
19
|
-
that I feel I need.
|
20
|
-
|
21
|
-
If I were smarter, maybe I wouldn't need something like this.
|
22
|
-
|
23
|
-
I wanted a way to test what solr is actually doing, and
|
24
|
-
this library is a way for me to start to do that in a fashion that's
|
25
|
-
more convenient that doing everything "by hand" in the admin dashboard
|
26
|
-
or running queries via URLs in my browser or using curl.
|
27
|
-
|
28
|
-
I wanted a way to figure out what fields (of what types) are being created,
|
29
|
-
how things were being tokenized, etc., but all within the comfort of a test
|
30
|
-
suite that I could run against solr configurations to make sure things
|
31
|
-
weren't breaking when I made changes. I wanted to build up a structure around relevance
|
32
|
-
ranking tests (still coming, sadly) and quickly swap out different
|
33
|
-
configs to make sure it all works as I expect.
|
34
|
-
|
35
|
-
So: a simple solr library, with more exposure than most of what's out there
|
36
|
-
to the solr administration API and the introspection/analysis it affords.
|
37
6
|
|
38
7
|
# Features:
|
39
8
|
|
40
|
-
*
|
41
|
-
*
|
42
|
-
*
|
9
|
+
* Get basic info about cores (size, number of docs, etc.)
|
10
|
+
* Basic (*very basic) add/delete/query
|
11
|
+
* Commit/optimize/clear (empty) an index
|
12
|
+
* Reload a core (presumably after editing a `schema.xml` or `solrconfig.xml`)
|
43
13
|
* Inspect lists of fields, dynamicFields, copyFields, and
|
44
14
|
fieldTypes
|
45
|
-
* Determine which fields (and their properties) would be
|
46
|
-
created when a given field name is indexed, taking into
|
47
|
-
account dynamicField and copyField directives.
|
48
15
|
* Get list of the tokens that would be created if you
|
49
16
|
send a string to a paricular fieldType (like in the
|
50
17
|
solr admin analysis page)
|
51
|
-
*
|
52
|
-
|
18
|
+
* Determine (usually) which fields (and their properties) would be
|
19
|
+
created when a given field name is indexed, taking into
|
20
|
+
account dynamicField and copyField directives.
|
21
|
+
|
53
22
|
|
54
23
|
Additional features when running against a localhost solr:
|
55
24
|
* Spin up a temporary core to play with
|
@@ -83,12 +52,14 @@ core.schema_file #=> <path>/<to>/<schema.xml>
|
|
83
52
|
# Remove all the indexed documents and (automatically) commit
|
84
53
|
core.clear
|
85
54
|
|
55
|
+
core.number_of_documents #=> 0. Automatic commit for #clear
|
56
|
+
|
86
57
|
# Add documents
|
87
58
|
#
|
88
59
|
# name_t is a text_general, multiValued, indexed, stored field
|
89
|
-
h1 = {:id => '
|
90
|
-
h2 = {:id => '
|
91
|
-
h3 = {:id => '
|
60
|
+
h1 = {:id => '1', :name_t=>"Bill Dueber"}
|
61
|
+
h2 = {:id => '2', :name_t=>"Danit Brown"}
|
62
|
+
h3 = {:id => '3', :name_t=>"Ziv Brown Dueber"}
|
92
63
|
|
93
64
|
core.add_docs(h1)
|
94
65
|
|
@@ -112,10 +83,10 @@ docs = core.fv_search(:name_t, 'Brown')
|
|
112
83
|
docs.class #=> SimpleSolrClient::Response::QueryResponse
|
113
84
|
|
114
85
|
docs.size #=> 2
|
115
|
-
docs
|
86
|
+
docs.map{|d| d['name_t']} #=> [['Danit Brown'], ['Ziv Brown Dueber']]
|
116
87
|
|
117
88
|
# Special-case id/score as regular methods
|
118
|
-
docs.first.id #=> '
|
89
|
+
docs.first.id #=> '2'
|
119
90
|
docs.first.score #=> 0.625
|
120
91
|
|
121
92
|
# Figure out where documents fall. "Ziv Brown Dueber" contains both
|
@@ -123,8 +94,8 @@ docs.first.score #=> 0.625
|
|
123
94
|
docs = core.fv_search(:name_t, 'Brown Dueber')
|
124
95
|
docs.size #=> 3
|
125
96
|
|
126
|
-
docs.rank('
|
127
|
-
docs.rank('
|
97
|
+
docs.rank('3') #=> 1 (check by id)
|
98
|
+
docs.rank('3') < docs.rank('b') #=> true
|
128
99
|
|
129
100
|
# Of course, we can do it by score
|
130
101
|
docs.score('z') > docs.score('d')
|
@@ -135,40 +106,116 @@ core.delete('name_t:Dueber').commit.number_of_documents #=> 1
|
|
135
106
|
|
136
107
|
```
|
137
108
|
|
138
|
-
##
|
109
|
+
## Field Types and analysis
|
139
110
|
|
140
|
-
|
141
|
-
|
142
|
-
|
111
|
+
Field Types are created by getting data from the API and also
|
112
|
+
parsing XML out of the schema.xml (for later creating a new
|
113
|
+
schema.xml if you'd like).
|
114
|
+
|
115
|
+
You can also ask a field type how it would tokenize an input
|
116
|
+
string via indexing or querying.
|
117
|
+
|
118
|
+
NOTE: FieldTypes _should_ be able to, say, report their XML serialization even
|
119
|
+
when outside of a particular schema object, but right now that doesn't
|
120
|
+
work. If you make changes to a field type, the only way to see the new
|
121
|
+
serialization is to call `schema.to_xml` on whichever schema you added
|
122
|
+
it to via `schema.add_field_type(ft)`
|
123
|
+
|
124
|
+
|
125
|
+
|
126
|
+
```ruby
|
127
|
+
|
128
|
+
core.schema.field_types.size #=> 23
|
129
|
+
ft = schema.field_type('text') #=> SimpleSolrClient::Schema::FieldType
|
130
|
+
ft.name #=> 'text'
|
131
|
+
ft.solr_class #=> 'solr.TextField'
|
132
|
+
ft.multi #=> true
|
133
|
+
ft.stored #=> true
|
134
|
+
ft.indexed #=> true
|
135
|
+
# etc.
|
136
|
+
|
137
|
+
newft = SimpleSolrClient::Schema::FieldType.new_from_xml(xmlstring)
|
138
|
+
schema.add_field_type(newft)
|
139
|
+
|
140
|
+
ft.name #=> text
|
141
|
+
ft.query_tokens "Don't forget me when I'm getting H20"
|
142
|
+
#=> ["don't", "forget", "me", "when", "i'm", ["getting", "get"], "h20"]
|
143
|
+
|
144
|
+
ft.index_tokens 'When it rains, it pours'
|
145
|
+
#=> ["when", "it", ["rains", "rain"], "it", ["pours", "pour"]]
|
146
|
+
|
147
|
+
|
148
|
+
# Check for validity
|
149
|
+
|
150
|
+
int_type = core.schema.field_type('int')
|
151
|
+
int_type.index_tokens("33") => ["33"]
|
152
|
+
int_type.index_token_valid?("33") #=> true
|
153
|
+
|
154
|
+
int_type.index_token_valid?("33.3") #=> false
|
155
|
+
int_type.index_tokens('33.3') #=> RuntimeError
|
156
|
+
|
157
|
+
|
158
|
+
```
|
159
|
+
|
160
|
+
|
161
|
+
|
162
|
+
## Saving/reloading a changed configuration
|
163
|
+
|
164
|
+
Whether you change a solr install via editing a text file or
|
165
|
+
by using `schema.write`, you can always reload a core.
|
166
|
+
|
167
|
+
```ruby
|
168
|
+
core.reload
|
169
|
+
```
|
170
|
+
|
171
|
+
If you're working on localhost, you can make programmatic changes
|
172
|
+
to the schema and then ask for a write/reload cycle. It uses the API
|
173
|
+
to find the path to the schema.xml file and overwrites it.
|
174
|
+
|
175
|
+
```ruby
|
143
176
|
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
177
|
+
schema = core.schema
|
178
|
+
core.add_field Field.new(:name=>'price', :type_name=>'float')
|
179
|
+
schema.write
|
180
|
+
schema = core.reload.schema
|
181
|
+
```
|
182
|
+
|
183
|
+
|
184
|
+
|
185
|
+
## The `schema` object
|
150
186
|
|
187
|
+
Each core exposes a `schema` object that allows you to find out about
|
188
|
+
the fields, copyfields, and field types, and how they interact with
|
189
|
+
query and index calls (like the analysis screen in the admin interface)
|
151
190
|
|
152
191
|
```ruby
|
153
192
|
|
154
193
|
# Get a list of cores
|
155
|
-
client.cores #=> ['core1']
|
194
|
+
client.cores #=> ['core1', 'core2']
|
156
195
|
core = client.core('core1')
|
157
196
|
|
158
197
|
# Get an object representing the schema.xml file
|
159
198
|
schema = core.schema #=> SimpleSolrClient::Schema object
|
160
199
|
|
161
|
-
# Get lists of field, dynamicFields,
|
200
|
+
# Get lists of field, dynamicFields, and copyFields
|
162
201
|
# all as SimpleSolrClient::Schema::XXX objects
|
163
202
|
|
164
203
|
explicit_fields = schema.fields
|
165
204
|
dynamic_fields = schema.dynamic_fields
|
166
205
|
copy_fields = schema.copy_fields
|
206
|
+
|
207
|
+
# Get a list of FieldType object
|
167
208
|
field_types = schema.field_types
|
209
|
+
field_type_names = schema.field_types.map(&:name)
|
210
|
+
|
211
|
+
# Check out a specific field type and how it parses stuff
|
212
|
+
mytexttype = schema.field_type('mytexttype')
|
213
|
+
mytexttype.index_tokens('bill dueber solr-stuff') #=> ['bill', 'dueber', 'solr', 'stuff']
|
214
|
+
mytexttype.query_tokens('bill dueber solr-stuff') #=> ['bill', 'dueber', 'solr', 'stuff']
|
168
215
|
|
169
216
|
```
|
170
217
|
|
171
|
-
### Regular fields
|
218
|
+
### Regular (non-dynamic) fields
|
172
219
|
|
173
220
|
Internally I call these "explicit_fields" as opposed to dynamic fields.
|
174
221
|
|
@@ -245,48 +292,6 @@ schema.add_copy_field(cf)
|
|
245
292
|
```
|
246
293
|
|
247
294
|
|
248
|
-
### Field Types
|
249
|
-
|
250
|
-
Field Types are created by getting data from the API and also
|
251
|
-
parsing XML out of the schema.xml (for later creating a new
|
252
|
-
schema.xml if you'd like).
|
253
|
-
|
254
|
-
You can also ask a field type how it would tokenize an input
|
255
|
-
string via indexing or querying.
|
256
|
-
|
257
|
-
|
258
|
-
FieldTypes _should_ be able to, say, report their XML serialization even
|
259
|
-
when outside of a particular schema object, but right now that doesn't
|
260
|
-
work. If you make changes to a field type, the only way to see the new
|
261
|
-
serialization is to call `schema.to_xml` on whichever schema you added
|
262
|
-
it to via `schema.add_field_type(ft)`
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
```ruby
|
267
|
-
|
268
|
-
schema.field_types.size #=> 23
|
269
|
-
ft = schema.field_type('text') #=> SimpleSolrClient::Schema::FieldType
|
270
|
-
ft.name #=> 'text'
|
271
|
-
ft.solr_class #=> 'solr.TextField'
|
272
|
-
ft.multi #=> true
|
273
|
-
ft.stored #=> true
|
274
|
-
ft.indexed #=> true
|
275
|
-
# etc.
|
276
|
-
|
277
|
-
newft = SimpleSolrClient::Schema::FieldType.new_from_xml(xmlstring)
|
278
|
-
schema.add_field_type(newft)
|
279
|
-
|
280
|
-
ft.name #=> text
|
281
|
-
ft.query_tokens "Don't forget me when I'm getting H20"
|
282
|
-
#=> ["don't", "forget", "me", "when", "i'm", ["getting", "get"], "h20"]
|
283
|
-
|
284
|
-
ft.index_tokens 'When it rains, it pours'
|
285
|
-
#=> ["when", "it", ["rains", "rain"], "it", ["pours", "pour"]]
|
286
|
-
|
287
|
-
```
|
288
|
-
|
289
|
-
|
290
295
|
## What will I get if I index a field named `str`?
|
291
296
|
|
292
297
|
Dynamic- and copy-fields are very convenient, but it can make it hard to
|
@@ -313,28 +318,6 @@ rs.find_all{|f| f.indexed}.map(&:name) #=> ['name_t']
|
|
313
318
|
```
|
314
319
|
|
315
320
|
|
316
|
-
## Saving/reloading a changed schema
|
317
|
-
|
318
|
-
Whether you change a solr install via editing a text file or
|
319
|
-
by using `schema.write`, you can always reload a core.
|
320
|
-
|
321
|
-
```ruby
|
322
|
-
core.reload
|
323
|
-
```
|
324
|
-
|
325
|
-
If you're working on localhost, you can make programmatic changes
|
326
|
-
to the schema and then ask for a write/reload cycle. It uses the API
|
327
|
-
to find the path to the schema.xml file and overwrites it.
|
328
|
-
|
329
|
-
```ruby
|
330
|
-
|
331
|
-
schema = core.schema
|
332
|
-
core.add_field Field.new(:name=>'price', :type_name=>'float')
|
333
|
-
schema.write
|
334
|
-
schema = core.reload.schema
|
335
|
-
```
|
336
|
-
|
337
|
-
|
338
321
|
## Installation
|
339
322
|
|
340
323
|
$ gem install simple_solr
|
data/bin/solr_shell
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
|
4
|
+
# If we're loading from source instead of a gem, we need to
|
5
|
+
# set the load path directly
|
6
|
+
self_load_path = File.expand_path("../lib", File.dirname(__FILE__))
|
7
|
+
unless $LOAD_PATH.include? self_load_path
|
8
|
+
$LOAD_PATH << self_load_path
|
9
|
+
end
|
10
|
+
|
11
|
+
require 'simple_solr_client'
|
12
|
+
|
13
|
+
require 'pry'
|
14
|
+
|
15
|
+
solr_url = ARGV[0]
|
16
|
+
|
17
|
+
client = SimpleSolrClient::Client.new(solr_url)
|
18
|
+
|
19
|
+
|
20
|
+
|
21
|
+
|
22
|
+
pry = Pry.new
|
23
|
+
cb = pry.current_binding
|
24
|
+
puts "
|
25
|
+
Connected to #{client.url}
|
26
|
+
|
27
|
+
"
|
28
|
+
|
29
|
+
cb.local_variable_set(:solr, client)
|
30
|
+
puts " * Created variable 'solr' for the main solr client"
|
31
|
+
|
32
|
+
client.cores.each do |core|
|
33
|
+
cb.local_variable_set :"#{core}", client.core(core)
|
34
|
+
puts " * Created variable '#{core}' pointing to its core"
|
35
|
+
end
|
36
|
+
|
37
|
+
puts "\n\n"
|
38
|
+
|
39
|
+
|
40
|
+
cb.pry
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module SimpleSolrClient
|
2
|
+
# Attributes of the solr process itself
|
3
|
+
class System
|
4
|
+
|
5
|
+
def initialize(sysresp)
|
6
|
+
@resp = sysresp
|
7
|
+
end
|
8
|
+
|
9
|
+
# @return [String] Full lucene version, with release data and everything
|
10
|
+
def lucene_full_version
|
11
|
+
@resp['lucene']['lucene-impl-version']
|
12
|
+
end
|
13
|
+
|
14
|
+
# @return [String] Lucene version as M.m.p
|
15
|
+
def lucene_semver_version
|
16
|
+
@resp['lucene']['lucene-spec-version']
|
17
|
+
end
|
18
|
+
|
19
|
+
# @return [Integer] The major lucene version (e.g., 7)
|
20
|
+
def lucene_major_version
|
21
|
+
lucene_full_version.split('.').first.to_i
|
22
|
+
end
|
23
|
+
|
24
|
+
# @return [String] Full lucene version, with release data and everything
|
25
|
+
def solr_full_version
|
26
|
+
@resp['lucene']['solr-impl-version']
|
27
|
+
end
|
28
|
+
|
29
|
+
# @return [String] Lucene version as M.m.p
|
30
|
+
def solr_semver_version
|
31
|
+
@resp['lucene']['solr-spec-version']
|
32
|
+
end
|
33
|
+
|
34
|
+
# @return [Integer] The major lucene version (e.g., 7)
|
35
|
+
def solr_major_version
|
36
|
+
solr_semver_version.split('.').first.to_i
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -3,6 +3,7 @@ require 'simple_solr_client/response/generic_response'
|
|
3
3
|
require 'securerandom'
|
4
4
|
|
5
5
|
require 'simple_solr_client/core'
|
6
|
+
require 'simple_solr_client/client/system'
|
6
7
|
|
7
8
|
module SimpleSolrClient
|
8
9
|
|
@@ -13,10 +14,18 @@ module SimpleSolrClient
|
|
13
14
|
|
14
15
|
attr_reader :base_url, :rawclient
|
15
16
|
|
16
|
-
def initialize(
|
17
|
-
|
17
|
+
def initialize(url_or_port)
|
18
|
+
url = if url_or_port.is_a?(Integer)
|
19
|
+
"http://localhost:#{url_or_port}/solr"
|
20
|
+
else
|
21
|
+
url_or_port
|
22
|
+
end
|
23
|
+
|
24
|
+
@base_url = url.chomp('/')
|
18
25
|
@client_url = @base_url
|
19
|
-
@rawclient
|
26
|
+
@rawclient = HTTPClient.new
|
27
|
+
|
28
|
+
# raise "Can't connect to Solr at #{url}" unless self.up?
|
20
29
|
end
|
21
30
|
|
22
31
|
# Construct a URL for the given arguments that hit the configured solr
|
@@ -31,12 +40,45 @@ module SimpleSolrClient
|
|
31
40
|
[@client_url, *args].join('/').chomp('/')
|
32
41
|
end
|
33
42
|
|
43
|
+
def ping
|
44
|
+
get('admin/ping')
|
45
|
+
end
|
46
|
+
|
47
|
+
# Get info about the solr system itself
|
48
|
+
def system
|
49
|
+
@system ||= SimpleSolrClient::System.new(get('admin/info/system'))
|
50
|
+
end
|
51
|
+
|
52
|
+
# @return [String] The solr semver version
|
53
|
+
def version
|
54
|
+
system.solr_semver_version
|
55
|
+
end
|
56
|
+
|
57
|
+
# @return [Integer] the solr major version
|
58
|
+
def major_version
|
59
|
+
system.solr_major_version
|
60
|
+
end
|
61
|
+
|
62
|
+
# Is the server up (and responding to a ping?)
|
63
|
+
# @return [Boolean]
|
64
|
+
def up?
|
65
|
+
begin
|
66
|
+
ping.status == 'OK'
|
67
|
+
rescue
|
68
|
+
false
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
|
34
73
|
|
35
|
-
# Call a get on the underlying http client and return the content
|
36
|
-
#
|
37
|
-
#
|
74
|
+
# Call a 'get' on the underlying http client and return the content
|
75
|
+
# Will use whatever the URL is for the current context ("client" or
|
76
|
+
# "core"), although you can pass in :force_top_level=>true for those
|
77
|
+
# cases when you absolutely have to use the client-level url and not a
|
38
78
|
# core level URL
|
39
|
-
|
79
|
+
#
|
80
|
+
# Error handling? What error handling???
|
81
|
+
def raw_get_content(path, args = {})
|
40
82
|
if args.delete(:force_top_level_url)
|
41
83
|
u = top_level_url(path)
|
42
84
|
else
|
@@ -50,10 +92,10 @@ module SimpleSolrClient
|
|
50
92
|
# @param [String] path The parts of the URL that comes after the core
|
51
93
|
# @param [Hash] args The url arguments
|
52
94
|
# @return [Hash] the parsed-out response
|
53
|
-
def _get(path, args={})
|
95
|
+
def _get(path, args = {})
|
54
96
|
path.sub! /\A\//, ''
|
55
97
|
args['wt'] = 'json'
|
56
|
-
res
|
98
|
+
res = JSON.parse(raw_get_content(path, args))
|
57
99
|
if res['error']
|
58
100
|
raise RuntimeError.new, res['error']
|
59
101
|
end
|
@@ -89,25 +131,27 @@ module SimpleSolrClient
|
|
89
131
|
# @param [String] corename The name of the core (which must already exist!)
|
90
132
|
# @return [SimpleSolrClient::Core]
|
91
133
|
def core(corename)
|
92
|
-
|
134
|
+
raise "Core #{corename} not found" unless cores.include? corename.to_s
|
135
|
+
SimpleSolrClient::Core.new(@base_url, corename.to_s)
|
93
136
|
end
|
94
137
|
|
95
138
|
|
139
|
+
# Get all the cores
|
96
140
|
def cores
|
97
|
-
cdata = get('admin/cores', {:force_top_level_url=>true}).status.keys
|
141
|
+
cdata = get('admin/cores', {:force_top_level_url => true}).status.keys
|
98
142
|
end
|
99
143
|
|
100
144
|
|
101
145
|
# Create a new, temporary core
|
102
146
|
#noinspection RubyWrongHash
|
103
147
|
def new_core(corename)
|
104
|
-
dir
|
148
|
+
dir = temp_core_dir_setup(corename)
|
105
149
|
|
106
150
|
args = {
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
151
|
+
:wt => 'json',
|
152
|
+
:action => 'CREATE',
|
153
|
+
:name => corename,
|
154
|
+
:instanceDir => dir
|
111
155
|
}
|
112
156
|
|
113
157
|
get('admin/cores', args)
|
@@ -121,7 +165,7 @@ module SimpleSolrClient
|
|
121
165
|
|
122
166
|
# Set up files for a temp core
|
123
167
|
def temp_core_dir_setup(corename)
|
124
|
-
dest = Dir.mktmpdir("simple_solr_#{corename}")
|
168
|
+
dest = Dir.mktmpdir("simple_solr_#{corename}_#{SecureRandom.uuid}")
|
125
169
|
src = SAMPLE_CORE_DIR
|
126
170
|
FileUtils.cp_r File.join(src, '.'), dest
|
127
171
|
dest
|
@@ -1,17 +1,4 @@
|
|
1
1
|
module SimpleSolrClient::Core::Admin
|
2
|
-
def ping
|
3
|
-
get('admin/ping')
|
4
|
-
end
|
5
|
-
|
6
|
-
# Is the server up (and responding to a ping?)
|
7
|
-
# @return [Boolean]
|
8
|
-
def up?
|
9
|
-
begin
|
10
|
-
ping.status == 'OK'
|
11
|
-
rescue
|
12
|
-
false
|
13
|
-
end
|
14
|
-
end
|
15
2
|
|
16
3
|
# Send a commit command
|
17
4
|
# @return self
|
@@ -18,22 +18,38 @@ module SimpleSolrClient::Core::CoreData
|
|
18
18
|
core_data_hash['index']
|
19
19
|
end
|
20
20
|
|
21
|
-
|
22
|
-
core_data_hash['isDefaultCore']
|
23
|
-
end
|
24
|
-
|
21
|
+
# Time of last modification
|
25
22
|
def last_modified
|
26
23
|
Time.parse index['lastModified']
|
27
24
|
end
|
28
25
|
|
26
|
+
# Total documents
|
29
27
|
def number_of_documents
|
30
28
|
index['numDocs']
|
31
29
|
end
|
32
30
|
|
31
|
+
alias_method :numDocs, :number_of_documents
|
32
|
+
alias_method :num_docs, :number_of_documents
|
33
|
+
|
34
|
+
# The (local to the server) data directory
|
33
35
|
def data_dir
|
34
36
|
core_data_hash['dataDir']
|
35
37
|
end
|
36
38
|
|
39
|
+
|
40
|
+
# Get the index size in megabytes
|
41
|
+
def size
|
42
|
+
str = index['size']
|
43
|
+
num, unit = str.split(/\s+/).compact.map(&:strip)
|
44
|
+
num = num.to_f
|
45
|
+
case unit
|
46
|
+
when "MB"
|
47
|
+
num * 1
|
48
|
+
when "GB"
|
49
|
+
num * 1000
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
37
53
|
def instance_dir
|
38
54
|
core_data_hash['instanceDir']
|
39
55
|
end
|
@@ -45,7 +61,6 @@ module SimpleSolrClient::Core::CoreData
|
|
45
61
|
def schema_file
|
46
62
|
File.join(instance_dir, 'conf', core_data_hash['schema'])
|
47
63
|
end
|
48
|
-
|
49
64
|
end
|
50
65
|
|
51
66
|
|
@@ -25,7 +25,12 @@ class SimpleSolrClient::Core
|
|
25
25
|
attr_reader :core
|
26
26
|
alias_method :name, :core
|
27
27
|
|
28
|
-
def initialize(url, core)
|
28
|
+
def initialize(url, core=nil)
|
29
|
+
if core.nil?
|
30
|
+
components = url.gsub(%r[/\Z], '').split('/')
|
31
|
+
core = components.last
|
32
|
+
url = components[0..-2].join('/')
|
33
|
+
end
|
29
34
|
super(url)
|
30
35
|
@core = core
|
31
36
|
end
|
@@ -1,4 +1,3 @@
|
|
1
|
-
|
2
1
|
# Figure out how the field type will parse out tokens
|
3
2
|
# and change them in the analysis chain. Just calls the
|
4
3
|
# provided solr analysis endpoints
|
@@ -6,6 +5,17 @@
|
|
6
5
|
# To be mixed into FieldType
|
7
6
|
|
8
7
|
class SimpleSolrClient::Schema
|
8
|
+
|
9
|
+
class InvalidTokenError < RuntimeError
|
10
|
+
attr_accessor :resp
|
11
|
+
|
12
|
+
|
13
|
+
def initialize(msg, resp)
|
14
|
+
super(msg)
|
15
|
+
@resp = resp
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
9
19
|
module Analysis
|
10
20
|
|
11
21
|
#https://lucene.apache.org/solr/4_1_0/solr-core/org/apache/solr/handler/FieldAnalysisRequestHandler.html
|
@@ -16,6 +26,7 @@ class SimpleSolrClient::Schema
|
|
16
26
|
'analysis.query' => val,
|
17
27
|
}
|
18
28
|
resp = @core.get(target, h)
|
29
|
+
|
19
30
|
ftdata = resp['analysis']['field_types'][name][type]
|
20
31
|
rv = []
|
21
32
|
ftdata.last.each do |t|
|
@@ -30,6 +41,7 @@ class SimpleSolrClient::Schema
|
|
30
41
|
rv
|
31
42
|
end
|
32
43
|
|
44
|
+
|
33
45
|
private :fieldtype_tokens
|
34
46
|
|
35
47
|
# Get an array of tokens as analyzed/transformed at index time
|
@@ -47,6 +59,15 @@ class SimpleSolrClient::Schema
|
|
47
59
|
fieldtype_tokens(val, 'index')
|
48
60
|
end
|
49
61
|
|
62
|
+
|
63
|
+
def index_input_valid?(val)
|
64
|
+
index_tokens(val)
|
65
|
+
rescue SimpleSolrClient::Schema::InvalidTokenError, RuntimeError => e
|
66
|
+
puts "IN HERE"
|
67
|
+
require 'pry'; binding.pry
|
68
|
+
end
|
69
|
+
|
70
|
+
|
50
71
|
# Get an array of tokens as analyzed/transformed at query time
|
51
72
|
# See #fieldtype_index_tokens
|
52
73
|
def query_tokens(val)
|
@@ -12,11 +12,22 @@ class SimpleSolrClient::Schema
|
|
12
12
|
@dynamic = false
|
13
13
|
end
|
14
14
|
|
15
|
-
|
16
|
-
|
15
|
+
# We'll defer to the field type when calling any of the attribute
|
16
|
+
# methods
|
17
|
+
([TEXT_ATTR_MAP.keys, BOOL_ATTR_MAP.keys].flatten - [:type_name]).each do |x|
|
18
|
+
define_method(x) do
|
19
|
+
rv = super()
|
20
|
+
if rv.nil?
|
21
|
+
self.type[x]
|
22
|
+
else
|
23
|
+
rv
|
24
|
+
end
|
25
|
+
end
|
17
26
|
end
|
18
27
|
|
19
|
-
|
28
|
+
|
29
|
+
|
30
|
+
# We can only resolve the actual type in the presence of a
|
20
31
|
# particular schema
|
21
32
|
def resolve_type(schema)
|
22
33
|
self.type = schema.field_type(self.type_name)
|
@@ -2,61 +2,52 @@
|
|
2
2
|
module SimpleSolrClient
|
3
3
|
class Schema
|
4
4
|
class Field_or_Type
|
5
|
+
|
6
|
+
include Comparable
|
5
7
|
attr_accessor :name,
|
6
8
|
:type_name
|
7
|
-
|
8
|
-
:stored,
|
9
|
-
:multi,
|
10
|
-
:sort_missing_last,
|
11
|
-
:precision_step,
|
12
|
-
:position_increment_gap
|
9
|
+
|
13
10
|
|
14
11
|
# Take in a hash, and set anything in it that we recognize.
|
15
|
-
# Sloppy from a data point of view, but make
|
12
|
+
# Sloppy from a data point of view, but make for easy
|
16
13
|
# duplication and creation from xml/json
|
17
14
|
|
18
|
-
def initialize(h={})
|
15
|
+
def initialize(h = {})
|
16
|
+
@attributes = {}
|
19
17
|
h.each_pair do |k, v|
|
20
18
|
begin
|
21
19
|
self[k] = v
|
22
20
|
rescue
|
23
21
|
end
|
24
|
-
|
25
22
|
end
|
26
23
|
end
|
27
24
|
|
28
|
-
|
29
25
|
TEXT_ATTR_MAP = {
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
26
|
+
:name => 'name',
|
27
|
+
:type_name => 'type',
|
28
|
+
:precision_step => 'precisionStep',
|
29
|
+
:position_increment_gap => 'positionIncrementGap'
|
34
30
|
}
|
35
31
|
|
36
32
|
BOOL_ATTR_MAP = {
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
33
|
+
:stored => 'stored',
|
34
|
+
:indexed => 'indexed',
|
35
|
+
:multi => 'multiValued',
|
36
|
+
:multivalued => 'multiValued',
|
37
|
+
:multiValued => 'multiValued',
|
38
|
+
:multi_valued => 'multiValued',
|
39
|
+
:sort_missing_last => 'sortMissingLast',
|
40
|
+
:docvalues => 'docValues',
|
41
|
+
:docValues => 'docvalues',
|
42
|
+
:doc_values => 'docvalues',
|
41
43
|
}
|
42
44
|
|
43
|
-
|
44
|
-
|
45
|
-
# once I get the mappings nailed down.
|
46
|
-
[TEXT_ATTR_MAP.keys, BOOL_ATTR_MAP.keys].flatten.delete_if { |x| [:type_name].include? x }.each do |x|
|
47
|
-
define_method(x) do
|
48
|
-
local = instance_variable_get("@#{x}".to_sym)
|
49
|
-
if local.nil?
|
50
|
-
self.type[x] if self.type
|
51
|
-
else
|
52
|
-
local
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
45
|
+
|
46
|
+
|
56
47
|
|
57
48
|
def ==(other)
|
58
49
|
if other.respond_to? :name
|
59
|
-
name == other.name
|
50
|
+
name == other.name and type_name == other.type_name
|
60
51
|
else
|
61
52
|
name == other
|
62
53
|
end
|
@@ -67,44 +58,53 @@ module SimpleSolrClient
|
|
67
58
|
f = self.new
|
68
59
|
|
69
60
|
TEXT_ATTR_MAP.merge(BOOL_ATTR_MAP).each_pair do |field, xmlattr|
|
61
|
+
define_method(field.to_sym) do
|
62
|
+
self[field.to_sym]
|
63
|
+
end
|
70
64
|
f[field] = h[xmlattr]
|
71
65
|
end
|
66
|
+
|
67
|
+
# Make some "method?" for the boolean attributes
|
68
|
+
BOOL_ATTR_MAP.keys.each do |methname|
|
69
|
+
q_methname = ((methname.to_s) + '?').to_sym
|
70
|
+
alias_method q_methname, methname
|
71
|
+
end
|
72
|
+
|
72
73
|
# Set the name "manually" to force the
|
73
74
|
# matcher
|
74
75
|
f.name = h['name']
|
75
|
-
|
76
76
|
f
|
77
77
|
end
|
78
78
|
|
79
79
|
|
80
80
|
# Reverse the process to get XML
|
81
|
-
def to_xml_node
|
81
|
+
def to_xml_node
|
82
82
|
doc ||= Nokogiri::XML::Document.new
|
83
83
|
xml = xml_node(doc)
|
84
84
|
TEXT_ATTR_MAP.merge(BOOL_ATTR_MAP).each_pair do |field, xmlattr|
|
85
|
-
iv
|
85
|
+
iv = instance_variable_get("@#{field}".to_sym)
|
86
86
|
xml[xmlattr] = iv unless iv.nil?
|
87
87
|
end
|
88
88
|
xml
|
89
89
|
end
|
90
90
|
|
91
|
+
def to_xml
|
92
|
+
to_xml_node.to_xml
|
93
|
+
end
|
94
|
+
|
91
95
|
# Allow access to methods via [], for easy looping
|
92
96
|
def [](k)
|
93
|
-
|
97
|
+
@attributes[k.to_sym]
|
94
98
|
end
|
95
99
|
|
96
100
|
def []=(k, v)
|
97
|
-
|
101
|
+
@attributes[k.to_sym] = v
|
98
102
|
end
|
99
103
|
|
100
104
|
|
101
105
|
# Make a hash out of it, for easy feeding back into another call to #new
|
102
106
|
def to_h
|
103
|
-
|
104
|
-
instance_variables.each do |iv|
|
105
|
-
h[iv.to_s.sub('@', '')] = instance_variable_get(iv)
|
106
|
-
end
|
107
|
-
h
|
107
|
+
@attributes
|
108
108
|
end
|
109
109
|
|
110
110
|
end
|
@@ -27,21 +27,6 @@ class SimpleSolrClient::Schema
|
|
27
27
|
nil
|
28
28
|
end
|
29
29
|
|
30
|
-
# Create a Nokogiri node out of the currently-set
|
31
|
-
# element attributes (indexed, stored, etc.) and the
|
32
|
-
# XML
|
33
|
-
def xml_node(doc)
|
34
|
-
ft = Nokogiri::XML::Element.new('fieldType', doc)
|
35
|
-
ft['class'] = self.solr_class
|
36
|
-
xmldoc = Nokogiri.XML(xml)
|
37
|
-
unless xmldoc.children.empty?
|
38
|
-
xmldoc.children.first.children.each do |c|
|
39
|
-
ft.add_child(c)
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
ft
|
44
|
-
end
|
45
30
|
|
46
31
|
def self.new_from_solr_hash(h)
|
47
32
|
ft = super
|
@@ -10,8 +10,6 @@ class SimpleSolrClient::Schema
|
|
10
10
|
# A simplistic representation of a schema
|
11
11
|
|
12
12
|
|
13
|
-
attr_reader :xmldoc
|
14
|
-
|
15
13
|
def initialize(core)
|
16
14
|
@core = core
|
17
15
|
@fields = {}
|
@@ -103,11 +101,7 @@ class SimpleSolrClient::Schema
|
|
103
101
|
|
104
102
|
|
105
103
|
# For loading, we get the information about the fields via the API,
|
106
|
-
# but grab an XML document for modifying/writing
|
107
104
|
def load
|
108
|
-
@xmldoc = Nokogiri.XML(@core.raw_get_content('admin/file', {:file => 'schema.xml'})) do |config|
|
109
|
-
config.noent
|
110
|
-
end
|
111
105
|
load_explicit_fields
|
112
106
|
load_dynamic_fields
|
113
107
|
load_copy_fields
|
@@ -144,48 +138,10 @@ class SimpleSolrClient::Schema
|
|
144
138
|
@field_types = {}
|
145
139
|
@core.get('schema/fieldtypes')['fieldTypes'].each do |fthash|
|
146
140
|
ft = FieldType.new_from_solr_hash(fthash)
|
147
|
-
type_name = ft.name
|
148
|
-
attr = "[@name=\"#{type_name}\"]"
|
149
|
-
node = @xmldoc.css("fieldType#{attr}").first || @xmldoc.css("fieldtype#{attr}").first
|
150
|
-
unless node
|
151
|
-
puts "Failed for type #{type_name}"
|
152
|
-
end
|
153
|
-
ft.xml = node.to_xml
|
154
141
|
add_field_type(ft)
|
155
142
|
end
|
156
143
|
end
|
157
144
|
|
158
|
-
def clean_schema_xml
|
159
|
-
d = @xmldoc.dup
|
160
|
-
d.xpath('//comment()').remove
|
161
|
-
d.css('field').remove
|
162
|
-
d.css('fieldType').remove
|
163
|
-
d.css('fieldtype').remove
|
164
|
-
d.css('dynamicField').remove
|
165
|
-
d.css('copyField').remove
|
166
|
-
d.css('dynamicfield').remove
|
167
|
-
d.css('copyfield').remove
|
168
|
-
d.css('schema').children.find_all { |x| x.name == 'text' }.each { |x| x.remove }
|
169
|
-
d
|
170
|
-
end
|
171
|
-
|
172
|
-
def to_xml
|
173
|
-
# Get a clean schema XML document
|
174
|
-
d = clean_schema_xml
|
175
|
-
s = d.css('schema').first
|
176
|
-
[fields, dynamic_fields, copy_fields, field_types].flatten.each do |f|
|
177
|
-
s.add_child f.to_xml_node
|
178
|
-
end
|
179
|
-
d.to_xml
|
180
|
-
end
|
181
|
-
|
182
|
-
|
183
|
-
def write
|
184
|
-
File.open(@core.schema_file, 'w:utf-8') do |out|
|
185
|
-
out.puts self.to_xml
|
186
|
-
end
|
187
|
-
end
|
188
|
-
|
189
145
|
def reload
|
190
146
|
@core.reload
|
191
147
|
end
|
data/simple_solr_client.gemspec
CHANGED
data/spec/minitest_helper.rb
CHANGED
@@ -8,14 +8,14 @@ Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new
|
|
8
8
|
require 'singleton'
|
9
9
|
|
10
10
|
ENV['TEST_SOLR_URL'] ||= 'http://localhost:8983/solr'
|
11
|
-
ENV['
|
11
|
+
ENV['TEST_SOLR_CORE_NAME'] ||= 'core1'
|
12
12
|
|
13
13
|
class TestClient
|
14
14
|
include Singleton
|
15
15
|
attr_reader :client, :core
|
16
16
|
def initialize
|
17
17
|
@client = SimpleSolrClient::Client.new ENV['TEST_SOLR_URL']
|
18
|
-
@core = @client.core ENV['
|
18
|
+
@core = @client.core ENV['TEST_SOLR_CORE_NAME'] || 'simple_solr_test'
|
19
19
|
end
|
20
20
|
end
|
21
21
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: simple_solr_client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bill Dueber
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-07-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: httpclient
|
@@ -94,20 +94,38 @@ dependencies:
|
|
94
94
|
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: pry
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
97
111
|
description:
|
98
112
|
email:
|
99
113
|
- bill@dueber.com
|
100
|
-
executables:
|
114
|
+
executables:
|
115
|
+
- solr_shell
|
101
116
|
extensions: []
|
102
117
|
extra_rdoc_files: []
|
103
118
|
files:
|
119
|
+
- CHANGES.md
|
104
120
|
- Gemfile
|
105
121
|
- LICENSE.txt
|
106
122
|
- README.md
|
107
123
|
- Rakefile
|
124
|
+
- bin/solr_shell
|
108
125
|
- lib/simple_solr_client.rb
|
109
126
|
- lib/simple_solr_client/client.rb
|
110
127
|
- lib/simple_solr_client/client/core_admin.rb
|
128
|
+
- lib/simple_solr_client/client/system.rb
|
111
129
|
- lib/simple_solr_client/core.rb
|
112
130
|
- lib/simple_solr_client/core/admin.rb
|
113
131
|
- lib/simple_solr_client/core/core_data.rb
|
@@ -255,7 +273,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
255
273
|
version: '0'
|
256
274
|
requirements: []
|
257
275
|
rubyforge_project:
|
258
|
-
rubygems_version: 2.
|
276
|
+
rubygems_version: 2.6.13
|
259
277
|
signing_key:
|
260
278
|
specification_version: 4
|
261
279
|
summary: Interact with a Solr API via JSON
|
@@ -267,4 +285,3 @@ test_files:
|
|
267
285
|
- spec/load_spec.rb
|
268
286
|
- spec/minitest_helper.rb
|
269
287
|
- spec/schema_spec.rb
|
270
|
-
has_rdoc:
|