simple_solr_client 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGES.md +5 -0
- data/README.md +108 -125
- data/bin/solr_shell +40 -0
- data/lib/simple_solr_client/client/system.rb +39 -0
- data/lib/simple_solr_client/client.rb +61 -17
- data/lib/simple_solr_client/core/admin.rb +0 -13
- data/lib/simple_solr_client/core/core_data.rb +20 -5
- data/lib/simple_solr_client/core.rb +6 -1
- data/lib/simple_solr_client/schema/analysis.rb +22 -1
- data/lib/simple_solr_client/schema/field.rb +14 -3
- data/lib/simple_solr_client/schema/field_or_type.rb +42 -42
- data/lib/simple_solr_client/schema/field_type.rb +0 -15
- data/lib/simple_solr_client/schema.rb +0 -44
- data/lib/simple_solr_client/version.rb +1 -1
- data/simple_solr_client.gemspec +1 -0
- data/spec/minitest_helper.rb +2 -2
- metadata +22 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7bd1347fc6a4bdc8dd5cbf6fc240e8341cc1421c
|
4
|
+
data.tar.gz: 34cf45357e585520d46ebeab5756010eab14c0d0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a10b32a6954ee985251d9d3386e0b9dca42bfb3efd677ef3469f4e6b82226efbfd1c70d0e1ceab691f5ce5f8aa293d3154558680918edd5fbd25b30887387e96
|
7
|
+
data.tar.gz: fd691d29fdbd45b255a024be57d89436a9fc2bd8a14177b23d449ff8dafb4c14613392e06229579e4aba60e235f6c17e90c118223d26dc970ef5a6cce35d4b34
|
data/CHANGES.md
ADDED
data/README.md
CHANGED
@@ -1,55 +1,24 @@
|
|
1
1
|
# SimpleSolrClient
|
2
2
|
|
3
|
-
|
4
|
-
|
3
|
+
A simple client and accompanying shell to help test and give simple
|
4
|
+
commands to a solr instance.
|
5
5
|
|
6
|
-
A Solr client specifically designed to try to help you test what the heck
|
7
|
-
solr is actually doing.
|
8
|
-
|
9
|
-
Most useful when running on the same machine as the solr install, but
|
10
|
-
still useful even when you're not.
|
11
|
-
|
12
|
-
|
13
|
-
## Motivation
|
14
|
-
|
15
|
-
Solr is complex.
|
16
|
-
|
17
|
-
It's complex enough, and fuddles with enough edge cases, that reading
|
18
|
-
the documentation and/or the code doesn't get me the understanding
|
19
|
-
that I feel I need.
|
20
|
-
|
21
|
-
If I were smarter, maybe I wouldn't need something like this.
|
22
|
-
|
23
|
-
I wanted a way to test what solr is actually doing, and
|
24
|
-
this library is a way for me to start to do that in a fashion that's
|
25
|
-
more convenient that doing everything "by hand" in the admin dashboard
|
26
|
-
or running queries via URLs in my browser or using curl.
|
27
|
-
|
28
|
-
I wanted a way to figure out what fields (of what types) are being created,
|
29
|
-
how things were being tokenized, etc., but all within the comfort of a test
|
30
|
-
suite that I could run against solr configurations to make sure things
|
31
|
-
weren't breaking when I made changes. I wanted to build up a structure around relevance
|
32
|
-
ranking tests (still coming, sadly) and quickly swap out different
|
33
|
-
configs to make sure it all works as I expect.
|
34
|
-
|
35
|
-
So: a simple solr library, with more exposure than most of what's out there
|
36
|
-
to the solr administration API and the introspection/analysis it affords.
|
37
6
|
|
38
7
|
# Features:
|
39
8
|
|
40
|
-
*
|
41
|
-
*
|
42
|
-
*
|
9
|
+
* Get basic info about cores (size, number of docs, etc.)
|
10
|
+
* Basic (*very basic) add/delete/query
|
11
|
+
* Commit/optimize/clear (empty) an index
|
12
|
+
* Reload a core (presumably after editing a `schema.xml` or `solrconfig.xml`)
|
43
13
|
* Inspect lists of fields, dynamicFields, copyFields, and
|
44
14
|
fieldTypes
|
45
|
-
* Determine which fields (and their properties) would be
|
46
|
-
created when a given field name is indexed, taking into
|
47
|
-
account dynamicField and copyField directives.
|
48
15
|
* Get list of the tokens that would be created if you
|
49
16
|
send a string to a paricular fieldType (like in the
|
50
17
|
solr admin analysis page)
|
51
|
-
*
|
52
|
-
|
18
|
+
* Determine (usually) which fields (and their properties) would be
|
19
|
+
created when a given field name is indexed, taking into
|
20
|
+
account dynamicField and copyField directives.
|
21
|
+
|
53
22
|
|
54
23
|
Additional features when running against a localhost solr:
|
55
24
|
* Spin up a temporary core to play with
|
@@ -83,12 +52,14 @@ core.schema_file #=> <path>/<to>/<schema.xml>
|
|
83
52
|
# Remove all the indexed documents and (automatically) commit
|
84
53
|
core.clear
|
85
54
|
|
55
|
+
core.number_of_documents #=> 0. Automatic commit for #clear
|
56
|
+
|
86
57
|
# Add documents
|
87
58
|
#
|
88
59
|
# name_t is a text_general, multiValued, indexed, stored field
|
89
|
-
h1 = {:id => '
|
90
|
-
h2 = {:id => '
|
91
|
-
h3 = {:id => '
|
60
|
+
h1 = {:id => '1', :name_t=>"Bill Dueber"}
|
61
|
+
h2 = {:id => '2', :name_t=>"Danit Brown"}
|
62
|
+
h3 = {:id => '3', :name_t=>"Ziv Brown Dueber"}
|
92
63
|
|
93
64
|
core.add_docs(h1)
|
94
65
|
|
@@ -112,10 +83,10 @@ docs = core.fv_search(:name_t, 'Brown')
|
|
112
83
|
docs.class #=> SimpleSolrClient::Response::QueryResponse
|
113
84
|
|
114
85
|
docs.size #=> 2
|
115
|
-
docs
|
86
|
+
docs.map{|d| d['name_t']} #=> [['Danit Brown'], ['Ziv Brown Dueber']]
|
116
87
|
|
117
88
|
# Special-case id/score as regular methods
|
118
|
-
docs.first.id #=> '
|
89
|
+
docs.first.id #=> '2'
|
119
90
|
docs.first.score #=> 0.625
|
120
91
|
|
121
92
|
# Figure out where documents fall. "Ziv Brown Dueber" contains both
|
@@ -123,8 +94,8 @@ docs.first.score #=> 0.625
|
|
123
94
|
docs = core.fv_search(:name_t, 'Brown Dueber')
|
124
95
|
docs.size #=> 3
|
125
96
|
|
126
|
-
docs.rank('
|
127
|
-
docs.rank('
|
97
|
+
docs.rank('3') #=> 1 (check by id)
|
98
|
+
docs.rank('3') < docs.rank('b') #=> true
|
128
99
|
|
129
100
|
# Of course, we can do it by score
|
130
101
|
docs.score('z') > docs.score('d')
|
@@ -135,40 +106,116 @@ core.delete('name_t:Dueber').commit.number_of_documents #=> 1
|
|
135
106
|
|
136
107
|
```
|
137
108
|
|
138
|
-
##
|
109
|
+
## Field Types and analysis
|
139
110
|
|
140
|
-
|
141
|
-
|
142
|
-
|
111
|
+
Field Types are created by getting data from the API and also
|
112
|
+
parsing XML out of the schema.xml (for later creating a new
|
113
|
+
schema.xml if you'd like).
|
114
|
+
|
115
|
+
You can also ask a field type how it would tokenize an input
|
116
|
+
string via indexing or querying.
|
117
|
+
|
118
|
+
NOTE: FieldTypes _should_ be able to, say, report their XML serialization even
|
119
|
+
when outside of a particular schema object, but right now that doesn't
|
120
|
+
work. If you make changes to a field type, the only way to see the new
|
121
|
+
serialization is to call `schema.to_xml` on whichever schema you added
|
122
|
+
it to via `schema.add_field_type(ft)`
|
123
|
+
|
124
|
+
|
125
|
+
|
126
|
+
```ruby
|
127
|
+
|
128
|
+
core.schema.field_types.size #=> 23
|
129
|
+
ft = schema.field_type('text') #=> SimpleSolrClient::Schema::FieldType
|
130
|
+
ft.name #=> 'text'
|
131
|
+
ft.solr_class #=> 'solr.TextField'
|
132
|
+
ft.multi #=> true
|
133
|
+
ft.stored #=> true
|
134
|
+
ft.indexed #=> true
|
135
|
+
# etc.
|
136
|
+
|
137
|
+
newft = SimpleSolrClient::Schema::FieldType.new_from_xml(xmlstring)
|
138
|
+
schema.add_field_type(newft)
|
139
|
+
|
140
|
+
ft.name #=> text
|
141
|
+
ft.query_tokens "Don't forget me when I'm getting H20"
|
142
|
+
#=> ["don't", "forget", "me", "when", "i'm", ["getting", "get"], "h20"]
|
143
|
+
|
144
|
+
ft.index_tokens 'When it rains, it pours'
|
145
|
+
#=> ["when", "it", ["rains", "rain"], "it", ["pours", "pour"]]
|
146
|
+
|
147
|
+
|
148
|
+
# Check for validity
|
149
|
+
|
150
|
+
int_type = core.schema.field_type('int')
|
151
|
+
int_type.index_tokens("33") => ["33"]
|
152
|
+
int_type.index_token_valid?("33") #=> true
|
153
|
+
|
154
|
+
int_type.index_token_valid?("33.3") #=> false
|
155
|
+
int_type.index_tokens('33.3') #=> RuntimeError
|
156
|
+
|
157
|
+
|
158
|
+
```
|
159
|
+
|
160
|
+
|
161
|
+
|
162
|
+
## Saving/reloading a changed configuration
|
163
|
+
|
164
|
+
Whether you change a solr install via editing a text file or
|
165
|
+
by using `schema.write`, you can always reload a core.
|
166
|
+
|
167
|
+
```ruby
|
168
|
+
core.reload
|
169
|
+
```
|
170
|
+
|
171
|
+
If you're working on localhost, you can make programmatic changes
|
172
|
+
to the schema and then ask for a write/reload cycle. It uses the API
|
173
|
+
to find the path to the schema.xml file and overwrites it.
|
174
|
+
|
175
|
+
```ruby
|
143
176
|
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
177
|
+
schema = core.schema
|
178
|
+
core.add_field Field.new(:name=>'price', :type_name=>'float')
|
179
|
+
schema.write
|
180
|
+
schema = core.reload.schema
|
181
|
+
```
|
182
|
+
|
183
|
+
|
184
|
+
|
185
|
+
## The `schema` object
|
150
186
|
|
187
|
+
Each core exposes a `schema` object that allows you to find out about
|
188
|
+
the fields, copyfields, and field types, and how they interact with
|
189
|
+
query and index calls (like the analysis screen in the admin interface)
|
151
190
|
|
152
191
|
```ruby
|
153
192
|
|
154
193
|
# Get a list of cores
|
155
|
-
client.cores #=> ['core1']
|
194
|
+
client.cores #=> ['core1', 'core2']
|
156
195
|
core = client.core('core1')
|
157
196
|
|
158
197
|
# Get an object representing the schema.xml file
|
159
198
|
schema = core.schema #=> SimpleSolrClient::Schema object
|
160
199
|
|
161
|
-
# Get lists of field, dynamicFields,
|
200
|
+
# Get lists of field, dynamicFields, and copyFields
|
162
201
|
# all as SimpleSolrClient::Schema::XXX objects
|
163
202
|
|
164
203
|
explicit_fields = schema.fields
|
165
204
|
dynamic_fields = schema.dynamic_fields
|
166
205
|
copy_fields = schema.copy_fields
|
206
|
+
|
207
|
+
# Get a list of FieldType object
|
167
208
|
field_types = schema.field_types
|
209
|
+
field_type_names = schema.field_types.map(&:name)
|
210
|
+
|
211
|
+
# Check out a specific field type and how it parses stuff
|
212
|
+
mytexttype = schema.field_type('mytexttype')
|
213
|
+
mytexttype.index_tokens('bill dueber solr-stuff') #=> ['bill', 'dueber', 'solr', 'stuff']
|
214
|
+
mytexttype.query_tokens('bill dueber solr-stuff') #=> ['bill', 'dueber', 'solr', 'stuff']
|
168
215
|
|
169
216
|
```
|
170
217
|
|
171
|
-
### Regular fields
|
218
|
+
### Regular (non-dynamic) fields
|
172
219
|
|
173
220
|
Internally I call these "explicit_fields" as opposed to dynamic fields.
|
174
221
|
|
@@ -245,48 +292,6 @@ schema.add_copy_field(cf)
|
|
245
292
|
```
|
246
293
|
|
247
294
|
|
248
|
-
### Field Types
|
249
|
-
|
250
|
-
Field Types are created by getting data from the API and also
|
251
|
-
parsing XML out of the schema.xml (for later creating a new
|
252
|
-
schema.xml if you'd like).
|
253
|
-
|
254
|
-
You can also ask a field type how it would tokenize an input
|
255
|
-
string via indexing or querying.
|
256
|
-
|
257
|
-
|
258
|
-
FieldTypes _should_ be able to, say, report their XML serialization even
|
259
|
-
when outside of a particular schema object, but right now that doesn't
|
260
|
-
work. If you make changes to a field type, the only way to see the new
|
261
|
-
serialization is to call `schema.to_xml` on whichever schema you added
|
262
|
-
it to via `schema.add_field_type(ft)`
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
```ruby
|
267
|
-
|
268
|
-
schema.field_types.size #=> 23
|
269
|
-
ft = schema.field_type('text') #=> SimpleSolrClient::Schema::FieldType
|
270
|
-
ft.name #=> 'text'
|
271
|
-
ft.solr_class #=> 'solr.TextField'
|
272
|
-
ft.multi #=> true
|
273
|
-
ft.stored #=> true
|
274
|
-
ft.indexed #=> true
|
275
|
-
# etc.
|
276
|
-
|
277
|
-
newft = SimpleSolrClient::Schema::FieldType.new_from_xml(xmlstring)
|
278
|
-
schema.add_field_type(newft)
|
279
|
-
|
280
|
-
ft.name #=> text
|
281
|
-
ft.query_tokens "Don't forget me when I'm getting H20"
|
282
|
-
#=> ["don't", "forget", "me", "when", "i'm", ["getting", "get"], "h20"]
|
283
|
-
|
284
|
-
ft.index_tokens 'When it rains, it pours'
|
285
|
-
#=> ["when", "it", ["rains", "rain"], "it", ["pours", "pour"]]
|
286
|
-
|
287
|
-
```
|
288
|
-
|
289
|
-
|
290
295
|
## What will I get if I index a field named `str`?
|
291
296
|
|
292
297
|
Dynamic- and copy-fields are very convenient, but it can make it hard to
|
@@ -313,28 +318,6 @@ rs.find_all{|f| f.indexed}.map(&:name) #=> ['name_t']
|
|
313
318
|
```
|
314
319
|
|
315
320
|
|
316
|
-
## Saving/reloading a changed schema
|
317
|
-
|
318
|
-
Whether you change a solr install via editing a text file or
|
319
|
-
by using `schema.write`, you can always reload a core.
|
320
|
-
|
321
|
-
```ruby
|
322
|
-
core.reload
|
323
|
-
```
|
324
|
-
|
325
|
-
If you're working on localhost, you can make programmatic changes
|
326
|
-
to the schema and then ask for a write/reload cycle. It uses the API
|
327
|
-
to find the path to the schema.xml file and overwrites it.
|
328
|
-
|
329
|
-
```ruby
|
330
|
-
|
331
|
-
schema = core.schema
|
332
|
-
core.add_field Field.new(:name=>'price', :type_name=>'float')
|
333
|
-
schema.write
|
334
|
-
schema = core.reload.schema
|
335
|
-
```
|
336
|
-
|
337
|
-
|
338
321
|
## Installation
|
339
322
|
|
340
323
|
$ gem install simple_solr
|
data/bin/solr_shell
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
|
4
|
+
# If we're loading from source instead of a gem, we need to
|
5
|
+
# set the load path directly
|
6
|
+
self_load_path = File.expand_path("../lib", File.dirname(__FILE__))
|
7
|
+
unless $LOAD_PATH.include? self_load_path
|
8
|
+
$LOAD_PATH << self_load_path
|
9
|
+
end
|
10
|
+
|
11
|
+
require 'simple_solr_client'
|
12
|
+
|
13
|
+
require 'pry'
|
14
|
+
|
15
|
+
solr_url = ARGV[0]
|
16
|
+
|
17
|
+
client = SimpleSolrClient::Client.new(solr_url)
|
18
|
+
|
19
|
+
|
20
|
+
|
21
|
+
|
22
|
+
pry = Pry.new
|
23
|
+
cb = pry.current_binding
|
24
|
+
puts "
|
25
|
+
Connected to #{client.url}
|
26
|
+
|
27
|
+
"
|
28
|
+
|
29
|
+
cb.local_variable_set(:solr, client)
|
30
|
+
puts " * Created variable 'solr' for the main solr client"
|
31
|
+
|
32
|
+
client.cores.each do |core|
|
33
|
+
cb.local_variable_set :"#{core}", client.core(core)
|
34
|
+
puts " * Created variable '#{core}' pointing to its core"
|
35
|
+
end
|
36
|
+
|
37
|
+
puts "\n\n"
|
38
|
+
|
39
|
+
|
40
|
+
cb.pry
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module SimpleSolrClient
|
2
|
+
# Attributes of the solr process itself
|
3
|
+
class System
|
4
|
+
|
5
|
+
def initialize(sysresp)
|
6
|
+
@resp = sysresp
|
7
|
+
end
|
8
|
+
|
9
|
+
# @return [String] Full lucene version, with release data and everything
|
10
|
+
def lucene_full_version
|
11
|
+
@resp['lucene']['lucene-impl-version']
|
12
|
+
end
|
13
|
+
|
14
|
+
# @return [String] Lucene version as M.m.p
|
15
|
+
def lucene_semver_version
|
16
|
+
@resp['lucene']['lucene-spec-version']
|
17
|
+
end
|
18
|
+
|
19
|
+
# @return [Integer] The major lucene version (e.g., 7)
|
20
|
+
def lucene_major_version
|
21
|
+
lucene_full_version.split('.').first.to_i
|
22
|
+
end
|
23
|
+
|
24
|
+
# @return [String] Full lucene version, with release data and everything
|
25
|
+
def solr_full_version
|
26
|
+
@resp['lucene']['solr-impl-version']
|
27
|
+
end
|
28
|
+
|
29
|
+
# @return [String] Lucene version as M.m.p
|
30
|
+
def solr_semver_version
|
31
|
+
@resp['lucene']['solr-spec-version']
|
32
|
+
end
|
33
|
+
|
34
|
+
# @return [Integer] The major lucene version (e.g., 7)
|
35
|
+
def solr_major_version
|
36
|
+
solr_semver_version.split('.').first.to_i
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -3,6 +3,7 @@ require 'simple_solr_client/response/generic_response'
|
|
3
3
|
require 'securerandom'
|
4
4
|
|
5
5
|
require 'simple_solr_client/core'
|
6
|
+
require 'simple_solr_client/client/system'
|
6
7
|
|
7
8
|
module SimpleSolrClient
|
8
9
|
|
@@ -13,10 +14,18 @@ module SimpleSolrClient
|
|
13
14
|
|
14
15
|
attr_reader :base_url, :rawclient
|
15
16
|
|
16
|
-
def initialize(
|
17
|
-
|
17
|
+
def initialize(url_or_port)
|
18
|
+
url = if url_or_port.is_a?(Integer)
|
19
|
+
"http://localhost:#{url_or_port}/solr"
|
20
|
+
else
|
21
|
+
url_or_port
|
22
|
+
end
|
23
|
+
|
24
|
+
@base_url = url.chomp('/')
|
18
25
|
@client_url = @base_url
|
19
|
-
@rawclient
|
26
|
+
@rawclient = HTTPClient.new
|
27
|
+
|
28
|
+
# raise "Can't connect to Solr at #{url}" unless self.up?
|
20
29
|
end
|
21
30
|
|
22
31
|
# Construct a URL for the given arguments that hit the configured solr
|
@@ -31,12 +40,45 @@ module SimpleSolrClient
|
|
31
40
|
[@client_url, *args].join('/').chomp('/')
|
32
41
|
end
|
33
42
|
|
43
|
+
def ping
|
44
|
+
get('admin/ping')
|
45
|
+
end
|
46
|
+
|
47
|
+
# Get info about the solr system itself
|
48
|
+
def system
|
49
|
+
@system ||= SimpleSolrClient::System.new(get('admin/info/system'))
|
50
|
+
end
|
51
|
+
|
52
|
+
# @return [String] The solr semver version
|
53
|
+
def version
|
54
|
+
system.solr_semver_version
|
55
|
+
end
|
56
|
+
|
57
|
+
# @return [Integer] the solr major version
|
58
|
+
def major_version
|
59
|
+
system.solr_major_version
|
60
|
+
end
|
61
|
+
|
62
|
+
# Is the server up (and responding to a ping?)
|
63
|
+
# @return [Boolean]
|
64
|
+
def up?
|
65
|
+
begin
|
66
|
+
ping.status == 'OK'
|
67
|
+
rescue
|
68
|
+
false
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
|
34
73
|
|
35
|
-
# Call a get on the underlying http client and return the content
|
36
|
-
#
|
37
|
-
#
|
74
|
+
# Call a 'get' on the underlying http client and return the content
|
75
|
+
# Will use whatever the URL is for the current context ("client" or
|
76
|
+
# "core"), although you can pass in :force_top_level=>true for those
|
77
|
+
# cases when you absolutely have to use the client-level url and not a
|
38
78
|
# core level URL
|
39
|
-
|
79
|
+
#
|
80
|
+
# Error handling? What error handling???
|
81
|
+
def raw_get_content(path, args = {})
|
40
82
|
if args.delete(:force_top_level_url)
|
41
83
|
u = top_level_url(path)
|
42
84
|
else
|
@@ -50,10 +92,10 @@ module SimpleSolrClient
|
|
50
92
|
# @param [String] path The parts of the URL that comes after the core
|
51
93
|
# @param [Hash] args The url arguments
|
52
94
|
# @return [Hash] the parsed-out response
|
53
|
-
def _get(path, args={})
|
95
|
+
def _get(path, args = {})
|
54
96
|
path.sub! /\A\//, ''
|
55
97
|
args['wt'] = 'json'
|
56
|
-
res
|
98
|
+
res = JSON.parse(raw_get_content(path, args))
|
57
99
|
if res['error']
|
58
100
|
raise RuntimeError.new, res['error']
|
59
101
|
end
|
@@ -89,25 +131,27 @@ module SimpleSolrClient
|
|
89
131
|
# @param [String] corename The name of the core (which must already exist!)
|
90
132
|
# @return [SimpleSolrClient::Core]
|
91
133
|
def core(corename)
|
92
|
-
|
134
|
+
raise "Core #{corename} not found" unless cores.include? corename.to_s
|
135
|
+
SimpleSolrClient::Core.new(@base_url, corename.to_s)
|
93
136
|
end
|
94
137
|
|
95
138
|
|
139
|
+
# Get all the cores
|
96
140
|
def cores
|
97
|
-
cdata = get('admin/cores', {:force_top_level_url=>true}).status.keys
|
141
|
+
cdata = get('admin/cores', {:force_top_level_url => true}).status.keys
|
98
142
|
end
|
99
143
|
|
100
144
|
|
101
145
|
# Create a new, temporary core
|
102
146
|
#noinspection RubyWrongHash
|
103
147
|
def new_core(corename)
|
104
|
-
dir
|
148
|
+
dir = temp_core_dir_setup(corename)
|
105
149
|
|
106
150
|
args = {
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
151
|
+
:wt => 'json',
|
152
|
+
:action => 'CREATE',
|
153
|
+
:name => corename,
|
154
|
+
:instanceDir => dir
|
111
155
|
}
|
112
156
|
|
113
157
|
get('admin/cores', args)
|
@@ -121,7 +165,7 @@ module SimpleSolrClient
|
|
121
165
|
|
122
166
|
# Set up files for a temp core
|
123
167
|
def temp_core_dir_setup(corename)
|
124
|
-
dest = Dir.mktmpdir("simple_solr_#{corename}")
|
168
|
+
dest = Dir.mktmpdir("simple_solr_#{corename}_#{SecureRandom.uuid}")
|
125
169
|
src = SAMPLE_CORE_DIR
|
126
170
|
FileUtils.cp_r File.join(src, '.'), dest
|
127
171
|
dest
|
@@ -1,17 +1,4 @@
|
|
1
1
|
module SimpleSolrClient::Core::Admin
|
2
|
-
def ping
|
3
|
-
get('admin/ping')
|
4
|
-
end
|
5
|
-
|
6
|
-
# Is the server up (and responding to a ping?)
|
7
|
-
# @return [Boolean]
|
8
|
-
def up?
|
9
|
-
begin
|
10
|
-
ping.status == 'OK'
|
11
|
-
rescue
|
12
|
-
false
|
13
|
-
end
|
14
|
-
end
|
15
2
|
|
16
3
|
# Send a commit command
|
17
4
|
# @return self
|
@@ -18,22 +18,38 @@ module SimpleSolrClient::Core::CoreData
|
|
18
18
|
core_data_hash['index']
|
19
19
|
end
|
20
20
|
|
21
|
-
|
22
|
-
core_data_hash['isDefaultCore']
|
23
|
-
end
|
24
|
-
|
21
|
+
# Time of last modification
|
25
22
|
def last_modified
|
26
23
|
Time.parse index['lastModified']
|
27
24
|
end
|
28
25
|
|
26
|
+
# Total documents
|
29
27
|
def number_of_documents
|
30
28
|
index['numDocs']
|
31
29
|
end
|
32
30
|
|
31
|
+
alias_method :numDocs, :number_of_documents
|
32
|
+
alias_method :num_docs, :number_of_documents
|
33
|
+
|
34
|
+
# The (local to the server) data directory
|
33
35
|
def data_dir
|
34
36
|
core_data_hash['dataDir']
|
35
37
|
end
|
36
38
|
|
39
|
+
|
40
|
+
# Get the index size in megabytes
|
41
|
+
def size
|
42
|
+
str = index['size']
|
43
|
+
num, unit = str.split(/\s+/).compact.map(&:strip)
|
44
|
+
num = num.to_f
|
45
|
+
case unit
|
46
|
+
when "MB"
|
47
|
+
num * 1
|
48
|
+
when "GB"
|
49
|
+
num * 1000
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
37
53
|
def instance_dir
|
38
54
|
core_data_hash['instanceDir']
|
39
55
|
end
|
@@ -45,7 +61,6 @@ module SimpleSolrClient::Core::CoreData
|
|
45
61
|
def schema_file
|
46
62
|
File.join(instance_dir, 'conf', core_data_hash['schema'])
|
47
63
|
end
|
48
|
-
|
49
64
|
end
|
50
65
|
|
51
66
|
|
@@ -25,7 +25,12 @@ class SimpleSolrClient::Core
|
|
25
25
|
attr_reader :core
|
26
26
|
alias_method :name, :core
|
27
27
|
|
28
|
-
def initialize(url, core)
|
28
|
+
def initialize(url, core=nil)
|
29
|
+
if core.nil?
|
30
|
+
components = url.gsub(%r[/\Z], '').split('/')
|
31
|
+
core = components.last
|
32
|
+
url = components[0..-2].join('/')
|
33
|
+
end
|
29
34
|
super(url)
|
30
35
|
@core = core
|
31
36
|
end
|
@@ -1,4 +1,3 @@
|
|
1
|
-
|
2
1
|
# Figure out how the field type will parse out tokens
|
3
2
|
# and change them in the analysis chain. Just calls the
|
4
3
|
# provided solr analysis endpoints
|
@@ -6,6 +5,17 @@
|
|
6
5
|
# To be mixed into FieldType
|
7
6
|
|
8
7
|
class SimpleSolrClient::Schema
|
8
|
+
|
9
|
+
class InvalidTokenError < RuntimeError
|
10
|
+
attr_accessor :resp
|
11
|
+
|
12
|
+
|
13
|
+
def initialize(msg, resp)
|
14
|
+
super(msg)
|
15
|
+
@resp = resp
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
9
19
|
module Analysis
|
10
20
|
|
11
21
|
#https://lucene.apache.org/solr/4_1_0/solr-core/org/apache/solr/handler/FieldAnalysisRequestHandler.html
|
@@ -16,6 +26,7 @@ class SimpleSolrClient::Schema
|
|
16
26
|
'analysis.query' => val,
|
17
27
|
}
|
18
28
|
resp = @core.get(target, h)
|
29
|
+
|
19
30
|
ftdata = resp['analysis']['field_types'][name][type]
|
20
31
|
rv = []
|
21
32
|
ftdata.last.each do |t|
|
@@ -30,6 +41,7 @@ class SimpleSolrClient::Schema
|
|
30
41
|
rv
|
31
42
|
end
|
32
43
|
|
44
|
+
|
33
45
|
private :fieldtype_tokens
|
34
46
|
|
35
47
|
# Get an array of tokens as analyzed/transformed at index time
|
@@ -47,6 +59,15 @@ class SimpleSolrClient::Schema
|
|
47
59
|
fieldtype_tokens(val, 'index')
|
48
60
|
end
|
49
61
|
|
62
|
+
|
63
|
+
def index_input_valid?(val)
|
64
|
+
index_tokens(val)
|
65
|
+
rescue SimpleSolrClient::Schema::InvalidTokenError, RuntimeError => e
|
66
|
+
puts "IN HERE"
|
67
|
+
require 'pry'; binding.pry
|
68
|
+
end
|
69
|
+
|
70
|
+
|
50
71
|
# Get an array of tokens as analyzed/transformed at query time
|
51
72
|
# See #fieldtype_index_tokens
|
52
73
|
def query_tokens(val)
|
@@ -12,11 +12,22 @@ class SimpleSolrClient::Schema
|
|
12
12
|
@dynamic = false
|
13
13
|
end
|
14
14
|
|
15
|
-
|
16
|
-
|
15
|
+
# We'll defer to the field type when calling any of the attribute
|
16
|
+
# methods
|
17
|
+
([TEXT_ATTR_MAP.keys, BOOL_ATTR_MAP.keys].flatten - [:type_name]).each do |x|
|
18
|
+
define_method(x) do
|
19
|
+
rv = super()
|
20
|
+
if rv.nil?
|
21
|
+
self.type[x]
|
22
|
+
else
|
23
|
+
rv
|
24
|
+
end
|
25
|
+
end
|
17
26
|
end
|
18
27
|
|
19
|
-
|
28
|
+
|
29
|
+
|
30
|
+
# We can only resolve the actual type in the presence of a
|
20
31
|
# particular schema
|
21
32
|
def resolve_type(schema)
|
22
33
|
self.type = schema.field_type(self.type_name)
|
@@ -2,61 +2,52 @@
|
|
2
2
|
module SimpleSolrClient
|
3
3
|
class Schema
|
4
4
|
class Field_or_Type
|
5
|
+
|
6
|
+
include Comparable
|
5
7
|
attr_accessor :name,
|
6
8
|
:type_name
|
7
|
-
|
8
|
-
:stored,
|
9
|
-
:multi,
|
10
|
-
:sort_missing_last,
|
11
|
-
:precision_step,
|
12
|
-
:position_increment_gap
|
9
|
+
|
13
10
|
|
14
11
|
# Take in a hash, and set anything in it that we recognize.
|
15
|
-
# Sloppy from a data point of view, but make
|
12
|
+
# Sloppy from a data point of view, but make for easy
|
16
13
|
# duplication and creation from xml/json
|
17
14
|
|
18
|
-
def initialize(h={})
|
15
|
+
def initialize(h = {})
|
16
|
+
@attributes = {}
|
19
17
|
h.each_pair do |k, v|
|
20
18
|
begin
|
21
19
|
self[k] = v
|
22
20
|
rescue
|
23
21
|
end
|
24
|
-
|
25
22
|
end
|
26
23
|
end
|
27
24
|
|
28
|
-
|
29
25
|
TEXT_ATTR_MAP = {
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
26
|
+
:name => 'name',
|
27
|
+
:type_name => 'type',
|
28
|
+
:precision_step => 'precisionStep',
|
29
|
+
:position_increment_gap => 'positionIncrementGap'
|
34
30
|
}
|
35
31
|
|
36
32
|
BOOL_ATTR_MAP = {
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
33
|
+
:stored => 'stored',
|
34
|
+
:indexed => 'indexed',
|
35
|
+
:multi => 'multiValued',
|
36
|
+
:multivalued => 'multiValued',
|
37
|
+
:multiValued => 'multiValued',
|
38
|
+
:multi_valued => 'multiValued',
|
39
|
+
:sort_missing_last => 'sortMissingLast',
|
40
|
+
:docvalues => 'docValues',
|
41
|
+
:docValues => 'docvalues',
|
42
|
+
:doc_values => 'docvalues',
|
41
43
|
}
|
42
44
|
|
43
|
-
|
44
|
-
|
45
|
-
# once I get the mappings nailed down.
|
46
|
-
[TEXT_ATTR_MAP.keys, BOOL_ATTR_MAP.keys].flatten.delete_if { |x| [:type_name].include? x }.each do |x|
|
47
|
-
define_method(x) do
|
48
|
-
local = instance_variable_get("@#{x}".to_sym)
|
49
|
-
if local.nil?
|
50
|
-
self.type[x] if self.type
|
51
|
-
else
|
52
|
-
local
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
45
|
+
|
46
|
+
|
56
47
|
|
57
48
|
def ==(other)
|
58
49
|
if other.respond_to? :name
|
59
|
-
name == other.name
|
50
|
+
name == other.name and type_name == other.type_name
|
60
51
|
else
|
61
52
|
name == other
|
62
53
|
end
|
@@ -67,44 +58,53 @@ module SimpleSolrClient
|
|
67
58
|
f = self.new
|
68
59
|
|
69
60
|
TEXT_ATTR_MAP.merge(BOOL_ATTR_MAP).each_pair do |field, xmlattr|
|
61
|
+
define_method(field.to_sym) do
|
62
|
+
self[field.to_sym]
|
63
|
+
end
|
70
64
|
f[field] = h[xmlattr]
|
71
65
|
end
|
66
|
+
|
67
|
+
# Make some "method?" for the boolean attributes
|
68
|
+
BOOL_ATTR_MAP.keys.each do |methname|
|
69
|
+
q_methname = ((methname.to_s) + '?').to_sym
|
70
|
+
alias_method q_methname, methname
|
71
|
+
end
|
72
|
+
|
72
73
|
# Set the name "manually" to force the
|
73
74
|
# matcher
|
74
75
|
f.name = h['name']
|
75
|
-
|
76
76
|
f
|
77
77
|
end
|
78
78
|
|
79
79
|
|
80
80
|
# Reverse the process to get XML
|
81
|
-
def to_xml_node
|
81
|
+
def to_xml_node
|
82
82
|
doc ||= Nokogiri::XML::Document.new
|
83
83
|
xml = xml_node(doc)
|
84
84
|
TEXT_ATTR_MAP.merge(BOOL_ATTR_MAP).each_pair do |field, xmlattr|
|
85
|
-
iv
|
85
|
+
iv = instance_variable_get("@#{field}".to_sym)
|
86
86
|
xml[xmlattr] = iv unless iv.nil?
|
87
87
|
end
|
88
88
|
xml
|
89
89
|
end
|
90
90
|
|
91
|
+
def to_xml
|
92
|
+
to_xml_node.to_xml
|
93
|
+
end
|
94
|
+
|
91
95
|
# Allow access to methods via [], for easy looping
|
92
96
|
def [](k)
|
93
|
-
|
97
|
+
@attributes[k.to_sym]
|
94
98
|
end
|
95
99
|
|
96
100
|
def []=(k, v)
|
97
|
-
|
101
|
+
@attributes[k.to_sym] = v
|
98
102
|
end
|
99
103
|
|
100
104
|
|
101
105
|
# Make a hash out of it, for easy feeding back into another call to #new
|
102
106
|
def to_h
|
103
|
-
|
104
|
-
instance_variables.each do |iv|
|
105
|
-
h[iv.to_s.sub('@', '')] = instance_variable_get(iv)
|
106
|
-
end
|
107
|
-
h
|
107
|
+
@attributes
|
108
108
|
end
|
109
109
|
|
110
110
|
end
|
@@ -27,21 +27,6 @@ class SimpleSolrClient::Schema
|
|
27
27
|
nil
|
28
28
|
end
|
29
29
|
|
30
|
-
# Create a Nokogiri node out of the currently-set
|
31
|
-
# element attributes (indexed, stored, etc.) and the
|
32
|
-
# XML
|
33
|
-
def xml_node(doc)
|
34
|
-
ft = Nokogiri::XML::Element.new('fieldType', doc)
|
35
|
-
ft['class'] = self.solr_class
|
36
|
-
xmldoc = Nokogiri.XML(xml)
|
37
|
-
unless xmldoc.children.empty?
|
38
|
-
xmldoc.children.first.children.each do |c|
|
39
|
-
ft.add_child(c)
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
ft
|
44
|
-
end
|
45
30
|
|
46
31
|
def self.new_from_solr_hash(h)
|
47
32
|
ft = super
|
@@ -10,8 +10,6 @@ class SimpleSolrClient::Schema
|
|
10
10
|
# A simplistic representation of a schema
|
11
11
|
|
12
12
|
|
13
|
-
attr_reader :xmldoc
|
14
|
-
|
15
13
|
def initialize(core)
|
16
14
|
@core = core
|
17
15
|
@fields = {}
|
@@ -103,11 +101,7 @@ class SimpleSolrClient::Schema
|
|
103
101
|
|
104
102
|
|
105
103
|
# For loading, we get the information about the fields via the API,
|
106
|
-
# but grab an XML document for modifying/writing
|
107
104
|
def load
|
108
|
-
@xmldoc = Nokogiri.XML(@core.raw_get_content('admin/file', {:file => 'schema.xml'})) do |config|
|
109
|
-
config.noent
|
110
|
-
end
|
111
105
|
load_explicit_fields
|
112
106
|
load_dynamic_fields
|
113
107
|
load_copy_fields
|
@@ -144,48 +138,10 @@ class SimpleSolrClient::Schema
|
|
144
138
|
@field_types = {}
|
145
139
|
@core.get('schema/fieldtypes')['fieldTypes'].each do |fthash|
|
146
140
|
ft = FieldType.new_from_solr_hash(fthash)
|
147
|
-
type_name = ft.name
|
148
|
-
attr = "[@name=\"#{type_name}\"]"
|
149
|
-
node = @xmldoc.css("fieldType#{attr}").first || @xmldoc.css("fieldtype#{attr}").first
|
150
|
-
unless node
|
151
|
-
puts "Failed for type #{type_name}"
|
152
|
-
end
|
153
|
-
ft.xml = node.to_xml
|
154
141
|
add_field_type(ft)
|
155
142
|
end
|
156
143
|
end
|
157
144
|
|
158
|
-
def clean_schema_xml
|
159
|
-
d = @xmldoc.dup
|
160
|
-
d.xpath('//comment()').remove
|
161
|
-
d.css('field').remove
|
162
|
-
d.css('fieldType').remove
|
163
|
-
d.css('fieldtype').remove
|
164
|
-
d.css('dynamicField').remove
|
165
|
-
d.css('copyField').remove
|
166
|
-
d.css('dynamicfield').remove
|
167
|
-
d.css('copyfield').remove
|
168
|
-
d.css('schema').children.find_all { |x| x.name == 'text' }.each { |x| x.remove }
|
169
|
-
d
|
170
|
-
end
|
171
|
-
|
172
|
-
def to_xml
|
173
|
-
# Get a clean schema XML document
|
174
|
-
d = clean_schema_xml
|
175
|
-
s = d.css('schema').first
|
176
|
-
[fields, dynamic_fields, copy_fields, field_types].flatten.each do |f|
|
177
|
-
s.add_child f.to_xml_node
|
178
|
-
end
|
179
|
-
d.to_xml
|
180
|
-
end
|
181
|
-
|
182
|
-
|
183
|
-
def write
|
184
|
-
File.open(@core.schema_file, 'w:utf-8') do |out|
|
185
|
-
out.puts self.to_xml
|
186
|
-
end
|
187
|
-
end
|
188
|
-
|
189
145
|
def reload
|
190
146
|
@core.reload
|
191
147
|
end
|
data/simple_solr_client.gemspec
CHANGED
data/spec/minitest_helper.rb
CHANGED
@@ -8,14 +8,14 @@ Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new
|
|
8
8
|
require 'singleton'
|
9
9
|
|
10
10
|
ENV['TEST_SOLR_URL'] ||= 'http://localhost:8983/solr'
|
11
|
-
ENV['
|
11
|
+
ENV['TEST_SOLR_CORE_NAME'] ||= 'core1'
|
12
12
|
|
13
13
|
class TestClient
|
14
14
|
include Singleton
|
15
15
|
attr_reader :client, :core
|
16
16
|
def initialize
|
17
17
|
@client = SimpleSolrClient::Client.new ENV['TEST_SOLR_URL']
|
18
|
-
@core = @client.core ENV['
|
18
|
+
@core = @client.core ENV['TEST_SOLR_CORE_NAME'] || 'simple_solr_test'
|
19
19
|
end
|
20
20
|
end
|
21
21
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: simple_solr_client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bill Dueber
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-07-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: httpclient
|
@@ -94,20 +94,38 @@ dependencies:
|
|
94
94
|
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: pry
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
97
111
|
description:
|
98
112
|
email:
|
99
113
|
- bill@dueber.com
|
100
|
-
executables:
|
114
|
+
executables:
|
115
|
+
- solr_shell
|
101
116
|
extensions: []
|
102
117
|
extra_rdoc_files: []
|
103
118
|
files:
|
119
|
+
- CHANGES.md
|
104
120
|
- Gemfile
|
105
121
|
- LICENSE.txt
|
106
122
|
- README.md
|
107
123
|
- Rakefile
|
124
|
+
- bin/solr_shell
|
108
125
|
- lib/simple_solr_client.rb
|
109
126
|
- lib/simple_solr_client/client.rb
|
110
127
|
- lib/simple_solr_client/client/core_admin.rb
|
128
|
+
- lib/simple_solr_client/client/system.rb
|
111
129
|
- lib/simple_solr_client/core.rb
|
112
130
|
- lib/simple_solr_client/core/admin.rb
|
113
131
|
- lib/simple_solr_client/core/core_data.rb
|
@@ -255,7 +273,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
255
273
|
version: '0'
|
256
274
|
requirements: []
|
257
275
|
rubyforge_project:
|
258
|
-
rubygems_version: 2.
|
276
|
+
rubygems_version: 2.6.13
|
259
277
|
signing_key:
|
260
278
|
specification_version: 4
|
261
279
|
summary: Interact with a Solr API via JSON
|
@@ -267,4 +285,3 @@ test_files:
|
|
267
285
|
- spec/load_spec.rb
|
268
286
|
- spec/minitest_helper.rb
|
269
287
|
- spec/schema_spec.rb
|
270
|
-
has_rdoc:
|