typesensual 0.1.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +8 -6
- data/README.md +24 -19
- data/lib/tasks/typesensual.rake +11 -1
- data/lib/typesensual/callbacks.rb +2 -3
- data/lib/typesensual/collection.rb +14 -1
- data/lib/typesensual/config.rb +13 -3
- data/lib/typesensual/field.rb +1 -1
- data/lib/typesensual/index.rb +20 -11
- data/lib/typesensual/rake_helper.rb +19 -0
- data/lib/typesensual/schema.rb +14 -1
- data/lib/typesensual/search/grouped_hit.rb +37 -0
- data/lib/typesensual/search/hit.rb +1 -1
- data/lib/typesensual/search/results.rb +4 -0
- data/lib/typesensual/search.rb +57 -1
- data/lib/typesensual/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 16c9d2bb0a45de749680a833abf874d3f42da4190ea66b88e5d1f19a8f2d902c
|
4
|
+
data.tar.gz: d4f9dc200a97e182929b8cdc6fbb48edc1da69e6444137e2f7f670197742fa2a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7d3130f6ac4d1b5d2722c4a6020fab73dc6ce1b91b7095e0f91de2d50723a60c5edb5434f651f874182c868a22e4015b12aa0eed097270562bc2bd291640a2af
|
7
|
+
data.tar.gz: c1fc489fab6f2c1fc29f57a0662883115b37b1790a02eab0db6193be53fbf4174de8e464b937f42b54ee2c3098d57abc8dc529f5d081728cb63861c75b302242
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
typesensual (0.
|
4
|
+
typesensual (0.3.0)
|
5
5
|
activesupport (>= 6.1.5)
|
6
6
|
paint (>= 2.0.0)
|
7
7
|
typesense (>= 0.13.0)
|
@@ -9,7 +9,7 @@ PATH
|
|
9
9
|
GEM
|
10
10
|
remote: https://rubygems.org/
|
11
11
|
specs:
|
12
|
-
activesupport (7.0.
|
12
|
+
activesupport (7.0.7)
|
13
13
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
14
14
|
i18n (>= 1.6, < 2)
|
15
15
|
minitest (>= 5.1)
|
@@ -22,11 +22,11 @@ GEM
|
|
22
22
|
ethon (0.16.0)
|
23
23
|
ffi (>= 1.15.0)
|
24
24
|
ffi (1.15.5)
|
25
|
-
i18n (1.
|
25
|
+
i18n (1.14.1)
|
26
26
|
concurrent-ruby (~> 1.0)
|
27
27
|
json (2.6.3)
|
28
|
-
minitest (5.
|
29
|
-
oj (3.
|
28
|
+
minitest (5.19.0)
|
29
|
+
oj (3.16.0)
|
30
30
|
paint (2.3.0)
|
31
31
|
parallel (1.23.0)
|
32
32
|
parser (3.2.2.1)
|
@@ -79,7 +79,7 @@ GEM
|
|
79
79
|
simplecov_json_formatter (~> 0.1)
|
80
80
|
simplecov-html (0.12.3)
|
81
81
|
simplecov_json_formatter (0.1.4)
|
82
|
-
typesense (0.
|
82
|
+
typesense (0.15.0)
|
83
83
|
oj (~> 3.11)
|
84
84
|
typhoeus (~> 1.4)
|
85
85
|
typhoeus (1.4.0)
|
@@ -91,8 +91,10 @@ GEM
|
|
91
91
|
yard (0.9.34)
|
92
92
|
|
93
93
|
PLATFORMS
|
94
|
+
ruby
|
94
95
|
x86_64-darwin-19
|
95
96
|
x86_64-darwin-22
|
97
|
+
x86_64-linux
|
96
98
|
|
97
99
|
DEPENDENCIES
|
98
100
|
commonmarker
|
data/README.md
CHANGED
@@ -50,6 +50,14 @@ Typesensual.configure do |config|
|
|
50
50
|
end
|
51
51
|
```
|
52
52
|
|
53
|
+
Alternatively you can configure with env variables:
|
54
|
+
|
55
|
+
```env
|
56
|
+
TYPESENSUAL_NODES=http://node1:8108,http://node2:8108,http://node3:8108
|
57
|
+
TYPESENSUAL_API_KEY=xyz
|
58
|
+
TYPESENSUAL_ENV=test
|
59
|
+
```
|
60
|
+
|
53
61
|
### Creating your first index
|
54
62
|
|
55
63
|
Once the client is configured, you can create your first index. This is done by creating a subclass
|
@@ -57,7 +65,7 @@ of `Typesensual::Index` and defining your schema and how to load the data. For e
|
|
57
65
|
following index might be used to index movies from an ActiveRecord model:
|
58
66
|
|
59
67
|
```ruby
|
60
|
-
# app/indices/
|
68
|
+
# app/indices/movies_index.rb
|
61
69
|
class MoviesIndex < Typesensual::Index
|
62
70
|
# The schema of the collection
|
63
71
|
schema do
|
@@ -70,24 +78,7 @@ class MoviesIndex < Typesensual::Index
|
|
70
78
|
field 'genres', type: 'string[]', facet: true
|
71
79
|
end
|
72
80
|
|
73
|
-
def
|
74
|
-
movie = Movie.find(id).includes(:genres)
|
75
|
-
|
76
|
-
{
|
77
|
-
id: movie.id,
|
78
|
-
title: movie.title,
|
79
|
-
release_date: {
|
80
|
-
year: movie.release_date.year,
|
81
|
-
month: movie.release_date.month,
|
82
|
-
day: movie.release_date.day
|
83
|
-
},
|
84
|
-
average_rating: movie.average_rating,
|
85
|
-
user_count: movie.user_count,
|
86
|
-
genres: movie.genres.map(&:name)
|
87
|
-
}
|
88
|
-
end
|
89
|
-
|
90
|
-
def index_many(ids)
|
81
|
+
def index(ids)
|
91
82
|
Movies.where(id: ids).includes(:genres).find_each do |movie|
|
92
83
|
yield {
|
93
84
|
id: movie.id,
|
@@ -106,6 +97,20 @@ class MoviesIndex < Typesensual::Index
|
|
106
97
|
end
|
107
98
|
```
|
108
99
|
|
100
|
+
### Integrating with your model
|
101
|
+
|
102
|
+
If you use ActiveRecord, there's a set of premade callbacks you can use:
|
103
|
+
|
104
|
+
```ruby
|
105
|
+
class Movie < ApplicationRecord
|
106
|
+
after_commit MoviesIndex.ar_callbacks, on: %i[create update destroy]
|
107
|
+
end
|
108
|
+
```
|
109
|
+
|
110
|
+
You're free to use these callbacks as-is, or you can use them as a starting point for your own
|
111
|
+
integration. They're just calling `MoviesIndex.index_one` and `MoviesIndex.remove_one` under the
|
112
|
+
hood, so you can do the same in your own callbacks or outside of ActiveRecord.
|
113
|
+
|
109
114
|
### Loading data into your index
|
110
115
|
|
111
116
|
Once you have defined your index, you can load data into it and update the alias to point to the
|
data/lib/tasks/typesensual.rake
CHANGED
@@ -1,9 +1,11 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'typesensual/rake_helper'
|
4
|
+
|
3
5
|
namespace :typesensual do
|
4
6
|
desc 'List typesensual indices and their collections'
|
5
7
|
task list: :environment do
|
6
|
-
Typesensual::RakeHelper.
|
8
|
+
Typesensual::RakeHelper.list
|
7
9
|
end
|
8
10
|
|
9
11
|
desc 'Update the alias for an index'
|
@@ -21,4 +23,12 @@ namespace :typesensual do
|
|
21
23
|
model: args[:model]
|
22
24
|
)
|
23
25
|
end
|
26
|
+
|
27
|
+
desc 'Delete a version of an index'
|
28
|
+
task :drop_version, %i[index version] => :environment do |_, args|
|
29
|
+
Typesensual::RakeHelper.drop_version(
|
30
|
+
index: args[:index],
|
31
|
+
version: args[:version]
|
32
|
+
)
|
33
|
+
end
|
24
34
|
end
|
@@ -2,9 +2,8 @@
|
|
2
2
|
|
3
3
|
class Typesensual
|
4
4
|
class Callbacks
|
5
|
-
def initialize(index
|
5
|
+
def initialize(index)
|
6
6
|
@index = index
|
7
|
-
@should_update = should_update
|
8
7
|
end
|
9
8
|
|
10
9
|
def after_create_commit(record)
|
@@ -12,7 +11,7 @@ class Typesensual
|
|
12
11
|
end
|
13
12
|
|
14
13
|
def after_update_commit(record)
|
15
|
-
@
|
14
|
+
@index.index_one(record.id)
|
16
15
|
end
|
17
16
|
|
18
17
|
def after_destroy_commit(record)
|
@@ -181,9 +181,22 @@ class Typesensual
|
|
181
181
|
typesense_collection.documents[id.to_s].delete
|
182
182
|
end
|
183
183
|
|
184
|
+
# Remove multiple documents from typesense based on a filter
|
185
|
+
#
|
186
|
+
# @param filter_by [String] the filter to use to remove documents
|
187
|
+
# @return [void]
|
188
|
+
def remove_many!(filter_by:)
|
189
|
+
typesense_collection.documents.delete(filter_by: filter_by)
|
190
|
+
end
|
191
|
+
|
192
|
+
# Search for documents in typesense
|
193
|
+
#
|
194
|
+
# @param query [String] the query to search for
|
195
|
+
# @param query_by [String] the fields to search by
|
196
|
+
# @return [Search] the search object
|
184
197
|
def search(query:, query_by:)
|
185
198
|
Search.new(
|
186
|
-
collection:
|
199
|
+
collection: self,
|
187
200
|
query: query,
|
188
201
|
query_by: query_by
|
189
202
|
)
|
data/lib/typesensual/config.rb
CHANGED
@@ -2,21 +2,31 @@
|
|
2
2
|
|
3
3
|
class Typesensual
|
4
4
|
class Config
|
5
|
-
|
6
|
-
attr_writer :env, :client
|
5
|
+
attr_writer :env, :client, :nodes, :api_key
|
7
6
|
|
8
7
|
def initialize(&block)
|
9
8
|
yield self if block
|
10
9
|
end
|
11
10
|
|
12
11
|
def env
|
13
|
-
@env ||= (defined?(Rails) ? Rails.env : nil)
|
12
|
+
@env ||= ENV.fetch('TYPESENSUAL_ENV', (defined?(Rails) ? Rails.env : nil))
|
14
13
|
end
|
15
14
|
|
16
15
|
def client
|
17
16
|
@client ||= Typesense::Client.new(connection_options)
|
18
17
|
end
|
19
18
|
|
19
|
+
def nodes
|
20
|
+
@nodes ||= ENV['TYPESENSUAL_NODES']&.split(',')&.map do |node|
|
21
|
+
node_uri = URI.parse(node)
|
22
|
+
{ port: node_uri.port, host: node_uri.host, protocol: node_uri.scheme }
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def api_key
|
27
|
+
@api_key ||= ENV.fetch('TYPESENSUAL_API_KEY', nil)
|
28
|
+
end
|
29
|
+
|
20
30
|
private
|
21
31
|
|
22
32
|
def connection_options
|
data/lib/typesensual/field.rb
CHANGED
data/lib/typesensual/index.rb
CHANGED
@@ -21,6 +21,16 @@ class Typesensual
|
|
21
21
|
class Index
|
22
22
|
include StateHelpers
|
23
23
|
|
24
|
+
class << self
|
25
|
+
delegate :search, to: :collection
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.inherited(subclass)
|
29
|
+
super
|
30
|
+
# Copy the schema from the parent class to the subclass
|
31
|
+
subclass.instance_variable_set(:@schema, @schema&.dup)
|
32
|
+
end
|
33
|
+
|
24
34
|
# Get or set the name for this index
|
25
35
|
#
|
26
36
|
# @overload index_name(value)
|
@@ -94,7 +104,9 @@ class Typesensual
|
|
94
104
|
#
|
95
105
|
# See {Schema} for more information
|
96
106
|
def self.schema(&block)
|
97
|
-
@schema
|
107
|
+
@schema ||= Typesensual::Schema.new
|
108
|
+
@schema.instance_eval(&block) if block
|
109
|
+
@schema
|
98
110
|
end
|
99
111
|
|
100
112
|
# Updates the alias to point to the given collection name
|
@@ -122,28 +134,25 @@ class Typesensual
|
|
122
134
|
update_alias!(collection)
|
123
135
|
end
|
124
136
|
|
125
|
-
# The method to implement to index *one* record.
|
126
|
-
#
|
127
|
-
# @return [Hash] the document to upsert in Typesense
|
128
|
-
def index_one(_id); end
|
129
|
-
|
130
137
|
def self.index_one(id, collection: self.collection)
|
131
|
-
|
138
|
+
new.index([id]) do |record|
|
139
|
+
collection.insert_one!(record)
|
140
|
+
end
|
132
141
|
end
|
133
142
|
|
134
143
|
# The method to implement to index *many* records
|
135
|
-
#
|
144
|
+
# This method should yield successive records to index
|
136
145
|
#
|
137
146
|
# @yield [Hash] a document to upsert in Typesense
|
138
|
-
def
|
147
|
+
def index(ids)
|
139
148
|
ids.each do |id|
|
140
|
-
yield
|
149
|
+
yield({ id: id })
|
141
150
|
end
|
142
151
|
end
|
143
152
|
|
144
153
|
def self.index_many(ids, collection: self.collection, batch_size: 100)
|
145
154
|
collection.insert_many!(
|
146
|
-
new.enum_for(:
|
155
|
+
new.enum_for(:index, ids),
|
147
156
|
batch_size: batch_size
|
148
157
|
)
|
149
158
|
end
|
@@ -98,6 +98,25 @@ class Typesensual
|
|
98
98
|
created_at: new_coll.created_at.strftime('%Y-%m-%d %H:%M:%S')
|
99
99
|
)
|
100
100
|
end
|
101
|
+
|
102
|
+
# Drop a version of an index
|
103
|
+
#
|
104
|
+
# @param index [String] The name of the index to remove a version from
|
105
|
+
# @param version [String] The version to remove
|
106
|
+
# @example
|
107
|
+
# rake typesensual:drop_version[FooIndex,1]
|
108
|
+
def drop_version(index:, version:, output: $stdout)
|
109
|
+
index = index.safe_constantize
|
110
|
+
collection = index.collection_for(version: version)
|
111
|
+
|
112
|
+
collection.delete!
|
113
|
+
|
114
|
+
output.printf(
|
115
|
+
"==> Dropped version %<version>s of %<index>s\n",
|
116
|
+
version: version,
|
117
|
+
index: index.name
|
118
|
+
)
|
119
|
+
end
|
101
120
|
end
|
102
121
|
end
|
103
122
|
end
|
data/lib/typesensual/schema.rb
CHANGED
@@ -4,8 +4,21 @@ require 'typesensual/field'
|
|
4
4
|
|
5
5
|
class Typesensual
|
6
6
|
class Schema
|
7
|
+
# Duplicate fields from the original
|
8
|
+
def initialize_copy(original)
|
9
|
+
%w[
|
10
|
+
@fields
|
11
|
+
@token_separators
|
12
|
+
@symbols_to_index
|
13
|
+
@default_sorting_field
|
14
|
+
@enable_nested_fields
|
15
|
+
].each do |var|
|
16
|
+
instance_variable_set(var, original.instance_variable_get(var).dup)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
7
20
|
def initialize(&block)
|
8
|
-
instance_eval(&block)
|
21
|
+
instance_eval(&block) unless block.nil?
|
9
22
|
end
|
10
23
|
|
11
24
|
def field(name, type: 'auto', locale: nil, facet: nil, index: nil, optional: nil)
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class Typesensual
|
4
|
+
class Search
|
5
|
+
class GroupedHit
|
6
|
+
# {
|
7
|
+
# "group_key": [
|
8
|
+
# "420",
|
9
|
+
# "69"
|
10
|
+
# ],
|
11
|
+
# "hits": {
|
12
|
+
# ...
|
13
|
+
# },
|
14
|
+
# "found": 3
|
15
|
+
# }
|
16
|
+
# @param group [Hash] the Typesense hit hash
|
17
|
+
# * `group_key` [Array<any>] the grouping keys
|
18
|
+
# * `hits` [Hash] the Hits for the group
|
19
|
+
# * `found` [Integer] the number of hits in the group
|
20
|
+
def initialize(group)
|
21
|
+
@group = group
|
22
|
+
end
|
23
|
+
|
24
|
+
def count
|
25
|
+
@group['found']
|
26
|
+
end
|
27
|
+
|
28
|
+
def hits
|
29
|
+
@group['hits'].map { |hit| Hit.new(hit) }
|
30
|
+
end
|
31
|
+
|
32
|
+
def group_key
|
33
|
+
@group['group_key']
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -19,7 +19,7 @@ class Typesensual
|
|
19
19
|
# },
|
20
20
|
# "text_match": 130916
|
21
21
|
# }
|
22
|
-
# @param
|
22
|
+
# @param hit [Hash] the Typesense hit hash
|
23
23
|
# * `highlights` [Array<Hash>] the highlights for the hit
|
24
24
|
# * `document` [Hash] the matching document
|
25
25
|
# * `text_match` [Integer] the text matching score
|
data/lib/typesensual/search.rb
CHANGED
@@ -1,10 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'typesensual/search/hit'
|
4
|
+
require 'typesensual/search/grouped_hit'
|
4
5
|
require 'typesensual/search/results'
|
5
6
|
|
6
7
|
class Typesensual
|
7
8
|
class Search
|
9
|
+
include StateHelpers
|
10
|
+
|
8
11
|
# Initialize a new search object for a collection
|
9
12
|
#
|
10
13
|
# @param collection [Typesensual::Collection] the Typesensual collection object
|
@@ -20,6 +23,7 @@ class Typesensual
|
|
20
23
|
@facet_query = []
|
21
24
|
@include_fields = []
|
22
25
|
@exclude_fields = []
|
26
|
+
@group_by = []
|
23
27
|
@params = {}
|
24
28
|
|
25
29
|
@collection = collection
|
@@ -106,6 +110,11 @@ class Typesensual
|
|
106
110
|
self
|
107
111
|
end
|
108
112
|
|
113
|
+
def group_by(*fields)
|
114
|
+
@group_by += fields.map(&:to_s)
|
115
|
+
self
|
116
|
+
end
|
117
|
+
|
109
118
|
# Set additional parameters to pass to the search
|
110
119
|
# @param values [Hash] the parameters to set
|
111
120
|
def set(values)
|
@@ -114,6 +123,7 @@ class Typesensual
|
|
114
123
|
end
|
115
124
|
|
116
125
|
# Generate the query document
|
126
|
+
# @return [Hash] the query document
|
117
127
|
def query
|
118
128
|
{
|
119
129
|
collection: @collection.name,
|
@@ -125,13 +135,59 @@ class Typesensual
|
|
125
135
|
facet_by: @facet_by&.join(','),
|
126
136
|
facet_query: @facet_query&.join(','),
|
127
137
|
include_fields: @include_fields&.join(','),
|
128
|
-
exclude_fields: @exclude_fields&.join(',')
|
138
|
+
exclude_fields: @exclude_fields&.join(','),
|
139
|
+
group_by: @group_by&.join(',')
|
129
140
|
}.merge(@params).reject { |_, v| v.blank? }
|
130
141
|
end
|
131
142
|
|
132
143
|
# Load the results from the search query
|
144
|
+
# @return [Typesensual::Search::Results] the results of the search
|
133
145
|
def load
|
134
146
|
Results.new(@collection.typesense_collection.documents.search(query))
|
135
147
|
end
|
148
|
+
|
149
|
+
# Perform multiple searches in one request. There are two variants of this method, one which
|
150
|
+
# takes a list of anonymous queries and one which takes a hash of named queries. Named queries
|
151
|
+
# will probably be more readable for more than a couple of queries, but anonymous queries can be
|
152
|
+
# destructured directly.
|
153
|
+
#
|
154
|
+
# Both versions accept either a Search instance or a hash of search parameters.
|
155
|
+
#
|
156
|
+
# @overload multi(*searches)
|
157
|
+
# Perform an array of search queries in a single request. The return values are guaranteed to
|
158
|
+
# be in the same order as the provided searches.
|
159
|
+
#
|
160
|
+
# @param searches [<Typesensual::Search, Hash>] the searches to perform
|
161
|
+
# @return [<Typesensual::Search::Results>] the results of the searches
|
162
|
+
#
|
163
|
+
# @overload multi(searches)
|
164
|
+
# Perform multiple named search queries in a single request. The results will be keyed by the
|
165
|
+
# same names as the provided searches.
|
166
|
+
#
|
167
|
+
# @param searches [{Object => Typesensual::Search, Hash>] the searches to perform
|
168
|
+
# @return [{Object => Typesensual::Search::Results}] the results of the searches
|
169
|
+
def self.multi(*searches)
|
170
|
+
# If we have one argument and it's a hash, we're doing named searches
|
171
|
+
if searches.count == 1 && searches.first.is_a?(Hash)
|
172
|
+
keys = searches.first.keys
|
173
|
+
searches = searches.first.values
|
174
|
+
end
|
175
|
+
|
176
|
+
results = client.multi_search.perform({
|
177
|
+
searches: searches.flatten.map(&:query)
|
178
|
+
})
|
179
|
+
|
180
|
+
# Wrap our results in Result objects
|
181
|
+
wrapped_results = results['results'].map do |result|
|
182
|
+
Results.new(result)
|
183
|
+
end
|
184
|
+
|
185
|
+
# If we're doing named searches, re-key the results
|
186
|
+
if keys
|
187
|
+
keys.zip(wrapped_results).to_h
|
188
|
+
else
|
189
|
+
wrapped_results
|
190
|
+
end
|
191
|
+
end
|
136
192
|
end
|
137
193
|
end
|
data/lib/typesensual/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: typesensual
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Emma Lejeck
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-08-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -80,6 +80,7 @@ files:
|
|
80
80
|
- lib/typesensual/rake_helper.rb
|
81
81
|
- lib/typesensual/schema.rb
|
82
82
|
- lib/typesensual/search.rb
|
83
|
+
- lib/typesensual/search/grouped_hit.rb
|
83
84
|
- lib/typesensual/search/hit.rb
|
84
85
|
- lib/typesensual/search/results.rb
|
85
86
|
- lib/typesensual/state_helpers.rb
|