dvla-kaping 1.0.5 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.ruby-version +1 -1
- data/.tool-versions +1 -1
- data/CHANGELOG.md +9 -0
- data/README.md +56 -0
- data/lib/dvla/kaping/aws_client.rb +10 -2
- data/lib/dvla/kaping/config.rb +37 -6
- data/lib/dvla/kaping/query.rb +1 -4
- data/lib/dvla/kaping/query_term.rb +55 -2
- data/lib/dvla/kaping/search.rb +17 -4
- data/lib/dvla/kaping/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8c7792c198c21559e5be1660b1803961030f063030b7493025a64ff2e1ee1902
|
|
4
|
+
data.tar.gz: a722412fd145adc5fabb0e8c675d2a52c6eb522c27e5f74f589640d3c8662cb5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e34d02e54b495ee3f8502b28f5196fe2d6267e370c2ace1c8f23bb4df82dc770671020c3a56c43996db86adcd492aab6f4a5b706ec5c91824d6a267f0e3f8760
|
|
7
|
+
data.tar.gz: bc73e599125b348df8ef49e524678d9cd0156e00012fa9e1d0695c7c419bf85e2f3bb174f2aa5dbf72bae29566ac11f6a1cde4d42e2fb000d681da9c2a3ea386
|
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
4.0.
|
|
1
|
+
4.0.9
|
data/.tool-versions
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
ruby 4.0.
|
|
1
|
+
ruby 4.0.9
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
## [Unreleased]
|
|
2
2
|
|
|
3
|
+
## [1.0.6] - 2026-11-04
|
|
4
|
+
|
|
5
|
+
- Added `nested` query support to the query DSL for fields mapped as the OpenSearch `nested` type
|
|
6
|
+
- Fixed config loading on Ruby 3.4+/4.0 (`require 'erb'`), guarded a missing `kaping.yml`, corrected `KAPING_*` environment overrides, and fixed the `logger=`/`log_level` methods
|
|
7
|
+
- Applied the configured log level to the default logger and made log-level precedence explicit (`KAPING_LOG_LEVEL` env > YAML > `INFO`)
|
|
8
|
+
- Clarified AWS credential errors: unrecognised `credential_type` and unresolved credentials now raise actionable messages
|
|
9
|
+
- Memorised the connected search client so credentials are resolved once per process, with optional client injection and `reset_client!`
|
|
10
|
+
- Simplified the `exists` query term to take a single field name (legacy two-argument form still accepted); removed unused constants
|
|
11
|
+
|
|
3
12
|
## [1.0.5] - 2025-11-10
|
|
4
13
|
|
|
5
14
|
- Fixed AWS credential handling to return proper credentials from assume role
|
data/README.md
CHANGED
|
@@ -181,6 +181,62 @@ structure of the query from the ruby objects.
|
|
|
181
181
|
my_json_query = my_query.to_json
|
|
182
182
|
```
|
|
183
183
|
|
|
184
|
+
## Checking a field exists
|
|
185
|
+
|
|
186
|
+
Use `exists` with the field name to require (or, under `must_not`, exclude)
|
|
187
|
+
documents where a field is present:
|
|
188
|
+
|
|
189
|
+
```ruby
|
|
190
|
+
q = DVLA::Kaping::Query.new('bool')
|
|
191
|
+
q.must_not.exists('drivingLicence.migrationErrors')
|
|
192
|
+
|
|
193
|
+
# q.to_json will produce
|
|
194
|
+
'{"query":{"bool":{"must_not":[{"exists":{"field":"drivingLicence.migrationErrors"}}]}}}'
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
## Nested queries
|
|
198
|
+
|
|
199
|
+
For fields mapped as the OpenSearch [`nested`](https://docs.opensearch.org/latest/query-dsl/joining/nested/)
|
|
200
|
+
type, a flat `match_phrase` will not match. Use `nested`, passing the path and a
|
|
201
|
+
block that builds the inner conditions.
|
|
202
|
+
|
|
203
|
+
```ruby
|
|
204
|
+
q = DVLA::Kaping::Query.new('bool')
|
|
205
|
+
q.must.nested('entitlement') { |n| n.match_phrase('entitlement.categoryStatus', 'Valid') }
|
|
206
|
+
|
|
207
|
+
# q.to_json will produce
|
|
208
|
+
'{"query":{"bool":{"must":[{"nested":{"path":"entitlement","query":{"match_phrase":{"entitlement.categoryStatus":"Valid"}}}}]}}}'
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
Multiple conditions inside a single `nested` block must all match the **same**
|
|
212
|
+
nested object (they are combined into a `bool`/`must`):
|
|
213
|
+
|
|
214
|
+
```ruby
|
|
215
|
+
q.must.nested('entitlement') do |n|
|
|
216
|
+
n.match_phrase('entitlement.categoryCode', 'C').
|
|
217
|
+
match_phrase('entitlement.categoryStatus', 'Valid')
|
|
218
|
+
end
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
To require (or exclude) several independent nested objects, chain separate
|
|
222
|
+
`nested` clauses. For example, to exclude records that have any of several
|
|
223
|
+
category codes:
|
|
224
|
+
|
|
225
|
+
```ruby
|
|
226
|
+
q.must_not.
|
|
227
|
+
nested('entitlement') { |n| n.match_phrase('entitlement.categoryCode', 'C') }.
|
|
228
|
+
nested('entitlement') { |n| n.match_phrase('entitlement.categoryCode', 'D') }
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
The optional OpenSearch parameters `score_mode`, `ignore_unmapped` and
|
|
232
|
+
`inner_hits` may be passed as keyword arguments:
|
|
233
|
+
|
|
234
|
+
```ruby
|
|
235
|
+
q.must.nested('entitlement', score_mode: 'max', ignore_unmapped: true) do |n|
|
|
236
|
+
n.match_phrase('entitlement.categoryStatus', 'Valid')
|
|
237
|
+
end
|
|
238
|
+
```
|
|
239
|
+
|
|
184
240
|
## Code of Conduct
|
|
185
241
|
|
|
186
242
|
Everyone interacting in the Dvla::Kaping project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](./CODE_OF_CONDUCT.md).
|
|
@@ -16,7 +16,8 @@ module DVLA
|
|
|
16
16
|
end
|
|
17
17
|
|
|
18
18
|
def select_credentials
|
|
19
|
-
|
|
19
|
+
credential_type = Kaping.yaml.dig(:aws, :credential_type)
|
|
20
|
+
case credential_type
|
|
20
21
|
when 'profile'
|
|
21
22
|
assume_role_profile(@aws_account_id, @role)
|
|
22
23
|
when 'env'
|
|
@@ -24,13 +25,20 @@ module DVLA
|
|
|
24
25
|
when 'credentials'
|
|
25
26
|
Aws::CredentialProviderChain.new.resolve
|
|
26
27
|
else
|
|
27
|
-
|
|
28
|
+
raise ArgumentError,
|
|
29
|
+
"Unrecognised aws.credential_type: #{credential_type.inspect}. " \
|
|
30
|
+
'Set one of: profile, env or credentials'
|
|
28
31
|
end
|
|
29
32
|
end
|
|
30
33
|
|
|
31
34
|
def connect
|
|
32
35
|
credentials = select_credentials
|
|
33
36
|
|
|
37
|
+
if credentials.nil?
|
|
38
|
+
raise 'Unable to resolve AWS credentials. Check your credential_type and that a ' \
|
|
39
|
+
'profile/SSO session or environment credentials are available.'
|
|
40
|
+
end
|
|
41
|
+
|
|
34
42
|
signer = Aws::Sigv4::Signer.new(service: 'es',
|
|
35
43
|
region: @region,
|
|
36
44
|
credentials_provider: credentials)
|
data/lib/dvla/kaping/config.rb
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'erb'
|
|
3
4
|
require 'yaml'
|
|
5
|
+
require 'logger'
|
|
4
6
|
require 'active_support/core_ext/hash/deep_merge'
|
|
5
7
|
|
|
6
8
|
|
|
@@ -13,20 +15,39 @@ module DVLA
|
|
|
13
15
|
ATTRIBUTES = %w[host index result_size log_level].freeze
|
|
14
16
|
attr_accessor(*ATTRIBUTES)
|
|
15
17
|
|
|
18
|
+
# Maps the configured log level (symbol/string) to a Logger severity.
|
|
19
|
+
LOG_LEVELS = {
|
|
20
|
+
'debug' => Logger::DEBUG,
|
|
21
|
+
'info' => Logger::INFO,
|
|
22
|
+
'warn' => Logger::WARN,
|
|
23
|
+
'error' => Logger::ERROR,
|
|
24
|
+
'fatal' => Logger::FATAL,
|
|
25
|
+
'unknown' => Logger::UNKNOWN,
|
|
26
|
+
}.freeze
|
|
27
|
+
|
|
16
28
|
def initialize
|
|
17
29
|
@yaml = load_yaml(Dir["#{Dir.pwd}/**/kaping.yml"].first)
|
|
18
30
|
|
|
31
|
+
# Allow KAPING_* environment variables to override the loaded values,
|
|
32
|
+
# but only when the variable is actually set.
|
|
19
33
|
ATTRIBUTES.each do |attr|
|
|
20
|
-
|
|
34
|
+
env_value = ENV.fetch("KAPING_#{attr.upcase}", nil)
|
|
35
|
+
instance_variable_set(:"@#{attr}", env_value) unless env_value.nil?
|
|
21
36
|
end
|
|
22
37
|
|
|
23
|
-
@logger = Logger.new($stdout)
|
|
24
|
-
@log_level ||= 'INFO'
|
|
25
38
|
@result_size ||= 100
|
|
39
|
+
|
|
40
|
+
@logger = Logger.new($stdout)
|
|
41
|
+
@logger.level = resolved_log_severity
|
|
26
42
|
end
|
|
27
43
|
|
|
28
44
|
# step 1 - find kaping.yml
|
|
29
45
|
def load_yaml(path)
|
|
46
|
+
if path.nil?
|
|
47
|
+
warn('[WARN] No kaping.yml found')
|
|
48
|
+
return nil
|
|
49
|
+
end
|
|
50
|
+
|
|
30
51
|
path = "#{path}.yml" unless %w[.yaml .yml].include?(File.extname(path))
|
|
31
52
|
|
|
32
53
|
if File.exist?(path)
|
|
@@ -54,14 +75,24 @@ module DVLA
|
|
|
54
75
|
def logger=(new_logger)
|
|
55
76
|
if new_logger.is_a?(Logger)
|
|
56
77
|
@logger = new_logger
|
|
57
|
-
@logger.level =
|
|
78
|
+
@logger.level = resolved_log_severity
|
|
58
79
|
else
|
|
59
80
|
warn("[WARN] Custom logger is not an instance of Logger: '#{new_logger.class}'")
|
|
60
81
|
end
|
|
61
82
|
end
|
|
62
83
|
|
|
63
|
-
|
|
64
|
-
|
|
84
|
+
# The configured log level. Precedence: KAPING_LOG_LEVEL env var (captured
|
|
85
|
+
# into @log_level during initialize) > the loaded YAML > 'INFO'.
|
|
86
|
+
def log_level
|
|
87
|
+
@log_level || (@yaml && @yaml[:kaping_log_level]) || 'INFO'
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
private
|
|
91
|
+
|
|
92
|
+
# The configured log level mapped to a Logger severity, defaulting to INFO
|
|
93
|
+
# for anything not recognised.
|
|
94
|
+
def resolved_log_severity
|
|
95
|
+
LOG_LEVELS.fetch(log_level.to_s.downcase, Logger::INFO)
|
|
65
96
|
end
|
|
66
97
|
end
|
|
67
98
|
end
|
data/lib/dvla/kaping/query.rb
CHANGED
|
@@ -5,11 +5,8 @@ module DVLA
|
|
|
5
5
|
class Query
|
|
6
6
|
include QueryTerm
|
|
7
7
|
|
|
8
|
-
BOOL = 'bool'.freeze
|
|
9
|
-
MATCH = 'match'.freeze
|
|
10
|
-
|
|
11
8
|
attr_reader :query_type, :kwargs
|
|
12
|
-
protected attr_reader :operations
|
|
9
|
+
protected attr_reader :operations
|
|
13
10
|
protected attr_accessor :last_operation
|
|
14
11
|
|
|
15
12
|
def initialize(type, **kwargs)
|
|
@@ -23,9 +23,17 @@ module DVLA
|
|
|
23
23
|
self
|
|
24
24
|
end
|
|
25
25
|
|
|
26
|
-
|
|
26
|
+
# Checks that a field exists. Per the OpenSearch `exists` query the shape
|
|
27
|
+
# is { "exists": { "field": "<name>" } }.
|
|
28
|
+
#
|
|
29
|
+
# q.must_not.exists('drivingLicence.migrationErrors')
|
|
30
|
+
#
|
|
31
|
+
# The legacy two-argument form exists('field', '<name>') is still accepted
|
|
32
|
+
# for backward compatibility and reproduces its original output.
|
|
33
|
+
def exists(field, value = nil)
|
|
34
|
+
clause = value.nil? ? { field: field } : { "#{field}": value }
|
|
27
35
|
current_operation << {
|
|
28
|
-
exists:
|
|
36
|
+
exists: clause,
|
|
29
37
|
}
|
|
30
38
|
self
|
|
31
39
|
end
|
|
@@ -75,6 +83,51 @@ module DVLA
|
|
|
75
83
|
self
|
|
76
84
|
end
|
|
77
85
|
|
|
86
|
+
# Wrap one or more conditions in a nested query, for fields mapped as the
|
|
87
|
+
# OpenSearch `nested` type. See
|
|
88
|
+
# https://docs.opensearch.org/latest/query-dsl/joining/nested/
|
|
89
|
+
#
|
|
90
|
+
# The block yields an inner builder. Conditions added via `must` (the
|
|
91
|
+
# default context) are combined so they must all match the SAME nested
|
|
92
|
+
# object. The inner builder also supports `should`/`must_not`/`filter`
|
|
93
|
+
# for more complex nested conditions (e.g. "any of these codes").
|
|
94
|
+
#
|
|
95
|
+
# q.must.nested('entitlement') { |n| n.match_phrase('entitlement.categoryStatus', 'Valid') }
|
|
96
|
+
#
|
|
97
|
+
# q.must.nested('entitlement') do |n|
|
|
98
|
+
# n.match_phrase('entitlement.categoryStatus', 'Valid')
|
|
99
|
+
# n.should.
|
|
100
|
+
# match_phrase('entitlement.categoryCode', 'C').
|
|
101
|
+
# match_phrase('entitlement.categoryCode', 'D')
|
|
102
|
+
# end
|
|
103
|
+
#
|
|
104
|
+
# Optional OpenSearch parameters (score_mode, ignore_unmapped, inner_hits)
|
|
105
|
+
# may be passed as keyword arguments and are included only when provided.
|
|
106
|
+
def nested(path, score_mode: nil, ignore_unmapped: nil, inner_hits: nil)
|
|
107
|
+
raise ArgumentError, 'nested requires a block to build the inner query' unless block_given?
|
|
108
|
+
|
|
109
|
+
inner = DVLA::Kaping::Query.new('bool')
|
|
110
|
+
yield inner.must
|
|
111
|
+
inner_ops = inner.send(:operations).reject { |_op, clauses| clauses.respond_to?(:empty?) && clauses.empty? }
|
|
112
|
+
|
|
113
|
+
raise ArgumentError, 'nested block produced no conditions' if inner_ops.empty?
|
|
114
|
+
|
|
115
|
+
inner_query =
|
|
116
|
+
if inner_ops.keys == [:must] && inner_ops[:must].size == 1
|
|
117
|
+
inner_ops[:must].first
|
|
118
|
+
else
|
|
119
|
+
{ bool: inner_ops }
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
clause = { path: path, query: inner_query }
|
|
123
|
+
clause[:score_mode] = score_mode unless score_mode.nil?
|
|
124
|
+
clause[:ignore_unmapped] = ignore_unmapped unless ignore_unmapped.nil?
|
|
125
|
+
clause[:inner_hits] = inner_hits unless inner_hits.nil?
|
|
126
|
+
|
|
127
|
+
current_operation << { nested: clause }
|
|
128
|
+
self
|
|
129
|
+
end
|
|
130
|
+
|
|
78
131
|
def current_operation
|
|
79
132
|
operations[last_operation]
|
|
80
133
|
end
|
data/lib/dvla/kaping/search.rb
CHANGED
|
@@ -1,16 +1,29 @@
|
|
|
1
1
|
module DVLA
|
|
2
2
|
module Kaping
|
|
3
3
|
module Search
|
|
4
|
-
#
|
|
4
|
+
# Connected OpenSearch client, built once and reused across searches.
|
|
5
|
+
# Building a client resolves credentials and constructs a SigV4 signer,
|
|
6
|
+
# so doing it per query is wasteful. The credentials provider resolved
|
|
7
|
+
# for the `credentials` type is self-refreshing, so a reused client stays
|
|
8
|
+
# valid as the underlying session refreshes.
|
|
5
9
|
def search(body)
|
|
6
|
-
|
|
7
|
-
con = client.connect
|
|
8
|
-
con.search(
|
|
10
|
+
search_client.search(
|
|
9
11
|
index: Kaping.yaml[:kaping_index],
|
|
10
12
|
body: body,
|
|
11
13
|
size: Kaping.yaml[:kaping_result_size],
|
|
12
14
|
)
|
|
13
15
|
end
|
|
16
|
+
|
|
17
|
+
# The memoised connected client. Pass an explicit client to use your own,
|
|
18
|
+
# or call reset_client! to force a rebuild (e.g. after config changes).
|
|
19
|
+
def search_client(client = nil)
|
|
20
|
+
@search_client = client if client
|
|
21
|
+
@search_client ||= DVLA::Kaping::AWSClient.new.connect
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def reset_client!
|
|
25
|
+
@search_client = nil
|
|
26
|
+
end
|
|
14
27
|
end
|
|
15
28
|
end
|
|
16
29
|
end
|
data/lib/dvla/kaping/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: dvla-kaping
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.0.
|
|
4
|
+
version: 1.0.6
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Driver and Vehicle Licensing Agency (DVLA)
|
|
@@ -113,7 +113,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
113
113
|
- !ruby/object:Gem::Version
|
|
114
114
|
version: '0'
|
|
115
115
|
requirements: []
|
|
116
|
-
rubygems_version: 4.0.
|
|
116
|
+
rubygems_version: 4.0.20
|
|
117
117
|
specification_version: 4
|
|
118
118
|
summary: Idiomatic way to create DSL openSearch definitions
|
|
119
119
|
test_files: []
|