k9o 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +80 -1
- data/README.md +1516 -159
- data/k9o.gemspec +2 -2
- data/lib/k9o/ai.rb +15 -0
- data/lib/k9o/analytics.rb +403 -0
- data/lib/k9o/approvals.rb +253 -0
- data/lib/k9o/audit.rb +150 -0
- data/lib/k9o/client.rb +25 -4
- data/lib/k9o/crypto.rb +471 -0
- data/lib/k9o/custody.rb +28 -0
- data/lib/k9o/data.rb +125 -5
- data/lib/k9o/ed25519.rb +182 -0
- data/lib/k9o/end_users.rb +93 -0
- data/lib/k9o/envelope.rb +20 -5
- data/lib/k9o/functions.rb +82 -3
- data/lib/k9o/gate.rb +46 -7
- data/lib/k9o/http_transport.rb +71 -14
- data/lib/k9o/jcs.rb +117 -0
- data/lib/k9o/kerror.rb +57 -5
- data/lib/k9o/keys.rb +131 -0
- data/lib/k9o/kibo_client_core.rb +385 -0
- data/lib/k9o/kibo_ffi_core.rb +178 -0
- data/lib/k9o/locate.rb +118 -0
- data/lib/k9o/me.rb +1 -1
- data/lib/k9o/metadata.rb +342 -0
- data/lib/k9o/moat.rb +443 -0
- data/lib/k9o/org.rb +1 -1
- data/lib/k9o/query.rb +174 -0
- data/lib/k9o/rampart.rb +273 -0
- data/lib/k9o/realtime.rb +261 -10
- data/lib/k9o/schedules.rb +109 -0
- data/lib/k9o/schema.rb +35 -0
- data/lib/k9o/streams.rb +94 -0
- data/lib/k9o/vault.rb +3 -4
- data/lib/k9o/vectors.rb +124 -0
- data/lib/k9o/version.rb +1 -1
- data/lib/k9o.rb +17 -0
- data/sig/k9o/ai.rbs +4 -0
- data/sig/k9o/approvals.rbs +71 -0
- data/sig/k9o/datasets.rbs +125 -0
- data/sig/k9o/gate.rbs +2 -0
- data/sig/k9o/kerror.rbs +24 -0
- data/sig/k9o/kibo_client_core.rbs +25 -0
- data/sig/k9o/metadata.rbs +54 -0
- data/sig/k9o/moat.rbs +29 -0
- data/sig/k9o/query.rbs +26 -0
- data/sig/k9o/rampart.rbs +59 -0
- data/sig/k9o/realtime.rbs +34 -0
- data/sig/k9o/workload.rbs +64 -0
- metadata +33 -4
data/lib/k9o/moat.rb
ADDED
|
@@ -0,0 +1,443 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'digest'
|
|
5
|
+
|
|
6
|
+
module K9o
|
|
7
|
+
# Moat: a local guard around one model call. It runs in your process; it makes no network calls, needs no account
|
|
8
|
+
# and uses no other part of this SDK.
|
|
9
|
+
#
|
|
10
|
+
# moat = K9o::Moat.new(
|
|
11
|
+
# prompt: "Summarise the ticket in one sentence.",
|
|
12
|
+
# input_schema: { "type" => "string", "maxLength" => 4000 },
|
|
13
|
+
# output_schema: { "type" => "object", "required" => ["summary"], "properties" => { "summary" => { "type" => "string" } } },
|
|
14
|
+
# filters: { pii: ->(text) { barbican.inspect(text).decision == "deny" } }
|
|
15
|
+
# )
|
|
16
|
+
# messages = moat.messages(ticket_text) # [{ role: "system", ... }, { role: "user", ... }] or raises Refusal
|
|
17
|
+
# value = moat.check_output(model_text) # the parsed, validated value or raises Refusal
|
|
18
|
+
#
|
|
19
|
+
# It does four things:
|
|
20
|
+
#
|
|
21
|
+
# 1. Pins the system prompt. Moat builds the message list itself: the prompt is always the +system+ message and the
|
|
22
|
+
# untrusted input is always a +user+ message. Input that carries a role or system marker is refused. The pin is
|
|
23
|
+
# structural: it stops injected text from becoming the system prompt. It does not stop that text reaching the
|
|
24
|
+
# model, so keep +output_schema+ and the checks on what your code does with the answer.
|
|
25
|
+
# 2. Checks the input against +input_schema+ before it goes into a message.
|
|
26
|
+
# 3. Checks the model output against +output_schema+.
|
|
27
|
+
# 4. Runs the optional +pii+ filter over the output.
|
|
28
|
+
#
|
|
29
|
+
# A refusal is an exception: Moat::Refusal, with a +reason+ from the closed set REASONS and a short +detail+ that
|
|
30
|
+
# never repeats the input or the output.
|
|
31
|
+
module Moat
|
|
32
|
+
REASONS = %i[role_marker input_schema output_schema pii].freeze
|
|
33
|
+
|
|
34
|
+
# Raised when Moat refuses input or output. +reason+ is one of REASONS.
|
|
35
|
+
class Refusal < StandardError
|
|
36
|
+
attr_reader :reason, :detail
|
|
37
|
+
|
|
38
|
+
def initialize(reason, detail)
|
|
39
|
+
@reason = reason
|
|
40
|
+
@detail = detail
|
|
41
|
+
super("#{reason}: #{detail}")
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Raised at construction for a bad prompt, filter or schema.
|
|
46
|
+
class ConfigError < ArgumentError; end
|
|
47
|
+
|
|
48
|
+
# Raised at construction when a schema uses a keyword outside the supported subset.
|
|
49
|
+
class SchemaError < ConfigError; end
|
|
50
|
+
|
|
51
|
+
# Build a Guard. See Guard#initialize.
|
|
52
|
+
def self.new(**options)
|
|
53
|
+
Guard.new(**options)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# The JSON Schema subset Moat understands, and nothing else.
|
|
57
|
+
module Schema
|
|
58
|
+
TYPES = %w[object array string number integer boolean null].freeze
|
|
59
|
+
KEYWORDS = %w[type properties required additionalProperties items enum const minLength maxLength pattern
|
|
60
|
+
minimum maximum minItems maxItems].freeze
|
|
61
|
+
|
|
62
|
+
module_function
|
|
63
|
+
|
|
64
|
+
# Raise SchemaError unless +schema+ uses only the supported keywords with well-formed values.
|
|
65
|
+
def check_schema!(schema, path = '$')
|
|
66
|
+
raise SchemaError, "#{path}: a schema must be a Hash" unless schema.is_a?(Hash)
|
|
67
|
+
|
|
68
|
+
schema.each_key do |key|
|
|
69
|
+
unless KEYWORDS.include?(key.to_s)
|
|
70
|
+
raise SchemaError, "#{path}: unsupported keyword #{key.to_s.inspect}; supported: #{KEYWORDS.join(', ')}"
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
s = stringify(schema)
|
|
74
|
+
check_type!(s['type'], path) if s.key?('type')
|
|
75
|
+
check_properties!(s['properties'], path) if s.key?('properties')
|
|
76
|
+
check_required!(s['required'], path) if s.key?('required')
|
|
77
|
+
if s.key?('additionalProperties') && ![true, false].include?(s['additionalProperties'])
|
|
78
|
+
raise SchemaError, "#{path}.additionalProperties must be true or false"
|
|
79
|
+
end
|
|
80
|
+
check_schema!(s['items'], "#{path}.items") if s.key?('items')
|
|
81
|
+
raise SchemaError, "#{path}.enum must be a non-empty Array" if s.key?('enum') && !(s['enum'].is_a?(Array) && !s['enum'].empty?)
|
|
82
|
+
|
|
83
|
+
%w[minLength maxLength minItems maxItems].each do |kw|
|
|
84
|
+
next unless s.key?(kw)
|
|
85
|
+
|
|
86
|
+
raise SchemaError, "#{path}.#{kw} must be a whole number >= 0" unless s[kw].is_a?(Integer) && s[kw] >= 0
|
|
87
|
+
end
|
|
88
|
+
%w[minimum maximum].each do |kw|
|
|
89
|
+
next unless s.key?(kw)
|
|
90
|
+
|
|
91
|
+
raise SchemaError, "#{path}.#{kw} must be a number" unless s[kw].is_a?(Numeric) && s[kw].to_f.finite?
|
|
92
|
+
end
|
|
93
|
+
check_pattern!(s['pattern'], path) if s.key?('pattern')
|
|
94
|
+
true
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def stringify(schema)
|
|
98
|
+
schema.transform_keys(&:to_s)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def check_type!(type, path)
|
|
102
|
+
list = type.is_a?(Array) ? type : [type]
|
|
103
|
+
ok = !list.empty? && list.all? { |t| t.is_a?(String) && TYPES.include?(t) }
|
|
104
|
+
raise SchemaError, "#{path}.type must be one of #{TYPES.join(', ')} (or a list of them)" unless ok
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def check_properties!(props, path)
|
|
108
|
+
raise SchemaError, "#{path}.properties must be a Hash of schemas" unless props.is_a?(Hash)
|
|
109
|
+
|
|
110
|
+
props.each { |name, sub| check_schema!(sub, "#{path}.properties.#{name}") }
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def check_required!(required, path)
|
|
114
|
+
ok = required.is_a?(Array) && required.all? { |r| r.is_a?(String) || r.is_a?(Symbol) }
|
|
115
|
+
raise SchemaError, "#{path}.required must be an Array of names" unless ok
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def check_pattern!(pattern, path)
|
|
119
|
+
raise SchemaError, "#{path}.pattern must be a String" unless pattern.is_a?(String)
|
|
120
|
+
|
|
121
|
+
Regexp.new(pattern)
|
|
122
|
+
rescue RegexpError => e
|
|
123
|
+
raise SchemaError, "#{path}.pattern is not a valid regular expression: #{e.message}"
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# nil when +value+ (JSON data) satisfies +schema+, else a short description of the first failure.
|
|
127
|
+
def violation(schema, value, path = '$')
|
|
128
|
+
s = stringify(schema)
|
|
129
|
+
if s.key?('type') && !type_ok?(s['type'], value)
|
|
130
|
+
return "#{path}: expected #{Array(s['type']).join(' or ')}"
|
|
131
|
+
end
|
|
132
|
+
return "#{path}: not one of the allowed values" if s.key?('enum') && s['enum'].none? { |e| same?(e, value) }
|
|
133
|
+
return "#{path}: not the required constant" if s.key?('const') && !same?(s['const'], value)
|
|
134
|
+
|
|
135
|
+
case value
|
|
136
|
+
when String then string_violation(s, value, path)
|
|
137
|
+
when Numeric then number_violation(s, value, path)
|
|
138
|
+
when Array then array_violation(s, value, path)
|
|
139
|
+
when Hash then object_violation(s, value, path)
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def type_ok?(type, value)
|
|
144
|
+
Array(type).any? do |t|
|
|
145
|
+
case t
|
|
146
|
+
when 'object' then value.is_a?(Hash)
|
|
147
|
+
when 'array' then value.is_a?(Array)
|
|
148
|
+
when 'string' then value.is_a?(String)
|
|
149
|
+
when 'number' then value.is_a?(Numeric)
|
|
150
|
+
when 'integer' then value.is_a?(Integer) || (value.is_a?(Float) && value.finite? && value == value.floor)
|
|
151
|
+
when 'boolean' then value == true || value == false
|
|
152
|
+
when 'null' then value.nil?
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def same?(left, right)
|
|
158
|
+
return false unless left.class == right.class || (left.is_a?(Numeric) && right.is_a?(Numeric))
|
|
159
|
+
|
|
160
|
+
case left
|
|
161
|
+
when Hash
|
|
162
|
+
left = left.transform_keys(&:to_s)
|
|
163
|
+
left.size == right.size && left.all? { |k, v| right.key?(k) && same?(v, right[k]) }
|
|
164
|
+
when Array
|
|
165
|
+
left.size == right.size && left.each_with_index.all? { |v, i| same?(v, right[i]) }
|
|
166
|
+
else
|
|
167
|
+
left == right
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def string_violation(s, value, path)
|
|
172
|
+
return "#{path}: shorter than minLength" if s['minLength'] && value.length < s['minLength']
|
|
173
|
+
return "#{path}: longer than maxLength" if s['maxLength'] && value.length > s['maxLength']
|
|
174
|
+
return "#{path}: does not match pattern" if s['pattern'] && !Regexp.new(s['pattern']).match?(value)
|
|
175
|
+
|
|
176
|
+
nil
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def number_violation(s, value, path)
|
|
180
|
+
return "#{path}: below minimum" if s['minimum'] && value < s['minimum']
|
|
181
|
+
return "#{path}: above maximum" if s['maximum'] && value > s['maximum']
|
|
182
|
+
|
|
183
|
+
nil
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def array_violation(s, value, path)
|
|
187
|
+
return "#{path}: fewer than minItems" if s['minItems'] && value.size < s['minItems']
|
|
188
|
+
return "#{path}: more than maxItems" if s['maxItems'] && value.size > s['maxItems']
|
|
189
|
+
return nil unless s['items']
|
|
190
|
+
|
|
191
|
+
value.each_with_index do |item, index|
|
|
192
|
+
found = violation(s['items'], item, "#{path}[#{index}]")
|
|
193
|
+
return found if found
|
|
194
|
+
end
|
|
195
|
+
nil
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def object_violation(s, value, path)
|
|
199
|
+
props = (s['properties'] || {}).transform_keys(&:to_s)
|
|
200
|
+
Array(s['required']).map(&:to_s).each do |name|
|
|
201
|
+
return "#{path}.#{name}: required" unless value.key?(name)
|
|
202
|
+
end
|
|
203
|
+
value.each do |name, item|
|
|
204
|
+
if props.key?(name)
|
|
205
|
+
found = violation(props[name], item, "#{path}.#{name}")
|
|
206
|
+
return found if found
|
|
207
|
+
elsif s['additionalProperties'] == false
|
|
208
|
+
return "#{path}.#{name}: not allowed"
|
|
209
|
+
end
|
|
210
|
+
end
|
|
211
|
+
nil
|
|
212
|
+
end
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# Detects role and system markers in untrusted input.
|
|
216
|
+
module Markers
|
|
217
|
+
# Default-ignorable and format code points, removed before matching (one shared table, code point ranges).
|
|
218
|
+
IGNORABLE_RANGES = [
|
|
219
|
+
0x00AD..0x00AD, 0x034F..0x034F, 0x061C..0x061C, 0x115F..0x1160, 0x17B4..0x17B5, 0x180B..0x180F,
|
|
220
|
+
0x200B..0x200F, 0x202A..0x202E, 0x2060..0x206F, 0x3164..0x3164, 0xFE00..0xFE0F, 0xFEFF..0xFEFF,
|
|
221
|
+
0xFFA0..0xFFA0, 0xFFF0..0xFFF8, 0x1BCA0..0x1BCA3, 0x1D173..0x1D17A, 0xE0000..0xE0FFF
|
|
222
|
+
].freeze
|
|
223
|
+
IGNORABLE = Regexp.new(
|
|
224
|
+
'[' + IGNORABLE_RANGES.map { |r| format('\\u{%X}-\\u{%X}', r.first, r.last) }.join + ']'
|
|
225
|
+
).freeze
|
|
226
|
+
# Cyrillic and Greek look-alikes of the marker letters -> Latin letter (code point escapes; mirrors
|
|
227
|
+
# test/fixtures/moat-confusables.json, which a test compares against). Not general homoglyph detection.
|
|
228
|
+
CONFUSABLES = {
|
|
229
|
+
"\u{0430}" => 'a',
|
|
230
|
+
"\u{0410}" => 'a',
|
|
231
|
+
"\u{0441}" => 'c',
|
|
232
|
+
"\u{0421}" => 'c',
|
|
233
|
+
"\u{0435}" => 'e',
|
|
234
|
+
"\u{0415}" => 'e',
|
|
235
|
+
"\u{0456}" => 'i',
|
|
236
|
+
"\u{0406}" => 'i',
|
|
237
|
+
"\u{043E}" => 'o',
|
|
238
|
+
"\u{041E}" => 'o',
|
|
239
|
+
"\u{0440}" => 'p',
|
|
240
|
+
"\u{0420}" => 'p',
|
|
241
|
+
"\u{0455}" => 's',
|
|
242
|
+
"\u{0405}" => 's',
|
|
243
|
+
"\u{0443}" => 'y',
|
|
244
|
+
"\u{0423}" => 'y',
|
|
245
|
+
"\u{0501}" => 'd',
|
|
246
|
+
"\u{04CF}" => 'l',
|
|
247
|
+
"\u{04C0}" => 'l',
|
|
248
|
+
"\u{0422}" => 't',
|
|
249
|
+
"\u{041C}" => 'm',
|
|
250
|
+
"\u{041D}" => 'h',
|
|
251
|
+
"\u{0445}" => 'x',
|
|
252
|
+
"\u{0425}" => 'x',
|
|
253
|
+
"\u{0391}" => 'a',
|
|
254
|
+
"\u{03B1}" => 'a',
|
|
255
|
+
"\u{0395}" => 'e',
|
|
256
|
+
"\u{03B5}" => 'e',
|
|
257
|
+
"\u{0399}" => 'i',
|
|
258
|
+
"\u{03B9}" => 'i',
|
|
259
|
+
"\u{039F}" => 'o',
|
|
260
|
+
"\u{03BF}" => 'o',
|
|
261
|
+
"\u{03A1}" => 'p',
|
|
262
|
+
"\u{03C1}" => 'p',
|
|
263
|
+
"\u{03A4}" => 't',
|
|
264
|
+
"\u{03C4}" => 't',
|
|
265
|
+
"\u{03A5}" => 'y',
|
|
266
|
+
"\u{03C5}" => 'u',
|
|
267
|
+
"\u{03BD}" => 'v',
|
|
268
|
+
"\u{039C}" => 'm',
|
|
269
|
+
"\u{039D}" => 'n',
|
|
270
|
+
"\u{03A7}" => 'x',
|
|
271
|
+
"\u{03C7}" => 'x',
|
|
272
|
+
"\u{03F2}" => 'c',
|
|
273
|
+
"\u{03F9}" => 'c',
|
|
274
|
+
"\u{0397}" => 'h',
|
|
275
|
+
"\u{0442}" => 't'
|
|
276
|
+
}.freeze
|
|
277
|
+
CONFUSABLES_RE = Regexp.new("[" + CONFUSABLES.keys.join + "]").freeze
|
|
278
|
+
MARKS = /[\p{Mn}\p{Me}]/.freeze
|
|
279
|
+
HORIZONTAL_SPACE = /[\t\p{Zs}]+/.freeze
|
|
280
|
+
LINE_ROLE = /^[ \t]*(system|assistant|developer|tool|user)[ \t]*:/.freeze
|
|
281
|
+
STRINGS = ['<|system|>', '<|im_start|>', '<|im_end|>', '[inst]', '<<sys>>'].freeze
|
|
282
|
+
HEADINGS = /###[ \t]*(system|instruction)/.freeze
|
|
283
|
+
JSON_ROLE = /"role"\s*:\s*"\s*(system|developer|assistant)\s*"/.freeze
|
|
284
|
+
ROLE_VALUES = %w[system developer assistant].freeze
|
|
285
|
+
|
|
286
|
+
module_function
|
|
287
|
+
|
|
288
|
+
# The name of the first marker found anywhere in +value+ (JSON data), or nil.
|
|
289
|
+
def find(value)
|
|
290
|
+
case value
|
|
291
|
+
when String then in_text(value)
|
|
292
|
+
when Array then value.lazy.map { |item| find(item) }.find { |found| found }
|
|
293
|
+
when Hash then in_hash(value)
|
|
294
|
+
end
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
def in_hash(hash)
|
|
298
|
+
hash.each do |key, item|
|
|
299
|
+
if normalise(key.to_s) == 'role' && item.is_a?(String) && ROLE_VALUES.include?(normalise(item).strip)
|
|
300
|
+
return 'a role object'
|
|
301
|
+
end
|
|
302
|
+
found = find(key.to_s) || find(item)
|
|
303
|
+
return found if found
|
|
304
|
+
end
|
|
305
|
+
nil
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
def in_text(text)
|
|
309
|
+
t = normalise(text)
|
|
310
|
+
return 'a role line' if t.match?(LINE_ROLE)
|
|
311
|
+
return 'a chat-template marker' if STRINGS.any? { |m| t.include?(m) }
|
|
312
|
+
return 'a heading marker' if t.match?(HEADINGS)
|
|
313
|
+
return 'a role object' if t.match?(JSON_ROLE)
|
|
314
|
+
|
|
315
|
+
nil
|
|
316
|
+
end
|
|
317
|
+
|
|
318
|
+
def normalise(text)
|
|
319
|
+
nfkc = text.to_s.dup.force_encoding(Encoding::UTF_8).scrub('').unicode_normalize(:nfkc)
|
|
320
|
+
nfkc.unicode_normalize(:nfkd).gsub(MARKS, '').gsub(IGNORABLE, '')
|
|
321
|
+
.gsub(CONFUSABLES_RE, CONFUSABLES).downcase(:fold).gsub(HORIZONTAL_SPACE, ' ')
|
|
322
|
+
end
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
class Guard
|
|
326
|
+
# +prompt+: the pinned system prompt (a non-empty String).
|
|
327
|
+
# +input_schema+ / +output_schema+: optional schemas in the subset (see Schema); anything else raises SchemaError.
|
|
328
|
+
# +filters+: optional { pii: callable }. The callable gets the output as text and returns a truthy value when it
|
|
329
|
+
# finds personal data; its return value is not repeated in the refusal. An error inside it also refuses.
|
|
330
|
+
def initialize(prompt:, input_schema: nil, output_schema: nil, filters: nil)
|
|
331
|
+
raise ConfigError, 'prompt must be a non-empty String' unless prompt.is_a?(String) && !prompt.strip.empty?
|
|
332
|
+
|
|
333
|
+
Schema.check_schema!(input_schema, 'input_schema') unless input_schema.nil?
|
|
334
|
+
Schema.check_schema!(output_schema, 'output_schema') unless output_schema.nil?
|
|
335
|
+
@prompt = prompt.dup.freeze
|
|
336
|
+
@input_schema = input_schema
|
|
337
|
+
@output_schema = output_schema
|
|
338
|
+
@pii = pii_filter(filters)
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
# The message list for one model call: the pinned prompt as +system+, the untrusted input as +user+ text.
|
|
342
|
+
# Raises Refusal (:role_marker or :input_schema).
|
|
343
|
+
def messages(untrusted_input)
|
|
344
|
+
data = json_data(untrusted_input, :input_schema, 'input is not JSON data')
|
|
345
|
+
marker = Markers.find(data)
|
|
346
|
+
raise Refusal.new(:role_marker, "the input contains #{marker}") if marker
|
|
347
|
+
|
|
348
|
+
check_schema(@input_schema, data, :input_schema)
|
|
349
|
+
text = data.is_a?(String) ? data : JSON.generate(data)
|
|
350
|
+
[{ role: 'system', content: @prompt }, { role: 'user', content: text }].freeze
|
|
351
|
+
end
|
|
352
|
+
|
|
353
|
+
# The validated, filtered model output. A String is parsed as JSON unless the output schema's type is "string".
|
|
354
|
+
# Raises Refusal (:output_schema or :pii).
|
|
355
|
+
def check_output(model_output)
|
|
356
|
+
value = parse_output(model_output)
|
|
357
|
+
check_schema(@output_schema, value, :output_schema)
|
|
358
|
+
run_pii(value)
|
|
359
|
+
value
|
|
360
|
+
end
|
|
361
|
+
|
|
362
|
+
# Lowercase hex SHA-256 of the pinned prompt, so you can pass it to Rampart or record it.
|
|
363
|
+
def prompt_sha256
|
|
364
|
+
Digest::SHA256.hexdigest(@prompt.b)
|
|
365
|
+
end
|
|
366
|
+
|
|
367
|
+
private
|
|
368
|
+
|
|
369
|
+
def pii_filter(filters)
|
|
370
|
+
return nil if filters.nil?
|
|
371
|
+
raise ConfigError, 'filters must be a Hash' unless filters.is_a?(Hash)
|
|
372
|
+
|
|
373
|
+
named = filters.transform_keys(&:to_s)
|
|
374
|
+
unknown = named.keys - ['pii']
|
|
375
|
+
raise ConfigError, "unknown filter #{unknown.first.inspect}; supported: pii" unless unknown.empty?
|
|
376
|
+
|
|
377
|
+
callable = named['pii']
|
|
378
|
+
return nil if callable.nil?
|
|
379
|
+
raise ConfigError, 'filters[:pii] must respond to call' unless callable.respond_to?(:call)
|
|
380
|
+
|
|
381
|
+
callable
|
|
382
|
+
end
|
|
383
|
+
|
|
384
|
+
# +value+ as plain JSON data (string keys), or a Refusal when it holds anything JSON cannot.
|
|
385
|
+
def json_data(value, reason, detail)
|
|
386
|
+
plain(value)
|
|
387
|
+
rescue TypeError, EncodingError
|
|
388
|
+
raise Refusal.new(reason, detail)
|
|
389
|
+
end
|
|
390
|
+
|
|
391
|
+
def plain(value)
|
|
392
|
+
case value
|
|
393
|
+
when nil, true, false, String then value
|
|
394
|
+
when Symbol then value.to_s
|
|
395
|
+
when Integer then value
|
|
396
|
+
when Float
|
|
397
|
+
raise TypeError, 'not finite' unless value.finite?
|
|
398
|
+
|
|
399
|
+
value
|
|
400
|
+
when Array then value.map { |item| plain(item) }
|
|
401
|
+
when Hash
|
|
402
|
+
value.each_with_object({}) do |(key, item), out|
|
|
403
|
+
raise TypeError, 'keys must be strings' unless key.is_a?(String) || key.is_a?(Symbol)
|
|
404
|
+
|
|
405
|
+
out[key.to_s] = plain(item)
|
|
406
|
+
end
|
|
407
|
+
else
|
|
408
|
+
raise TypeError, 'not JSON data'
|
|
409
|
+
end
|
|
410
|
+
end
|
|
411
|
+
|
|
412
|
+
def parse_output(model_output)
|
|
413
|
+
string_schema = @output_schema && Schema.stringify(@output_schema)['type'] == 'string'
|
|
414
|
+
return model_output unless model_output.is_a?(String) && @output_schema && !string_schema
|
|
415
|
+
|
|
416
|
+
JSON.parse(model_output)
|
|
417
|
+
rescue JSON::ParserError
|
|
418
|
+
raise Refusal.new(:output_schema, 'the output is not valid JSON')
|
|
419
|
+
end
|
|
420
|
+
|
|
421
|
+
def check_schema(schema, value, reason)
|
|
422
|
+
return if schema.nil?
|
|
423
|
+
|
|
424
|
+
found = Schema.violation(schema, value)
|
|
425
|
+
raise Refusal.new(reason, found) if found
|
|
426
|
+
end
|
|
427
|
+
|
|
428
|
+
def run_pii(value)
|
|
429
|
+
return if @pii.nil?
|
|
430
|
+
|
|
431
|
+
text = value.is_a?(String) ? value : JSON.generate(value)
|
|
432
|
+
flagged = begin
|
|
433
|
+
@pii.call(text)
|
|
434
|
+
rescue StandardError => e
|
|
435
|
+
raise Refusal.new(:pii, "the PII filter failed (#{e.class}), so the output is withheld")
|
|
436
|
+
end
|
|
437
|
+
return if flagged.nil? || flagged == false || (flagged.respond_to?(:empty?) && flagged.empty?)
|
|
438
|
+
|
|
439
|
+
raise Refusal.new(:pii, 'the PII filter flagged the output')
|
|
440
|
+
end
|
|
441
|
+
end
|
|
442
|
+
end
|
|
443
|
+
end
|
data/lib/k9o/org.rb
CHANGED
|
@@ -143,7 +143,7 @@ module K9o
|
|
|
143
143
|
# POST /identity-providers { idpProvider, displayName, clientId, clientSecretRef, tierGating, ... }
|
|
144
144
|
# idp_provider: google github microsoft oidc saml scim ldap active_directory apple facebook twitter
|
|
145
145
|
# tier_gating: 'free_tier_ok' or 'ridge_plus'
|
|
146
|
-
# client_secret_ref:
|
|
146
|
+
# client_secret_ref: the OAuth client secret itself (the name is kept for compatibility); the gateway encrypts it and never returns it
|
|
147
147
|
def create(idp_provider:, display_name:, client_id:, client_secret_ref:, tier_gating:, **opts)
|
|
148
148
|
body = { 'idpProvider' => idp_provider, 'displayName' => display_name, 'clientId' => client_id,
|
|
149
149
|
'clientSecretRef' => client_secret_ref, 'tierGating' => tier_gating }
|
data/lib/k9o/query.rb
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module K9o
|
|
4
|
+
# `client.query(sql, params)` and `client.search(collection, column:, q:)`.
|
|
5
|
+
#
|
|
6
|
+
# Wire (project-plane):
|
|
7
|
+
# POST /@<slug>/query body { sql:, params? } -> { data: [row, ...], meta }
|
|
8
|
+
# GET /@<slug>/data/<collection>?textSearch=<column>:<q>... (the select builder's +text_search+)
|
|
9
|
+
#
|
|
10
|
+
# +params+ is an Array (positional, bound to $1, $2, ...; the SQL is sent as written) or a Hash of named values
|
|
11
|
+
# (+:name+ placeholders are rewritten to $n here and the values become the positional list). Values are bound by
|
|
12
|
+
# the database, never interpolated into the SQL text.
|
|
13
|
+
module Query
|
|
14
|
+
IDENT_START = /[A-Za-z_]/.freeze
|
|
15
|
+
IDENT_CHAR = /[A-Za-z0-9_]/.freeze
|
|
16
|
+
DOLLAR_TAG = /\A\$([A-Za-z_][A-Za-z0-9_]*)?\$/.freeze
|
|
17
|
+
|
|
18
|
+
module_function
|
|
19
|
+
|
|
20
|
+
def invalid(message, context = {})
|
|
21
|
+
KError.new(code: 'K9O_INVALID_FIELD_VALUE', message: message, context: context)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def check_value(value, where)
|
|
25
|
+
return value if value.nil? || value == true || value == false || value.is_a?(String)
|
|
26
|
+
return value if value.is_a?(Integer)
|
|
27
|
+
return value if value.is_a?(Float) && value.finite?
|
|
28
|
+
|
|
29
|
+
raise invalid("#{where} must be a string, number, boolean or nil; serialise anything else (for example as JSON text) first")
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def lookup(named, name)
|
|
33
|
+
return [true, named[name]] if named.key?(name)
|
|
34
|
+
return [true, named[name.to_sym]] if named.key?(name.to_sym)
|
|
35
|
+
|
|
36
|
+
[false, nil]
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Rewrite +:name+ placeholders to $n. Skips quoted text, comments, dollar-quoted strings and +::+ casts.
|
|
40
|
+
# Returns [sql, params].
|
|
41
|
+
def translate_named_params(sql, named)
|
|
42
|
+
order = []
|
|
43
|
+
out = +''
|
|
44
|
+
i = 0
|
|
45
|
+
n = sql.length
|
|
46
|
+
while i < n
|
|
47
|
+
c = sql[i]
|
|
48
|
+
nxt = i + 1 < n ? sql[i + 1] : ''
|
|
49
|
+
if c == "'" || ((c == 'E' || c == 'e') && nxt == "'" && !(i > 0 && sql[i - 1] =~ IDENT_CHAR))
|
|
50
|
+
escapes = c != "'"
|
|
51
|
+
j = escapes ? i + 2 : i + 1
|
|
52
|
+
while j < n
|
|
53
|
+
if escapes && sql[j] == '\\'
|
|
54
|
+
j += 2
|
|
55
|
+
next
|
|
56
|
+
end
|
|
57
|
+
if sql[j] == "'"
|
|
58
|
+
if sql[j + 1] == "'"
|
|
59
|
+
j += 2
|
|
60
|
+
next
|
|
61
|
+
end
|
|
62
|
+
break
|
|
63
|
+
end
|
|
64
|
+
j += 1
|
|
65
|
+
end
|
|
66
|
+
out << sql[i...[j + 1, n].min]
|
|
67
|
+
i = j + 1
|
|
68
|
+
elsif c == '"'
|
|
69
|
+
j = i + 1
|
|
70
|
+
while j < n
|
|
71
|
+
if sql[j] == '"'
|
|
72
|
+
if sql[j + 1] == '"'
|
|
73
|
+
j += 2
|
|
74
|
+
next
|
|
75
|
+
end
|
|
76
|
+
break
|
|
77
|
+
end
|
|
78
|
+
j += 1
|
|
79
|
+
end
|
|
80
|
+
out << sql[i...[j + 1, n].min]
|
|
81
|
+
i = j + 1
|
|
82
|
+
elsif c == '-' && nxt == '-'
|
|
83
|
+
j = sql.index("\n", i) || n
|
|
84
|
+
out << sql[i...j]
|
|
85
|
+
i = j
|
|
86
|
+
elsif c == '/' && nxt == '*'
|
|
87
|
+
depth = 1
|
|
88
|
+
j = i + 2
|
|
89
|
+
while j < n && depth > 0
|
|
90
|
+
if sql[j, 2] == '/*'
|
|
91
|
+
depth += 1
|
|
92
|
+
j += 2
|
|
93
|
+
elsif sql[j, 2] == '*/'
|
|
94
|
+
depth -= 1
|
|
95
|
+
j += 2
|
|
96
|
+
else
|
|
97
|
+
j += 1
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
out << sql[i...j]
|
|
101
|
+
i = j
|
|
102
|
+
elsif c == '$'
|
|
103
|
+
m = DOLLAR_TAG.match(sql[i..-1])
|
|
104
|
+
if m
|
|
105
|
+
close = sql.index(m[0], i + m[0].length)
|
|
106
|
+
stop = close.nil? ? n : close + m[0].length
|
|
107
|
+
out << sql[i...stop]
|
|
108
|
+
i = stop
|
|
109
|
+
else
|
|
110
|
+
out << c
|
|
111
|
+
i += 1
|
|
112
|
+
end
|
|
113
|
+
elsif c == ':' && nxt == ':'
|
|
114
|
+
out << '::'
|
|
115
|
+
i += 2
|
|
116
|
+
elsif c == ':' && nxt != '' && nxt =~ IDENT_START && !(i > 0 && sql[i - 1] =~ IDENT_CHAR)
|
|
117
|
+
j = i + 2
|
|
118
|
+
j += 1 while j < n && sql[j] =~ IDENT_CHAR
|
|
119
|
+
name = sql[(i + 1)...j]
|
|
120
|
+
unless order.include?(name)
|
|
121
|
+
found, = lookup(named, name)
|
|
122
|
+
raise invalid("no value for :#{name}", name: name) unless found
|
|
123
|
+
|
|
124
|
+
order << name
|
|
125
|
+
end
|
|
126
|
+
out << "$#{order.index(name) + 1}"
|
|
127
|
+
i = j
|
|
128
|
+
else
|
|
129
|
+
out << c
|
|
130
|
+
i += 1
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
values = order.map { |name| check_value(lookup(named, name)[1], "the value for :#{name}") }
|
|
134
|
+
[out, values]
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Builds [path, body] for POST /@<slug>/query.
|
|
138
|
+
def prepare(config, sql, params)
|
|
139
|
+
slug = Studio.require_project_slug(config, 'query')
|
|
140
|
+
text = sql
|
|
141
|
+
values = []
|
|
142
|
+
if params.is_a?(Hash)
|
|
143
|
+
text, values = translate_named_params(sql, params)
|
|
144
|
+
elsif !params.nil?
|
|
145
|
+
values = params.each_with_index.map { |v, i| check_value(v, "params[#{i}]") }
|
|
146
|
+
end
|
|
147
|
+
body = { 'sql' => text }
|
|
148
|
+
body['params'] = values unless values.empty?
|
|
149
|
+
[UrlGrammar.project_path(slug, 'query'), body]
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# Client-root methods mixed into K9oClient.
|
|
154
|
+
module QueryMethods
|
|
155
|
+
# POST /@<slug>/query — run one read-only SQL statement; +data+ is the array of rows.
|
|
156
|
+
# An Array binds positionally to $1, $2, ...; a Hash binds +:name+ placeholders.
|
|
157
|
+
def query(sql, params = nil)
|
|
158
|
+
path, body = Query.prepare(@config, sql, params)
|
|
159
|
+
HttpTransport.request(config: @config, method: 'POST', path: path, body: body)
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# Full-text search over one text column of a collection, best matches first. Sugar over
|
|
163
|
+
# +from(collection).select.text_search(column, q, ...)+; +limit+ and +cursor+ page the result.
|
|
164
|
+
def search(collection, column:, q:, language: nil, type: nil, limit: nil, cursor: nil)
|
|
165
|
+
opts = {}
|
|
166
|
+
opts[:language] = language unless language.nil?
|
|
167
|
+
opts[:type] = type unless type.nil?
|
|
168
|
+
builder = from(collection).select.text_search(column, q, **opts)
|
|
169
|
+
builder.limit(limit) unless limit.nil?
|
|
170
|
+
builder.cursor(cursor) unless cursor.nil?
|
|
171
|
+
builder.execute
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
end
|