squishling 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +123 -0
- data/README.md +117 -35
- data/docs/configuration.md +113 -32
- data/docs/failures.md +111 -14
- data/docs/harnesses.md +198 -0
- data/docs/measuring-tokens.md +148 -0
- data/docs/naming.md +43 -0
- data/docs/routing.md +46 -31
- data/docs/schemas.md +32 -2
- data/lib/squishling/appendices.rb +3 -3
- data/lib/squishling/class_methods.rb +81 -31
- data/lib/squishling/collisions.rb +54 -0
- data/lib/squishling/configuration.rb +58 -9
- data/lib/squishling/definition.rb +78 -39
- data/lib/squishling/errors.rb +29 -5
- data/lib/squishling/harness.rb +153 -0
- data/lib/squishling/invoker.rb +223 -74
- data/lib/squishling/judge.rb +142 -0
- data/lib/squishling/llm_client.rb +150 -0
- data/lib/squishling/model_path.rb +129 -0
- data/lib/squishling/output_check.rb +109 -0
- data/lib/squishling/params.rb +39 -2
- data/lib/squishling/router.rb +13 -7
- data/lib/squishling/schema.rb +45 -3
- data/lib/squishling/source.rb +5 -5
- data/lib/squishling/squawk.rb +52 -0
- data/lib/squishling/version.rb +1 -1
- data/lib/squishling.rb +25 -5
- metadata +13 -3
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Squishling
|
|
4
|
+
# The models a squished call tries, in order. Declared either as one model (model: "claude-haiku-4-5", a
|
|
5
|
+
# single attempt) or as an escalation: an Array of steps, each a model name or a Hash:
|
|
6
|
+
# escalation: [
|
|
7
|
+
# { model: "claude-haiku-4-5", attempts: 2 },
|
|
8
|
+
# "claude-sonnet-5-5",
|
|
9
|
+
# { model: "claude-opus-5-5", params: { thinking: { effort: :high } } }
|
|
10
|
+
# ]
|
|
11
|
+
# attempts: (default 1) retries a step; order: (all steps or none, unique, lowest first) makes the order
|
|
12
|
+
# explicit instead of positional. forward_rejected: false starts a step from the original input alone, without
|
|
13
|
+
# the previous step's rejected output.
|
|
14
|
+
module ModelPath
|
|
15
|
+
# One attempt: the model, its provider, the fully resolved generation params, and whether the previous
|
|
16
|
+
# step's rejected output is shown to it.
|
|
17
|
+
Step = Data.define(:model, :provider, :params, :forward_rejected) do
|
|
18
|
+
# The model for logs and messages (nil means RubyLLM's default model).
|
|
19
|
+
def display_name
|
|
20
|
+
model || "RubyLLM default model"
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
STEP_KEYS = %i[model provider params attempts order forward_rejected].freeze
|
|
25
|
+
|
|
26
|
+
module_function
|
|
27
|
+
|
|
28
|
+
# A single model: model:/default_model. Returns a one-step path, or nil when not declared.
|
|
29
|
+
def from_model(model, label)
|
|
30
|
+
return nil if model.nil?
|
|
31
|
+
|
|
32
|
+
if model.is_a?(Array) || model.is_a?(Hash)
|
|
33
|
+
raise ConfigurationError, "#{label} takes one model; use escalation: [...] to try several in order"
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
[normalize_step(model, label)].freeze
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# An escalation: escalation:/default_escalation. Returns the steps in order, or nil when not declared.
|
|
40
|
+
def from_escalation(escalation, label)
|
|
41
|
+
return nil if escalation.nil?
|
|
42
|
+
unless escalation.is_a?(Array)
|
|
43
|
+
raise ConfigurationError, "#{label} must be an Array of steps, got #{escalation.class}"
|
|
44
|
+
end
|
|
45
|
+
raise ConfigurationError, "#{label} can't be empty" if escalation.empty?
|
|
46
|
+
|
|
47
|
+
ordered(escalation.map { |step| normalize_step(step, label) }, label).freeze
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# model: and escalation: both describe what to try, so a level may declare only one of them.
|
|
51
|
+
def declare(model, escalation, label)
|
|
52
|
+
if !model.nil? && !escalation.nil?
|
|
53
|
+
raise ConfigurationError, "#{label}: declare model: or escalation:, not both (put the model in the escalation)"
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
from_model(model, "#{label} model") || from_escalation(escalation, "#{label} escalation")
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# One Step per attempt. The level's provider applies to steps that don't name their own, and each
|
|
60
|
+
# step's params are merged over the level-resolved params.
|
|
61
|
+
def steps(path, provider:, params:)
|
|
62
|
+
path.flat_map do |step|
|
|
63
|
+
attempt = Step.new(model: step[:model], provider: step[:provider] || provider,
|
|
64
|
+
params: Params.resolve(params, step[:params]), forward_rejected: step[:forward_rejected])
|
|
65
|
+
[attempt] * step[:attempts]
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def ordered(steps, label)
|
|
70
|
+
numbered = steps.count { |step| step.key?(:order) }
|
|
71
|
+
return steps.map { |step| step.except(:order) } if numbered.zero?
|
|
72
|
+
|
|
73
|
+
if numbered < steps.size
|
|
74
|
+
raise ConfigurationError, "#{label}: give every step an order: or none (#{numbered} of #{steps.size} have one)"
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
orders = steps.map { |step| step[:order] }
|
|
78
|
+
duplicates = orders.tally.select { |_, count| count > 1 }.keys
|
|
79
|
+
raise ConfigurationError, "#{label}: duplicate order: #{duplicates.join(', ')}" if duplicates.any?
|
|
80
|
+
|
|
81
|
+
steps.sort_by { |step| step[:order] }.map { |step| step.except(:order) }
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def normalize_step(step, label)
|
|
85
|
+
case step
|
|
86
|
+
when String, Symbol
|
|
87
|
+
{ model: model_name(step, label), provider: nil, params: nil, attempts: 1, forward_rejected: true }
|
|
88
|
+
when Hash then normalize_hash(step, label)
|
|
89
|
+
else
|
|
90
|
+
raise ConfigurationError, "#{label}: each step must be a model name or a Hash with model:, got #{step.inspect}"
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def normalize_hash(step, label)
|
|
95
|
+
step = step.transform_keys(&:to_sym)
|
|
96
|
+
unknown = step.keys - STEP_KEYS
|
|
97
|
+
raise ConfigurationError, "#{label}: unknown step option(s) #{unknown.join(', ')}" if unknown.any?
|
|
98
|
+
unless step[:model].is_a?(String) || step[:model].is_a?(Symbol)
|
|
99
|
+
raise ConfigurationError, "#{label}: a step needs a model: name, got #{step.inspect}"
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
model = model_name(step[:model], label)
|
|
103
|
+
attempts = step.fetch(:attempts, 1)
|
|
104
|
+
unless attempts.is_a?(Integer) && attempts.positive?
|
|
105
|
+
raise ConfigurationError, "#{label} (#{model}): attempts: must be a positive Integer, got #{attempts.inspect}"
|
|
106
|
+
end
|
|
107
|
+
if step.key?(:order) && !step[:order].is_a?(Integer)
|
|
108
|
+
raise ConfigurationError, "#{label} (#{model}): order: must be an Integer, got #{step[:order].inspect}"
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
forward_rejected = step.fetch(:forward_rejected, true)
|
|
112
|
+
unless [true, false].include?(forward_rejected)
|
|
113
|
+
raise ConfigurationError,
|
|
114
|
+
"#{label} (#{model}): forward_rejected: must be true or false, got #{forward_rejected.inspect}"
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
params = step[:params] && Params.normalize(step[:params], "#{label} (#{model}) params")
|
|
118
|
+
normalized = { model:, provider: step[:provider], params:, attempts:, forward_rejected: }
|
|
119
|
+
step.key?(:order) ? normalized.merge(order: step[:order]) : normalized
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def model_name(model, label)
|
|
123
|
+
name = model.to_s
|
|
124
|
+
raise ConfigurationError, "#{label}: model names can't be blank" if name.strip.empty?
|
|
125
|
+
|
|
126
|
+
name
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
end
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Squishling
|
|
4
|
+
# Turns a model response into a typed result: parses it, validates it against the schema, and runs the
|
|
5
|
+
# method's squish_validate check.
|
|
6
|
+
class OutputCheck
|
|
7
|
+
MAX_ERRORS = 20
|
|
8
|
+
UNREPRESENTABLE = "response contained a value JSON can't represent (such as Infinity or invalid UTF-8)"
|
|
9
|
+
|
|
10
|
+
def initialize(definition, receiver, inputs)
|
|
11
|
+
@definition = definition
|
|
12
|
+
@receiver = receiver
|
|
13
|
+
@inputs = inputs
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Parses and validates a response: [typed result, []] when it passes, [nil or result, errors] otherwise.
|
|
17
|
+
# The judge's role skips squish_validate, which checks the operation's output, not a verdict.
|
|
18
|
+
def call(content, prompt)
|
|
19
|
+
data, errors = parse(content)
|
|
20
|
+
errors = prompt.schema.validate(data) if errors.empty?
|
|
21
|
+
return [nil, cap(errors)] if errors.any?
|
|
22
|
+
|
|
23
|
+
result = prompt.schema.build(data, squished: true)
|
|
24
|
+
[result, prompt.role == :judge ? [] : validator_errors(result)]
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
# RubyLLM parses structured output itself and leaves the raw string when that fails, so
|
|
30
|
+
# content is a Hash on success, or a String/nil when the model refused, was cut off, or
|
|
31
|
+
# ignored the schema.
|
|
32
|
+
def parse(content)
|
|
33
|
+
return [nil, [UNREPRESENTABLE]] if content.is_a?(String) && !content.valid_encoding?
|
|
34
|
+
return [nil, ["response was empty"]] if content.nil? || (content.is_a?(String) && content.strip.empty?)
|
|
35
|
+
|
|
36
|
+
data = content.is_a?(String) ? JSON.parse(strip_code_fence(content)) : content
|
|
37
|
+
representable?(data) ? [data, []] : [nil, [UNREPRESENTABLE]]
|
|
38
|
+
rescue JSON::ParserError => e
|
|
39
|
+
[nil, ["response was not valid JSON#{parse_position(e)}"]]
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# JSON.parse turns an out-of-range number such as 1e400 into Infinity, and accepts invalid UTF-8 in a string.
|
|
43
|
+
# Both can satisfy the schema but can't be serialized again (and the deterministic path rejects them), so
|
|
44
|
+
# they are invalid output.
|
|
45
|
+
def representable?(data)
|
|
46
|
+
JSON.generate(data)
|
|
47
|
+
true
|
|
48
|
+
rescue JSON::GeneratorError
|
|
49
|
+
false
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# A long output can fail the schema thousands of times over; every error goes into the retry message and
|
|
53
|
+
# the log, so only the first few are kept.
|
|
54
|
+
def cap(errors)
|
|
55
|
+
return errors if errors.size <= MAX_ERRORS
|
|
56
|
+
|
|
57
|
+
errors.first(MAX_ERRORS) + ["... and #{errors.size - MAX_ERRORS} more errors"]
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# The parser's message quotes a snippet of the response, which must not reach error messages or logs
|
|
61
|
+
# (the raw output lives only in InvalidOutputError#raw), so only the position is kept.
|
|
62
|
+
def parse_position(error)
|
|
63
|
+
position = error.message.strip.match(/ at line (\d+) column (\d+)\z/)
|
|
64
|
+
position ? " (at line #{position[1]} column #{position[2]})" : ""
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Models without native structured output sometimes wrap JSON in a markdown code fence.
|
|
68
|
+
def strip_code_fence(text)
|
|
69
|
+
text[/\A\s*```(?:json)?\s*\n(.*?)\n\s*```\s*\z/m, 1] || text
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Runs the squish_validate check, if any. Exceptions it raises are the user's own and propagate as-is.
|
|
73
|
+
def validator_errors(result)
|
|
74
|
+
validator = @definition.validator
|
|
75
|
+
return [] unless validator
|
|
76
|
+
|
|
77
|
+
value = @receiver.instance_exec(result, **@inputs, &validator)
|
|
78
|
+
case value
|
|
79
|
+
when nil, true then []
|
|
80
|
+
when false then ["the output was rejected by squish_validate"]
|
|
81
|
+
when String, Array then Array(value).flatten.compact.map(&:to_s).reject { |message| message.strip.empty? }
|
|
82
|
+
else
|
|
83
|
+
return contract_errors(value) if value.respond_to?(:success?) && value.respond_to?(:errors)
|
|
84
|
+
|
|
85
|
+
raise ConfigurationError, "#{@definition.label}: squish_validate must return nil, true, false, a String, " \
|
|
86
|
+
"an Array of Strings, or a validation result, got #{value.class}"
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# A dry-validation style result: errors.to_h is { key => ["message", ...] }, nested for nested keys.
|
|
91
|
+
def contract_errors(value)
|
|
92
|
+
return [] if value.success?
|
|
93
|
+
|
|
94
|
+
errors = value.errors
|
|
95
|
+
errors = errors.to_h if !errors.is_a?(Array) && errors.respond_to?(:to_h)
|
|
96
|
+
messages = errors.is_a?(Hash) ? flatten_messages(errors) : Array(errors).map(&:to_s)
|
|
97
|
+
messages.empty? ? ["the output was rejected by squish_validate"] : messages
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def flatten_messages(errors, prefix = nil)
|
|
101
|
+
errors.flat_map do |key, value|
|
|
102
|
+
path = [prefix, key].compact.join(".")
|
|
103
|
+
next flatten_messages(value, path) if value.is_a?(Hash)
|
|
104
|
+
|
|
105
|
+
Array(value).map { |message| path.empty? ? message.to_s : "#{path} #{message}" }
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
data/lib/squishling/params.rb
CHANGED
|
@@ -9,10 +9,27 @@ module Squishling
|
|
|
9
9
|
# overriding RubyLLM's defaults, so these would silently replace the model, the conversation,
|
|
10
10
|
# or the strict output format.
|
|
11
11
|
RESERVED_KEYS = %i[
|
|
12
|
-
model messages input instructions contents system system_instruction
|
|
13
|
-
response_format text output_config tools tool_choice
|
|
12
|
+
model messages input inputs instructions contents system system_instruction systemInstruction cachedContent
|
|
13
|
+
stream stream_options store include response_format text output_config outputConfig tools tool_choice
|
|
14
|
+
toolConfig tool_config schema
|
|
14
15
|
].freeze
|
|
15
16
|
|
|
17
|
+
# Some protocols carry the structured-output format (or tool selection) inside a container that is
|
|
18
|
+
# itself allowed, because it also holds ordinary settings (Gemini's topK, Mistral's top_p, ...). RubyLLM
|
|
19
|
+
# deep-merges provider options, so these nested keys would override the strict format just like the
|
|
20
|
+
# top-level ones. Re-check each protocol's render_payload (ruby_llm protocols/*/chat.rb) on every
|
|
21
|
+
# ruby_llm upgrade.
|
|
22
|
+
GENERATION_CONFIG_RESERVED_KEYS = %i[
|
|
23
|
+
responseMimeType response_mime_type responseSchema response_schema responseJsonSchema response_json_schema
|
|
24
|
+
response_format tools tool_choice toolConfig tool_config
|
|
25
|
+
].freeze
|
|
26
|
+
COMPLETION_ARGS_RESERVED_KEYS = %i[response_format tools tool_choice].freeze
|
|
27
|
+
NESTED_RESERVED_KEYS = {
|
|
28
|
+
generationConfig: GENERATION_CONFIG_RESERVED_KEYS, # Gemini
|
|
29
|
+
generation_config: GENERATION_CONFIG_RESERVED_KEYS, # Gemini Interactions
|
|
30
|
+
completion_args: COMPLETION_ARGS_RESERVED_KEYS # Mistral Conversations
|
|
31
|
+
}.freeze
|
|
32
|
+
|
|
16
33
|
module_function
|
|
17
34
|
|
|
18
35
|
# Validates and symbolizes one layer. nil values are kept so a layer can unset an inherited key.
|
|
@@ -25,10 +42,30 @@ module Squishling
|
|
|
25
42
|
raise ConfigurationError, "#{label}: #{reserved.join(', ')} can't be set through params " \
|
|
26
43
|
"(controlled by Squishling/RubyLLM; use model:/provider:/output_schema)"
|
|
27
44
|
end
|
|
45
|
+
reject_nested_reserved!(params, label)
|
|
28
46
|
params[:thinking] = normalize_thinking(params[:thinking], label) unless params[:thinking].nil?
|
|
29
47
|
params.freeze
|
|
30
48
|
end
|
|
31
49
|
|
|
50
|
+
# Nested keys are compared as symbols so a string-keyed "responseMimeType" can't slip past the check.
|
|
51
|
+
# A non-Hash container would replace (not merge into) the one RubyLLM builds, wiping the strict format.
|
|
52
|
+
# Validated containers are copied and frozen so they can't be mutated after the check.
|
|
53
|
+
def reject_nested_reserved!(params, label)
|
|
54
|
+
nested = NESTED_RESERVED_KEYS.flat_map do |container, reserved|
|
|
55
|
+
value = params[container]
|
|
56
|
+
next [] if value.nil?
|
|
57
|
+
raise ConfigurationError, "#{label}: #{container} must be a Hash, got #{value.class}" unless value.is_a?(Hash)
|
|
58
|
+
|
|
59
|
+
params[container] = value.dup.freeze
|
|
60
|
+
(value.keys.filter_map { |key| key.to_sym if key.respond_to?(:to_sym) } & reserved)
|
|
61
|
+
.map { |key| "#{container}.#{key}" }
|
|
62
|
+
end
|
|
63
|
+
return if nested.empty?
|
|
64
|
+
|
|
65
|
+
raise ConfigurationError, "#{label}: #{nested.join(', ')} can't be set through params " \
|
|
66
|
+
"(controls the strict output format or tools; use output_schema)"
|
|
67
|
+
end
|
|
68
|
+
|
|
32
69
|
# Later layers override earlier ones key by key; nil removes the key (back to the provider default).
|
|
33
70
|
def resolve(*layers)
|
|
34
71
|
layers.compact.reduce({}) { |merged, layer| merged.merge(layer) }.compact
|
data/lib/squishling/router.rb
CHANGED
|
@@ -4,16 +4,16 @@ module Squishling
|
|
|
4
4
|
# Decides, per call, whether a squished method runs its Ruby implementation or the LLM.
|
|
5
5
|
module Router
|
|
6
6
|
# One squished call in progress. `phase` is :routing (deciding, e.g. running squish_when), :ruby (running
|
|
7
|
-
# the implementation), or :llm (on the LLM path, including any fallback); `
|
|
7
|
+
# the implementation), or :llm (on the LLM path, including any fallback); `handed_off` records that squish!
|
|
8
8
|
# already sent it to the LLM.
|
|
9
|
-
Frame = Struct.new(:receiver, :definition, :wrapper, :inputs, :phase, :
|
|
9
|
+
Frame = Struct.new(:receiver, :definition, :wrapper, :inputs, :phase, :handed_off)
|
|
10
10
|
|
|
11
11
|
FRAMES_KEY = :__squishling_frames__
|
|
12
12
|
|
|
13
13
|
class << self
|
|
14
14
|
def dispatch(receiver, name, args, kwargs, impl, wrapper:)
|
|
15
15
|
if (frame = enclosing_frame(receiver, name, wrapper))
|
|
16
|
-
return frame
|
|
16
|
+
return coerce_inner(frame, impl.call)
|
|
17
17
|
end
|
|
18
18
|
|
|
19
19
|
definition = receiver.class.squishling_definition(name)
|
|
@@ -27,7 +27,7 @@ module Squishling
|
|
|
27
27
|
value = impl.call
|
|
28
28
|
rescue NotImplementedError
|
|
29
29
|
# After squish!, the error came from the LLM path (a fallback, a proc), not a missing implementation.
|
|
30
|
-
raise if frame.
|
|
30
|
+
raise if frame.handed_off
|
|
31
31
|
|
|
32
32
|
return route_to_llm(frame, definition, "not implemented")
|
|
33
33
|
end
|
|
@@ -43,7 +43,7 @@ module Squishling
|
|
|
43
43
|
end
|
|
44
44
|
|
|
45
45
|
# squish!: hand the call in progress on this receiver to the LLM, with this call's overrides.
|
|
46
|
-
def
|
|
46
|
+
def hand_off(receiver, overrides)
|
|
47
47
|
frame = current_frame(receiver)
|
|
48
48
|
raise Error, "#{receiver.class}#squish! called outside a squished method" unless frame
|
|
49
49
|
unless frame.phase == :ruby
|
|
@@ -51,12 +51,18 @@ module Squishling
|
|
|
51
51
|
"not from squish_when or while already on the LLM path (e.g. from squish_fallback)"
|
|
52
52
|
end
|
|
53
53
|
|
|
54
|
-
frame.
|
|
54
|
+
frame.handed_off = true
|
|
55
55
|
route_to_llm(frame, frame.definition.for_call(**overrides), "squish!")
|
|
56
56
|
end
|
|
57
57
|
|
|
58
58
|
private
|
|
59
59
|
|
|
60
|
+
# An inner call (`super`, or the method called from squish_when or a fallback) hands its value to user code
|
|
61
|
+
# that may reshape it, so only a Hash is typed (for accessor access); the outermost return is validated in full.
|
|
62
|
+
def coerce_inner(frame, value)
|
|
63
|
+
value.is_a?(Hash) ? frame.definition.coerce(value) : value
|
|
64
|
+
end
|
|
65
|
+
|
|
60
66
|
def route_to_llm(frame, definition, reason)
|
|
61
67
|
log(definition, reason)
|
|
62
68
|
previous = frame.phase
|
|
@@ -73,7 +79,7 @@ module Squishling
|
|
|
73
79
|
|
|
74
80
|
# The call in progress that this one is part of, which runs its Ruby implementation without routing again:
|
|
75
81
|
# a subclass override's `super` (entered through an ancestor's wrapper), or a call made while deciding or on
|
|
76
|
-
# the LLM path (squish_when, squish_fallback,
|
|
82
|
+
# the LLM path (squish_when, squish_fallback, a purpose proc). Only recursion from the Ruby
|
|
77
83
|
# implementation itself is a new call, routed on its own.
|
|
78
84
|
def enclosing_frame(receiver, name, wrapper)
|
|
79
85
|
frame = frames.reverse_each.find do |candidate|
|
data/lib/squishling/schema.rb
CHANGED
|
@@ -7,6 +7,14 @@ module Squishling
|
|
|
7
7
|
CACHE = {}.compare_by_identity
|
|
8
8
|
CACHE_LOCK = Mutex.new
|
|
9
9
|
|
|
10
|
+
# Conditional keywords (Schematist's `given` and `dependent`) that providers' strict modes don't support.
|
|
11
|
+
# They're validated locally and kept out of the schema sent to the LLM.
|
|
12
|
+
LOCAL_ONLY_KEYWORDS = %w[if then else dependentRequired dependentSchemas].freeze
|
|
13
|
+
# Keywords whose value is a map of names to schemas: the names are data, never keywords to strip.
|
|
14
|
+
SCHEMA_MAPS = %w[properties patternProperties $defs definitions].freeze
|
|
15
|
+
# Keywords whose value is literal data, copied untouched.
|
|
16
|
+
DATA_KEYWORDS = %w[enum const default examples].freeze
|
|
17
|
+
|
|
10
18
|
class << self
|
|
11
19
|
def for(raw)
|
|
12
20
|
CACHE_LOCK.synchronize { CACHE[raw] ||= new(raw) }
|
|
@@ -18,9 +26,9 @@ module Squishling
|
|
|
18
26
|
end
|
|
19
27
|
end
|
|
20
28
|
|
|
21
|
-
# The payload handed to RubyLLM's `with_schema`: always strict.
|
|
29
|
+
# The payload handed to RubyLLM's `with_schema`: always strict, without LOCAL_ONLY_KEYWORDS.
|
|
22
30
|
attr_reader :llm_schema
|
|
23
|
-
# The
|
|
31
|
+
# The full JSON Schema used for validation and result typing.
|
|
24
32
|
attr_reader :json_schema
|
|
25
33
|
|
|
26
34
|
def initialize(raw)
|
|
@@ -36,7 +44,7 @@ module Squishling
|
|
|
36
44
|
@llm_schema = {
|
|
37
45
|
"name" => hash["name"] || body["title"],
|
|
38
46
|
"description" => hash["description"] || body["description"],
|
|
39
|
-
"schema" => @json_schema,
|
|
47
|
+
"schema" => provider_schema(@json_schema),
|
|
40
48
|
"strict" => true
|
|
41
49
|
}.compact
|
|
42
50
|
@validator = JSONSchemer.schema(@json_schema)
|
|
@@ -72,6 +80,40 @@ module Squishling
|
|
|
72
80
|
end
|
|
73
81
|
end
|
|
74
82
|
|
|
83
|
+
# A copy of the schema without the conditional keywords. An allOf made only of conditionals (how
|
|
84
|
+
# Schematist emits several `given` blocks) goes too.
|
|
85
|
+
def provider_schema(node)
|
|
86
|
+
case node
|
|
87
|
+
when Array then node.map { |item| provider_schema(item) }
|
|
88
|
+
when Hash
|
|
89
|
+
copy = node.each_with_object({}) do |(key, value), stripped|
|
|
90
|
+
next if LOCAL_ONLY_KEYWORDS.include?(key)
|
|
91
|
+
|
|
92
|
+
stripped[key] =
|
|
93
|
+
if DATA_KEYWORDS.include?(key)
|
|
94
|
+
value
|
|
95
|
+
elsif SCHEMA_MAPS.include?(key) && value.is_a?(Hash)
|
|
96
|
+
value.transform_values { |schema| provider_schema(schema) }
|
|
97
|
+
else
|
|
98
|
+
provider_schema(value)
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
without_conditional_all_of(copy, node)
|
|
102
|
+
else node
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def without_conditional_all_of(copy, original)
|
|
107
|
+
return copy unless original["allOf"].is_a?(Array)
|
|
108
|
+
|
|
109
|
+
kept = copy["allOf"].zip(original["allOf"]).reject { |_, branch| conditional_only?(branch) }.map(&:first)
|
|
110
|
+
kept.empty? ? copy.except("allOf") : copy.merge("allOf" => kept)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def conditional_only?(branch)
|
|
114
|
+
branch.is_a?(Hash) && branch.any? && (branch.keys - LOCAL_ONLY_KEYWORDS - %w[description $comment]).empty?
|
|
115
|
+
end
|
|
116
|
+
|
|
75
117
|
# Some schema objects emit a { name:, description:, schema: {...} } wrapper instead of a bare schema.
|
|
76
118
|
def wrapped?(hash)
|
|
77
119
|
hash["schema"].is_a?(Hash) && !hash.key?("type") && !hash.key?("properties")
|
data/lib/squishling/source.rb
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Squishling
|
|
4
|
-
# Ruby source for
|
|
4
|
+
# Ruby source for append_to_purpose items: a class or module body, or a single method, rendered as a
|
|
5
5
|
# fenced code block. Parsed with Prism (a default gem since Ruby 3.3), loaded the first time it's needed.
|
|
6
6
|
module Source
|
|
7
7
|
# owner => { method name, or nil for the module itself => rendered source }. Weak keys, so classes replaced
|
|
@@ -30,7 +30,7 @@ module Squishling
|
|
|
30
30
|
def module_source(mod)
|
|
31
31
|
constant = constant_location(mod)
|
|
32
32
|
sources = constant ? class_bodies(mod, constant) : method_defs(mod)
|
|
33
|
-
raise ConfigurationError, "
|
|
33
|
+
raise ConfigurationError, "append_to_purpose: source for #{mod.inspect} isn't available" if sources.empty?
|
|
34
34
|
|
|
35
35
|
[display_name(mod), sources.join("\n\n")]
|
|
36
36
|
end
|
|
@@ -58,7 +58,7 @@ module Squishling
|
|
|
58
58
|
def method_source(method)
|
|
59
59
|
label = method_label(method)
|
|
60
60
|
source = def_source(method)
|
|
61
|
-
raise ConfigurationError, "
|
|
61
|
+
raise ConfigurationError, "append_to_purpose: source for #{label} isn't available" unless source
|
|
62
62
|
|
|
63
63
|
[label, source]
|
|
64
64
|
end
|
|
@@ -84,7 +84,7 @@ module Squishling
|
|
|
84
84
|
# A squished method resolves to Squishling's wrapper; show the implementation beneath it.
|
|
85
85
|
def unwrap(method)
|
|
86
86
|
Wrapper.implementation(method) or
|
|
87
|
-
raise ConfigurationError, "
|
|
87
|
+
raise ConfigurationError, "append_to_purpose: #{method.name} has no implementation to show"
|
|
88
88
|
end
|
|
89
89
|
|
|
90
90
|
# Named after the def that's shown, so an alias is labeled by its original name.
|
|
@@ -162,7 +162,7 @@ module Squishling
|
|
|
162
162
|
text = File.read(file, encoding: Encoding::UTF_8).scrub
|
|
163
163
|
[Prism.parse(text).value, text.lines]
|
|
164
164
|
rescue SystemCallError => e
|
|
165
|
-
raise ConfigurationError, "
|
|
165
|
+
raise ConfigurationError, "append_to_purpose: can't read #{file} (#{e.class})"
|
|
166
166
|
end
|
|
167
167
|
|
|
168
168
|
# Give the node's first line its source line's indentation (it may start mid-line, as in
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Squishling
|
|
4
|
+
# The opt-in observability hook: a callable run after every LLM attempt with the attempt's raw output, so
|
|
5
|
+
# it can be sent somewhere (an error tracker, a tracing tool). It's the one sanctioned way for model output
|
|
6
|
+
# to leave the process; nothing runs unless one is configured.
|
|
7
|
+
module Squawk
|
|
8
|
+
KEYWORD_TYPES = %i[key keyreq].freeze
|
|
9
|
+
POSITIONAL_TYPES = %i[req opt].freeze
|
|
10
|
+
KEYWORD_CAPTURE_TYPES = %i[key keyreq keyrest].freeze
|
|
11
|
+
KEYWORDS = %i[output metadata error].freeze
|
|
12
|
+
|
|
13
|
+
module_function
|
|
14
|
+
|
|
15
|
+
# The hook itself, or nil/false to inherit/silence. Anything else must respond to call and take keywords
|
|
16
|
+
# only, which is checked here so a mismatch fails at setup, not inside the first attempt.
|
|
17
|
+
def validate(hook, where)
|
|
18
|
+
return hook if hook.nil? || hook == false
|
|
19
|
+
|
|
20
|
+
unless hook.respond_to?(:call)
|
|
21
|
+
raise ConfigurationError, "#{where} squawk must respond to call (or be false), got #{hook.inspect}"
|
|
22
|
+
end
|
|
23
|
+
if positional_only?(parameters(hook))
|
|
24
|
+
raise ConfigurationError,
|
|
25
|
+
"#{where} squawk is called with keywords (output:, metadata:, error:), not positional arguments"
|
|
26
|
+
end
|
|
27
|
+
unknown = parameters(hook).filter_map { |type, name| name if type == :keyreq && !KEYWORDS.include?(name) }
|
|
28
|
+
raise ConfigurationError, "#{where} squawk requires unknown keyword(s) #{unknown.join(', ')}" if unknown.any?
|
|
29
|
+
|
|
30
|
+
hook
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Calls the hook with the keywords it declares, or all of them when it takes **, so new metadata fields
|
|
34
|
+
# never break an existing lambda. Exceptions it raises are the user's own and propagate as-is.
|
|
35
|
+
def call(hook, **payload)
|
|
36
|
+
parameters = parameters(hook)
|
|
37
|
+
payload = payload.slice(*parameters.filter_map { |type, name| name if KEYWORD_TYPES.include?(type) }) \
|
|
38
|
+
unless parameters.any? { |type, _| type == :keyrest }
|
|
39
|
+
hook.call(**payload)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Positional parameters, or a bare splat, would receive the keywords as a Hash (or not at all).
|
|
43
|
+
def positional_only?(parameters)
|
|
44
|
+
types = parameters.map(&:first)
|
|
45
|
+
types.intersect?(POSITIONAL_TYPES) || (types.include?(:rest) && !types.intersect?(KEYWORD_CAPTURE_TYPES))
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def parameters(hook)
|
|
49
|
+
(hook.is_a?(Proc) || hook.is_a?(Method) ? hook : hook.method(:call)).parameters
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
data/lib/squishling/version.rb
CHANGED
data/lib/squishling.rb
CHANGED
|
@@ -9,15 +9,22 @@ require_relative "squishling/version"
|
|
|
9
9
|
require_relative "squishling/errors"
|
|
10
10
|
require_relative "squishling/configuration"
|
|
11
11
|
require_relative "squishling/params"
|
|
12
|
+
require_relative "squishling/squawk"
|
|
13
|
+
require_relative "squishling/model_path"
|
|
14
|
+
require_relative "squishling/harness"
|
|
12
15
|
require_relative "squishling/result"
|
|
13
16
|
require_relative "squishling/schema"
|
|
14
17
|
require_relative "squishling/source"
|
|
15
18
|
require_relative "squishling/appendices"
|
|
16
19
|
require_relative "squishling/definition"
|
|
20
|
+
require_relative "squishling/llm_client"
|
|
21
|
+
require_relative "squishling/output_check"
|
|
17
22
|
require_relative "squishling/invoker"
|
|
23
|
+
require_relative "squishling/judge"
|
|
18
24
|
require_relative "squishling/router"
|
|
19
25
|
require_relative "squishling/wrapper"
|
|
20
26
|
require_relative "squishling/class_methods"
|
|
27
|
+
require_relative "squishling/collisions"
|
|
21
28
|
|
|
22
29
|
# Include Squishling in a class to make it elastic: its squished methods either run their
|
|
23
30
|
# Ruby implementation or send their inputs through an LLM and return schema-validated results.
|
|
@@ -38,7 +45,11 @@ module Squishling
|
|
|
38
45
|
def included(base)
|
|
39
46
|
raise ConfigurationError, "Squishling can only be included in a class" unless base.is_a?(Class)
|
|
40
47
|
|
|
48
|
+
# A redundant include in a subclass overrides nothing new: the superclass's own methods already won.
|
|
49
|
+
Collisions.check!(base) unless base.superclass&.include?(Squishling)
|
|
41
50
|
base.extend(ClassMethods)
|
|
51
|
+
# `result` is a convenience alias for squishling_result; a class that already has a `result` keeps it.
|
|
52
|
+
base.include(ResultAlias) unless base.method_defined?(:result) || base.private_method_defined?(:result)
|
|
42
53
|
base.send(:squishling_install_wrapper)
|
|
43
54
|
end
|
|
44
55
|
end
|
|
@@ -52,14 +63,23 @@ module Squishling
|
|
|
52
63
|
frame.definition.build_result(attrs || kwargs, squished: false)
|
|
53
64
|
end
|
|
54
65
|
|
|
55
|
-
|
|
66
|
+
# Included separately, and only when the class has no `result` of its own, so `result` never shadows an
|
|
67
|
+
# inherited one. A module (not an alias on the class) keeps it out of the class's own source.
|
|
68
|
+
module ResultAlias
|
|
69
|
+
def result(...)
|
|
70
|
+
squishling_result(...)
|
|
71
|
+
end
|
|
72
|
+
end
|
|
56
73
|
|
|
57
74
|
# Hand the squished method currently executing to the LLM, e.g. from a `rescue` when the Ruby path can't
|
|
58
75
|
# handle this input. Returns the typed result (squished? true, or false when the declared fallback supplied
|
|
59
|
-
# it); return it from the method. The overrides apply to this call only:
|
|
76
|
+
# it); return it from the method. The overrides apply to this call only: append_to_purpose adds to (or,
|
|
60
77
|
# with false, replaces) the declared sections, context is sent alongside the declared squish_context,
|
|
61
|
-
# model/provider/
|
|
62
|
-
|
|
63
|
-
|
|
78
|
+
# model/escalation (one or the other)/provider/purpose/harness replace the declared ones, and params merge
|
|
79
|
+
# key by key over them.
|
|
80
|
+
def squish!(append_to_purpose: nil, context: nil, purpose: nil, model: nil, escalation: nil, provider: nil,
|
|
81
|
+
params: nil, harness: nil)
|
|
82
|
+
Router.hand_off(self, { append_to_purpose:, context:, purpose:, model:, escalation:, provider:, params:,
|
|
83
|
+
harness: })
|
|
64
84
|
end
|
|
65
85
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: squishling
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Michael Carroll
|
|
@@ -29,14 +29,14 @@ dependencies:
|
|
|
29
29
|
requirements:
|
|
30
30
|
- - "~>"
|
|
31
31
|
- !ruby/object:Gem::Version
|
|
32
|
-
version: '2.
|
|
32
|
+
version: '2.1'
|
|
33
33
|
type: :runtime
|
|
34
34
|
prerelease: false
|
|
35
35
|
version_requirements: !ruby/object:Gem::Requirement
|
|
36
36
|
requirements:
|
|
37
37
|
- - "~>"
|
|
38
38
|
- !ruby/object:Gem::Version
|
|
39
|
-
version: '2.
|
|
39
|
+
version: '2.1'
|
|
40
40
|
- !ruby/object:Gem::Dependency
|
|
41
41
|
name: schematist
|
|
42
42
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -72,20 +72,30 @@ files:
|
|
|
72
72
|
- SECURITY.md
|
|
73
73
|
- docs/configuration.md
|
|
74
74
|
- docs/failures.md
|
|
75
|
+
- docs/harnesses.md
|
|
76
|
+
- docs/measuring-tokens.md
|
|
77
|
+
- docs/naming.md
|
|
75
78
|
- docs/routing.md
|
|
76
79
|
- docs/schemas.md
|
|
77
80
|
- lib/squishling.rb
|
|
78
81
|
- lib/squishling/appendices.rb
|
|
79
82
|
- lib/squishling/class_methods.rb
|
|
83
|
+
- lib/squishling/collisions.rb
|
|
80
84
|
- lib/squishling/configuration.rb
|
|
81
85
|
- lib/squishling/definition.rb
|
|
82
86
|
- lib/squishling/errors.rb
|
|
87
|
+
- lib/squishling/harness.rb
|
|
83
88
|
- lib/squishling/invoker.rb
|
|
89
|
+
- lib/squishling/judge.rb
|
|
90
|
+
- lib/squishling/llm_client.rb
|
|
91
|
+
- lib/squishling/model_path.rb
|
|
92
|
+
- lib/squishling/output_check.rb
|
|
84
93
|
- lib/squishling/params.rb
|
|
85
94
|
- lib/squishling/result.rb
|
|
86
95
|
- lib/squishling/router.rb
|
|
87
96
|
- lib/squishling/schema.rb
|
|
88
97
|
- lib/squishling/source.rb
|
|
98
|
+
- lib/squishling/squawk.rb
|
|
89
99
|
- lib/squishling/version.rb
|
|
90
100
|
- lib/squishling/wrapper.rb
|
|
91
101
|
homepage: https://github.com/Coolhand-Labs/squishling
|