squishling 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +83 -0
- data/README.md +85 -30
- data/docs/configuration.md +40 -16
- data/docs/failures.md +68 -8
- data/docs/harnesses.md +198 -0
- data/docs/measuring-tokens.md +148 -0
- data/docs/naming.md +43 -0
- data/docs/routing.md +36 -27
- data/docs/schemas.md +4 -3
- data/lib/squishling/appendices.rb +3 -3
- data/lib/squishling/class_methods.rb +60 -29
- data/lib/squishling/collisions.rb +54 -0
- data/lib/squishling/configuration.rb +19 -0
- data/lib/squishling/definition.rb +48 -23
- data/lib/squishling/errors.rb +22 -1
- data/lib/squishling/harness.rb +153 -0
- data/lib/squishling/invoker.rb +188 -133
- data/lib/squishling/judge.rb +142 -0
- data/lib/squishling/llm_client.rb +150 -0
- data/lib/squishling/model_path.rb +21 -7
- data/lib/squishling/output_check.rb +109 -0
- data/lib/squishling/params.rb +37 -1
- data/lib/squishling/router.rb +8 -2
- data/lib/squishling/source.rb +5 -5
- data/lib/squishling/squawk.rb +52 -0
- data/lib/squishling/version.rb +1 -1
- data/lib/squishling.rb +24 -7
- metadata +12 -3
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Squishling
|
|
4
|
+
# RubyLLM requests for one squished method, with RubyLLM's errors mapped onto Squishling's taxonomy.
|
|
5
|
+
class LLMClient
|
|
6
|
+
# Where RubyLLM's own code lives, to tell its ArgumentErrors from the developer's (see #judge).
|
|
7
|
+
RUBY_LLM_LIB = File.dirname(RubyLLM.method(:judge).source_location.first)
|
|
8
|
+
private_constant :RUBY_LLM_LIB
|
|
9
|
+
|
|
10
|
+
# Runs each job on its own thread and returns, in order, each job's [value, exception]. Only provider
|
|
11
|
+
# requests run here: parsing, validation, and the developer's callbacks stay on the caller's thread. Fiber
|
|
12
|
+
# storage (RubyLLM's usage owner) is inherited by the threads, and RubyLLM's instrumentation context is
|
|
13
|
+
# carried over the way RubyLLM's own concurrent tool calls do it. Unlike those, the threads don't enter the
|
|
14
|
+
# Rails executor: the caller already holds it, and a second share can deadlock against a pending code reload.
|
|
15
|
+
# If the caller is interrupted while waiting (a timeout, Thread#raise), the requests still running are
|
|
16
|
+
# stopped rather than left running unobserved.
|
|
17
|
+
def self.concurrently(jobs)
|
|
18
|
+
return jobs.map { |job| capture(job) } if jobs.size < 2
|
|
19
|
+
|
|
20
|
+
context = instrumentation_context
|
|
21
|
+
threads = jobs.map do |job|
|
|
22
|
+
thread = Thread.new { with_instrumentation_context(context) { capture(job) } }
|
|
23
|
+
thread.report_on_exception = false
|
|
24
|
+
thread
|
|
25
|
+
end
|
|
26
|
+
threads.map(&:value)
|
|
27
|
+
ensure
|
|
28
|
+
threads&.each { |thread| thread.kill.join if thread.alive? }
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def self.capture(job)
|
|
32
|
+
[job.call, nil]
|
|
33
|
+
rescue Exception => e # rubocop:disable Lint/RescueException -- re-raised on the caller's thread
|
|
34
|
+
[nil, e]
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def self.instrumentation_context
|
|
38
|
+
instrumentation = defined?(RubyLLM::Support::Instrumentation) && RubyLLM::Support::Instrumentation
|
|
39
|
+
return unless instrumentation.respond_to?(:current_workflow) && instrumentation.respond_to?(:capture_context)
|
|
40
|
+
|
|
41
|
+
[instrumentation.current_workflow, instrumentation.capture_context]
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def self.with_instrumentation_context(context, &)
|
|
45
|
+
return yield unless context
|
|
46
|
+
|
|
47
|
+
instrumentation = RubyLLM::Support::Instrumentation
|
|
48
|
+
instrumentation.with_workflow(context.first) { instrumentation.with_context(context.last, &) }
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private_class_method :capture, :instrumentation_context, :with_instrumentation_context
|
|
52
|
+
|
|
53
|
+
def initialize(label)
|
|
54
|
+
@label = label
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# A fresh chat on one escalation step, with the system prompt, the strict output schema, and its params.
|
|
58
|
+
def chat(step, instructions:, schema:)
|
|
59
|
+
chat = build_chat(step)
|
|
60
|
+
chat.with_instructions(instructions)
|
|
61
|
+
chat.with_schema(schema)
|
|
62
|
+
apply_params(chat, step.params)
|
|
63
|
+
chat
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Transient HTTP failures are already retried by RubyLLM (RubyLLM.config.max_retries); anything
|
|
67
|
+
# that still fails is surfaced as an LLMError, which moves on to the next attempt of the
|
|
68
|
+
# escalation (or propagates from the last one). A 400 means the request we built is invalid (an
|
|
69
|
+
# unsupported param, a schema the provider rejects), so it's a setup mistake: escalating or a
|
|
70
|
+
# fallback would otherwise hide it on every call.
|
|
71
|
+
def ask(chat, message, step)
|
|
72
|
+
request(step) { chat.ask(message) }
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# A System One judgment (RubyLLM.judge) on one step. The step's params are sent as provider options.
|
|
76
|
+
# RubyLLM raises ArgumentError for a judgment it can't build (e.g. a model on a provider that takes none, or
|
|
77
|
+
# a provider option its protocol reserves), and a bare RubyLLM::Error without an HTTP response for a provider
|
|
78
|
+
# that doesn't support judgments at all. An ArgumentError from anywhere else (an instrumentation subscriber
|
|
79
|
+
# runs inside the call) is the developer's own and propagates as-is.
|
|
80
|
+
def judge(input, questions:, step:)
|
|
81
|
+
request(step) do
|
|
82
|
+
RubyLLM.judge(input, questions:, provider_options: step.params, **model_options(step))
|
|
83
|
+
rescue RubyLLM::ModelNotFoundError => e
|
|
84
|
+
raise ConfigurationError, "#{@label}: #{e.message}"
|
|
85
|
+
rescue ArgumentError => e
|
|
86
|
+
raise unless raised_by_ruby_llm?(e)
|
|
87
|
+
|
|
88
|
+
raise ConfigurationError, "#{@label}: #{e.message}"
|
|
89
|
+
rescue RubyLLM::Error => e
|
|
90
|
+
raise unless e.instance_of?(RubyLLM::Error) && e.response.nil?
|
|
91
|
+
|
|
92
|
+
raise ConfigurationError, "#{@label}: #{e.message}"
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
private
|
|
97
|
+
|
|
98
|
+
def raised_by_ruby_llm?(error)
|
|
99
|
+
error.backtrace&.first&.start_with?("#{RUBY_LLM_LIB}/") || false
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def build_chat(step)
|
|
103
|
+
RubyLLM.chat(**model_options(step))
|
|
104
|
+
rescue RubyLLM::ModelNotFoundError, RubyLLM::ConfigurationError => e
|
|
105
|
+
raise ConfigurationError, "#{@label}: #{e.message}"
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# RubyLLM validates some settings locally (e.g. an impossible thinking budget for the model)
|
|
109
|
+
# and raises ArgumentError before any request is sent.
|
|
110
|
+
def apply_params(chat, params)
|
|
111
|
+
Params.apply(chat, params)
|
|
112
|
+
rescue ArgumentError => e
|
|
113
|
+
raise ConfigurationError, "#{@label}: invalid params #{params.inspect} (#{e.message})"
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def request(step)
|
|
117
|
+
yield
|
|
118
|
+
rescue RubyLLM::ConfigurationError, RubyLLM::UnauthorizedError, RubyLLM::ForbiddenError => e
|
|
119
|
+
raise ConfigurationError, "#{@label}: #{e.class}: #{e.message}"
|
|
120
|
+
rescue RubyLLM::BadRequestError => e
|
|
121
|
+
raise ConfigurationError,
|
|
122
|
+
"#{@label}: the provider rejected the request (#{e.message})#{params_hint(step.params)}"
|
|
123
|
+
rescue RubyLLM::Error, Faraday::Error => e
|
|
124
|
+
raise LLMError, "#{@label}: #{e.class}: #{e.message}"
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# Models missing from RubyLLM's registry (e.g. newly released ones) are only usable when a
|
|
128
|
+
# provider is named, so RubyLLM is told to assume they exist.
|
|
129
|
+
def model_options(step)
|
|
130
|
+
model = step.model
|
|
131
|
+
provider = step.provider
|
|
132
|
+
options = { model:, provider: }.compact
|
|
133
|
+
options[:assume_model_exists] = true if model && provider && !known_model?(model, provider)
|
|
134
|
+
options
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def known_model?(model, provider)
|
|
138
|
+
RubyLLM.models.find(model, provider:)
|
|
139
|
+
true
|
|
140
|
+
rescue RubyLLM::ModelNotFoundError
|
|
141
|
+
false
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def params_hint(params)
|
|
145
|
+
return "" if params.empty?
|
|
146
|
+
|
|
147
|
+
". Check params #{params.inspect}; reasoning models often reject sampling params such as temperature and top_p"
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
end
|
|
@@ -9,12 +9,19 @@ module Squishling
|
|
|
9
9
|
# { model: "claude-opus-5-5", params: { thinking: { effort: :high } } }
|
|
10
10
|
# ]
|
|
11
11
|
# attempts: (default 1) retries a step; order: (all steps or none, unique, lowest first) makes the order
|
|
12
|
-
# explicit instead of positional.
|
|
12
|
+
# explicit instead of positional. forward_rejected: false starts a step from the original input alone, without
|
|
13
|
+
# the previous step's rejected output.
|
|
13
14
|
module ModelPath
|
|
14
|
-
# One attempt: the model, its provider,
|
|
15
|
-
|
|
15
|
+
# One attempt: the model, its provider, the fully resolved generation params, and whether the previous
|
|
16
|
+
# step's rejected output is shown to it.
|
|
17
|
+
Step = Data.define(:model, :provider, :params, :forward_rejected) do
|
|
18
|
+
# The model for logs and messages (nil means RubyLLM's default model).
|
|
19
|
+
def display_name
|
|
20
|
+
model || "RubyLLM default model"
|
|
21
|
+
end
|
|
22
|
+
end
|
|
16
23
|
|
|
17
|
-
STEP_KEYS = %i[model provider params attempts order].freeze
|
|
24
|
+
STEP_KEYS = %i[model provider params attempts order forward_rejected].freeze
|
|
18
25
|
|
|
19
26
|
module_function
|
|
20
27
|
|
|
@@ -54,7 +61,7 @@ module Squishling
|
|
|
54
61
|
def steps(path, provider:, params:)
|
|
55
62
|
path.flat_map do |step|
|
|
56
63
|
attempt = Step.new(model: step[:model], provider: step[:provider] || provider,
|
|
57
|
-
params: Params.resolve(params, step[:params]))
|
|
64
|
+
params: Params.resolve(params, step[:params]), forward_rejected: step[:forward_rejected])
|
|
58
65
|
[attempt] * step[:attempts]
|
|
59
66
|
end
|
|
60
67
|
end
|
|
@@ -76,7 +83,8 @@ module Squishling
|
|
|
76
83
|
|
|
77
84
|
def normalize_step(step, label)
|
|
78
85
|
case step
|
|
79
|
-
when String, Symbol
|
|
86
|
+
when String, Symbol
|
|
87
|
+
{ model: model_name(step, label), provider: nil, params: nil, attempts: 1, forward_rejected: true }
|
|
80
88
|
when Hash then normalize_hash(step, label)
|
|
81
89
|
else
|
|
82
90
|
raise ConfigurationError, "#{label}: each step must be a model name or a Hash with model:, got #{step.inspect}"
|
|
@@ -100,8 +108,14 @@ module Squishling
|
|
|
100
108
|
raise ConfigurationError, "#{label} (#{model}): order: must be an Integer, got #{step[:order].inspect}"
|
|
101
109
|
end
|
|
102
110
|
|
|
111
|
+
forward_rejected = step.fetch(:forward_rejected, true)
|
|
112
|
+
unless [true, false].include?(forward_rejected)
|
|
113
|
+
raise ConfigurationError,
|
|
114
|
+
"#{label} (#{model}): forward_rejected: must be true or false, got #{forward_rejected.inspect}"
|
|
115
|
+
end
|
|
116
|
+
|
|
103
117
|
params = step[:params] && Params.normalize(step[:params], "#{label} (#{model}) params")
|
|
104
|
-
normalized = { model:, provider: step[:provider], params:, attempts: }
|
|
118
|
+
normalized = { model:, provider: step[:provider], params:, attempts:, forward_rejected: }
|
|
105
119
|
step.key?(:order) ? normalized.merge(order: step[:order]) : normalized
|
|
106
120
|
end
|
|
107
121
|
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Squishling
|
|
4
|
+
# Turns a model response into a typed result: parses it, validates it against the schema, and runs the
|
|
5
|
+
# method's squish_validate check.
|
|
6
|
+
class OutputCheck
|
|
7
|
+
MAX_ERRORS = 20
|
|
8
|
+
UNREPRESENTABLE = "response contained a value JSON can't represent (such as Infinity or invalid UTF-8)"
|
|
9
|
+
|
|
10
|
+
def initialize(definition, receiver, inputs)
|
|
11
|
+
@definition = definition
|
|
12
|
+
@receiver = receiver
|
|
13
|
+
@inputs = inputs
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Parses and validates a response: [typed result, []] when it passes, [nil or result, errors] otherwise.
|
|
17
|
+
# The judge's role skips squish_validate, which checks the operation's output, not a verdict.
|
|
18
|
+
def call(content, prompt)
|
|
19
|
+
data, errors = parse(content)
|
|
20
|
+
errors = prompt.schema.validate(data) if errors.empty?
|
|
21
|
+
return [nil, cap(errors)] if errors.any?
|
|
22
|
+
|
|
23
|
+
result = prompt.schema.build(data, squished: true)
|
|
24
|
+
[result, prompt.role == :judge ? [] : validator_errors(result)]
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
# RubyLLM parses structured output itself and leaves the raw string when that fails, so
|
|
30
|
+
# content is a Hash on success, or a String/nil when the model refused, was cut off, or
|
|
31
|
+
# ignored the schema.
|
|
32
|
+
def parse(content)
|
|
33
|
+
return [nil, [UNREPRESENTABLE]] if content.is_a?(String) && !content.valid_encoding?
|
|
34
|
+
return [nil, ["response was empty"]] if content.nil? || (content.is_a?(String) && content.strip.empty?)
|
|
35
|
+
|
|
36
|
+
data = content.is_a?(String) ? JSON.parse(strip_code_fence(content)) : content
|
|
37
|
+
representable?(data) ? [data, []] : [nil, [UNREPRESENTABLE]]
|
|
38
|
+
rescue JSON::ParserError => e
|
|
39
|
+
[nil, ["response was not valid JSON#{parse_position(e)}"]]
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# JSON.parse turns an out-of-range number such as 1e400 into Infinity, and accepts invalid UTF-8 in a string.
|
|
43
|
+
# Both can satisfy the schema but can't be serialized again (and the deterministic path rejects them), so
|
|
44
|
+
# they are invalid output.
|
|
45
|
+
def representable?(data)
|
|
46
|
+
JSON.generate(data)
|
|
47
|
+
true
|
|
48
|
+
rescue JSON::GeneratorError
|
|
49
|
+
false
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# A long output can fail the schema thousands of times over; every error goes into the retry message and
|
|
53
|
+
# the log, so only the first few are kept.
|
|
54
|
+
def cap(errors)
|
|
55
|
+
return errors if errors.size <= MAX_ERRORS
|
|
56
|
+
|
|
57
|
+
errors.first(MAX_ERRORS) + ["... and #{errors.size - MAX_ERRORS} more errors"]
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# The parser's message quotes a snippet of the response, which must not reach error messages or logs
|
|
61
|
+
# (the raw output lives only in InvalidOutputError#raw), so only the position is kept.
|
|
62
|
+
def parse_position(error)
|
|
63
|
+
position = error.message.strip.match(/ at line (\d+) column (\d+)\z/)
|
|
64
|
+
position ? " (at line #{position[1]} column #{position[2]})" : ""
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Models without native structured output sometimes wrap JSON in a markdown code fence.
|
|
68
|
+
def strip_code_fence(text)
|
|
69
|
+
text[/\A\s*```(?:json)?\s*\n(.*?)\n\s*```\s*\z/m, 1] || text
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Runs the squish_validate check, if any. Exceptions it raises are the user's own and propagate as-is.
|
|
73
|
+
def validator_errors(result)
|
|
74
|
+
validator = @definition.validator
|
|
75
|
+
return [] unless validator
|
|
76
|
+
|
|
77
|
+
value = @receiver.instance_exec(result, **@inputs, &validator)
|
|
78
|
+
case value
|
|
79
|
+
when nil, true then []
|
|
80
|
+
when false then ["the output was rejected by squish_validate"]
|
|
81
|
+
when String, Array then Array(value).flatten.compact.map(&:to_s).reject { |message| message.strip.empty? }
|
|
82
|
+
else
|
|
83
|
+
return contract_errors(value) if value.respond_to?(:success?) && value.respond_to?(:errors)
|
|
84
|
+
|
|
85
|
+
raise ConfigurationError, "#{@definition.label}: squish_validate must return nil, true, false, a String, " \
|
|
86
|
+
"an Array of Strings, or a validation result, got #{value.class}"
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# A dry-validation style result: errors.to_h is { key => ["message", ...] }, nested for nested keys.
|
|
91
|
+
def contract_errors(value)
|
|
92
|
+
return [] if value.success?
|
|
93
|
+
|
|
94
|
+
errors = value.errors
|
|
95
|
+
errors = errors.to_h if !errors.is_a?(Array) && errors.respond_to?(:to_h)
|
|
96
|
+
messages = errors.is_a?(Hash) ? flatten_messages(errors) : Array(errors).map(&:to_s)
|
|
97
|
+
messages.empty? ? ["the output was rejected by squish_validate"] : messages
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def flatten_messages(errors, prefix = nil)
|
|
101
|
+
errors.flat_map do |key, value|
|
|
102
|
+
path = [prefix, key].compact.join(".")
|
|
103
|
+
next flatten_messages(value, path) if value.is_a?(Hash)
|
|
104
|
+
|
|
105
|
+
Array(value).map { |message| path.empty? ? message.to_s : "#{path} #{message}" }
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
data/lib/squishling/params.rb
CHANGED
|
@@ -11,9 +11,25 @@ module Squishling
|
|
|
11
11
|
RESERVED_KEYS = %i[
|
|
12
12
|
model messages input inputs instructions contents system system_instruction systemInstruction cachedContent
|
|
13
13
|
stream stream_options store include response_format text output_config outputConfig tools tool_choice
|
|
14
|
-
toolConfig schema
|
|
14
|
+
toolConfig tool_config schema
|
|
15
15
|
].freeze
|
|
16
16
|
|
|
17
|
+
# Some protocols carry the structured-output format (or tool selection) inside a container that is
|
|
18
|
+
# itself allowed, because it also holds ordinary settings (Gemini's topK, Mistral's top_p, ...). RubyLLM
|
|
19
|
+
# deep-merges provider options, so these nested keys would override the strict format just like the
|
|
20
|
+
# top-level ones. Re-check each protocol's render_payload (ruby_llm protocols/*/chat.rb) on every
|
|
21
|
+
# ruby_llm upgrade.
|
|
22
|
+
GENERATION_CONFIG_RESERVED_KEYS = %i[
|
|
23
|
+
responseMimeType response_mime_type responseSchema response_schema responseJsonSchema response_json_schema
|
|
24
|
+
response_format tools tool_choice toolConfig tool_config
|
|
25
|
+
].freeze
|
|
26
|
+
COMPLETION_ARGS_RESERVED_KEYS = %i[response_format tools tool_choice].freeze
|
|
27
|
+
NESTED_RESERVED_KEYS = {
|
|
28
|
+
generationConfig: GENERATION_CONFIG_RESERVED_KEYS, # Gemini
|
|
29
|
+
generation_config: GENERATION_CONFIG_RESERVED_KEYS, # Gemini Interactions
|
|
30
|
+
completion_args: COMPLETION_ARGS_RESERVED_KEYS # Mistral Conversations
|
|
31
|
+
}.freeze
|
|
32
|
+
|
|
17
33
|
module_function
|
|
18
34
|
|
|
19
35
|
# Validates and symbolizes one layer. nil values are kept so a layer can unset an inherited key.
|
|
@@ -26,10 +42,30 @@ module Squishling
|
|
|
26
42
|
raise ConfigurationError, "#{label}: #{reserved.join(', ')} can't be set through params " \
|
|
27
43
|
"(controlled by Squishling/RubyLLM; use model:/provider:/output_schema)"
|
|
28
44
|
end
|
|
45
|
+
reject_nested_reserved!(params, label)
|
|
29
46
|
params[:thinking] = normalize_thinking(params[:thinking], label) unless params[:thinking].nil?
|
|
30
47
|
params.freeze
|
|
31
48
|
end
|
|
32
49
|
|
|
50
|
+
# Nested keys are compared as symbols so a string-keyed "responseMimeType" can't slip past the check.
|
|
51
|
+
# A non-Hash container would replace (not merge into) the one RubyLLM builds, wiping the strict format.
|
|
52
|
+
# Validated containers are copied and frozen so they can't be mutated after the check.
|
|
53
|
+
def reject_nested_reserved!(params, label)
|
|
54
|
+
nested = NESTED_RESERVED_KEYS.flat_map do |container, reserved|
|
|
55
|
+
value = params[container]
|
|
56
|
+
next [] if value.nil?
|
|
57
|
+
raise ConfigurationError, "#{label}: #{container} must be a Hash, got #{value.class}" unless value.is_a?(Hash)
|
|
58
|
+
|
|
59
|
+
params[container] = value.dup.freeze
|
|
60
|
+
(value.keys.filter_map { |key| key.to_sym if key.respond_to?(:to_sym) } & reserved)
|
|
61
|
+
.map { |key| "#{container}.#{key}" }
|
|
62
|
+
end
|
|
63
|
+
return if nested.empty?
|
|
64
|
+
|
|
65
|
+
raise ConfigurationError, "#{label}: #{nested.join(', ')} can't be set through params " \
|
|
66
|
+
"(controls the strict output format or tools; use output_schema)"
|
|
67
|
+
end
|
|
68
|
+
|
|
33
69
|
# Later layers override earlier ones key by key; nil removes the key (back to the provider default).
|
|
34
70
|
def resolve(*layers)
|
|
35
71
|
layers.compact.reduce({}) { |merged, layer| merged.merge(layer) }.compact
|
data/lib/squishling/router.rb
CHANGED
|
@@ -13,7 +13,7 @@ module Squishling
|
|
|
13
13
|
class << self
|
|
14
14
|
def dispatch(receiver, name, args, kwargs, impl, wrapper:)
|
|
15
15
|
if (frame = enclosing_frame(receiver, name, wrapper))
|
|
16
|
-
return frame
|
|
16
|
+
return coerce_inner(frame, impl.call)
|
|
17
17
|
end
|
|
18
18
|
|
|
19
19
|
definition = receiver.class.squishling_definition(name)
|
|
@@ -57,6 +57,12 @@ module Squishling
|
|
|
57
57
|
|
|
58
58
|
private
|
|
59
59
|
|
|
60
|
+
# An inner call (`super`, or the method called from squish_when or a fallback) hands its value to user code
|
|
61
|
+
# that may reshape it, so only a Hash is typed (for accessor access); the outermost return is validated in full.
|
|
62
|
+
def coerce_inner(frame, value)
|
|
63
|
+
value.is_a?(Hash) ? frame.definition.coerce(value) : value
|
|
64
|
+
end
|
|
65
|
+
|
|
60
66
|
def route_to_llm(frame, definition, reason)
|
|
61
67
|
log(definition, reason)
|
|
62
68
|
previous = frame.phase
|
|
@@ -73,7 +79,7 @@ module Squishling
|
|
|
73
79
|
|
|
74
80
|
# The call in progress that this one is part of, which runs its Ruby implementation without routing again:
|
|
75
81
|
# a subclass override's `super` (entered through an ancestor's wrapper), or a call made while deciding or on
|
|
76
|
-
# the LLM path (squish_when, squish_fallback,
|
|
82
|
+
# the LLM path (squish_when, squish_fallback, a purpose proc). Only recursion from the Ruby
|
|
77
83
|
# implementation itself is a new call, routed on its own.
|
|
78
84
|
def enclosing_frame(receiver, name, wrapper)
|
|
79
85
|
frame = frames.reverse_each.find do |candidate|
|
data/lib/squishling/source.rb
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Squishling
|
|
4
|
-
# Ruby source for
|
|
4
|
+
# Ruby source for append_to_purpose items: a class or module body, or a single method, rendered as a
|
|
5
5
|
# fenced code block. Parsed with Prism (a default gem since Ruby 3.3), loaded the first time it's needed.
|
|
6
6
|
module Source
|
|
7
7
|
# owner => { method name, or nil for the module itself => rendered source }. Weak keys, so classes replaced
|
|
@@ -30,7 +30,7 @@ module Squishling
|
|
|
30
30
|
def module_source(mod)
|
|
31
31
|
constant = constant_location(mod)
|
|
32
32
|
sources = constant ? class_bodies(mod, constant) : method_defs(mod)
|
|
33
|
-
raise ConfigurationError, "
|
|
33
|
+
raise ConfigurationError, "append_to_purpose: source for #{mod.inspect} isn't available" if sources.empty?
|
|
34
34
|
|
|
35
35
|
[display_name(mod), sources.join("\n\n")]
|
|
36
36
|
end
|
|
@@ -58,7 +58,7 @@ module Squishling
|
|
|
58
58
|
def method_source(method)
|
|
59
59
|
label = method_label(method)
|
|
60
60
|
source = def_source(method)
|
|
61
|
-
raise ConfigurationError, "
|
|
61
|
+
raise ConfigurationError, "append_to_purpose: source for #{label} isn't available" unless source
|
|
62
62
|
|
|
63
63
|
[label, source]
|
|
64
64
|
end
|
|
@@ -84,7 +84,7 @@ module Squishling
|
|
|
84
84
|
# A squished method resolves to Squishling's wrapper; show the implementation beneath it.
|
|
85
85
|
def unwrap(method)
|
|
86
86
|
Wrapper.implementation(method) or
|
|
87
|
-
raise ConfigurationError, "
|
|
87
|
+
raise ConfigurationError, "append_to_purpose: #{method.name} has no implementation to show"
|
|
88
88
|
end
|
|
89
89
|
|
|
90
90
|
# Named after the def that's shown, so an alias is labeled by its original name.
|
|
@@ -162,7 +162,7 @@ module Squishling
|
|
|
162
162
|
text = File.read(file, encoding: Encoding::UTF_8).scrub
|
|
163
163
|
[Prism.parse(text).value, text.lines]
|
|
164
164
|
rescue SystemCallError => e
|
|
165
|
-
raise ConfigurationError, "
|
|
165
|
+
raise ConfigurationError, "append_to_purpose: can't read #{file} (#{e.class})"
|
|
166
166
|
end
|
|
167
167
|
|
|
168
168
|
# Give the node's first line its source line's indentation (it may start mid-line, as in
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Squishling
|
|
4
|
+
# The opt-in observability hook: a callable run after every LLM attempt with the attempt's raw output, so
|
|
5
|
+
# it can be sent somewhere (an error tracker, a tracing tool). It's the one sanctioned way for model output
|
|
6
|
+
# to leave the process; nothing runs unless one is configured.
|
|
7
|
+
module Squawk
|
|
8
|
+
KEYWORD_TYPES = %i[key keyreq].freeze
|
|
9
|
+
POSITIONAL_TYPES = %i[req opt].freeze
|
|
10
|
+
KEYWORD_CAPTURE_TYPES = %i[key keyreq keyrest].freeze
|
|
11
|
+
KEYWORDS = %i[output metadata error].freeze
|
|
12
|
+
|
|
13
|
+
module_function
|
|
14
|
+
|
|
15
|
+
# The hook itself, or nil/false to inherit/silence. Anything else must respond to call and take keywords
|
|
16
|
+
# only, which is checked here so a mismatch fails at setup, not inside the first attempt.
|
|
17
|
+
def validate(hook, where)
|
|
18
|
+
return hook if hook.nil? || hook == false
|
|
19
|
+
|
|
20
|
+
unless hook.respond_to?(:call)
|
|
21
|
+
raise ConfigurationError, "#{where} squawk must respond to call (or be false), got #{hook.inspect}"
|
|
22
|
+
end
|
|
23
|
+
if positional_only?(parameters(hook))
|
|
24
|
+
raise ConfigurationError,
|
|
25
|
+
"#{where} squawk is called with keywords (output:, metadata:, error:), not positional arguments"
|
|
26
|
+
end
|
|
27
|
+
unknown = parameters(hook).filter_map { |type, name| name if type == :keyreq && !KEYWORDS.include?(name) }
|
|
28
|
+
raise ConfigurationError, "#{where} squawk requires unknown keyword(s) #{unknown.join(', ')}" if unknown.any?
|
|
29
|
+
|
|
30
|
+
hook
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Calls the hook with the keywords it declares, or all of them when it takes **, so new metadata fields
|
|
34
|
+
# never break an existing lambda. Exceptions it raises are the user's own and propagate as-is.
|
|
35
|
+
def call(hook, **payload)
|
|
36
|
+
parameters = parameters(hook)
|
|
37
|
+
payload = payload.slice(*parameters.filter_map { |type, name| name if KEYWORD_TYPES.include?(type) }) \
|
|
38
|
+
unless parameters.any? { |type, _| type == :keyrest }
|
|
39
|
+
hook.call(**payload)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Positional parameters, or a bare splat, would receive the keywords as a Hash (or not at all).
|
|
43
|
+
def positional_only?(parameters)
|
|
44
|
+
types = parameters.map(&:first)
|
|
45
|
+
types.intersect?(POSITIONAL_TYPES) || (types.include?(:rest) && !types.intersect?(KEYWORD_CAPTURE_TYPES))
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def parameters(hook)
|
|
49
|
+
(hook.is_a?(Proc) || hook.is_a?(Method) ? hook : hook.method(:call)).parameters
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
data/lib/squishling/version.rb
CHANGED
data/lib/squishling.rb
CHANGED
|
@@ -9,16 +9,22 @@ require_relative "squishling/version"
|
|
|
9
9
|
require_relative "squishling/errors"
|
|
10
10
|
require_relative "squishling/configuration"
|
|
11
11
|
require_relative "squishling/params"
|
|
12
|
+
require_relative "squishling/squawk"
|
|
12
13
|
require_relative "squishling/model_path"
|
|
14
|
+
require_relative "squishling/harness"
|
|
13
15
|
require_relative "squishling/result"
|
|
14
16
|
require_relative "squishling/schema"
|
|
15
17
|
require_relative "squishling/source"
|
|
16
18
|
require_relative "squishling/appendices"
|
|
17
19
|
require_relative "squishling/definition"
|
|
20
|
+
require_relative "squishling/llm_client"
|
|
21
|
+
require_relative "squishling/output_check"
|
|
18
22
|
require_relative "squishling/invoker"
|
|
23
|
+
require_relative "squishling/judge"
|
|
19
24
|
require_relative "squishling/router"
|
|
20
25
|
require_relative "squishling/wrapper"
|
|
21
26
|
require_relative "squishling/class_methods"
|
|
27
|
+
require_relative "squishling/collisions"
|
|
22
28
|
|
|
23
29
|
# Include Squishling in a class to make it elastic: its squished methods either run their
|
|
24
30
|
# Ruby implementation or send their inputs through an LLM and return schema-validated results.
|
|
@@ -39,7 +45,11 @@ module Squishling
|
|
|
39
45
|
def included(base)
|
|
40
46
|
raise ConfigurationError, "Squishling can only be included in a class" unless base.is_a?(Class)
|
|
41
47
|
|
|
48
|
+
# A redundant include in a subclass overrides nothing new: the superclass's own methods already won.
|
|
49
|
+
Collisions.check!(base) unless base.superclass&.include?(Squishling)
|
|
42
50
|
base.extend(ClassMethods)
|
|
51
|
+
# `result` is a convenience alias for squishling_result; a class that already has a `result` keeps it.
|
|
52
|
+
base.include(ResultAlias) unless base.method_defined?(:result) || base.private_method_defined?(:result)
|
|
43
53
|
base.send(:squishling_install_wrapper)
|
|
44
54
|
end
|
|
45
55
|
end
|
|
@@ -53,16 +63,23 @@ module Squishling
|
|
|
53
63
|
frame.definition.build_result(attrs || kwargs, squished: false)
|
|
54
64
|
end
|
|
55
65
|
|
|
56
|
-
|
|
66
|
+
# Included separately, and only when the class has no `result` of its own, so `result` never shadows an
|
|
67
|
+
# inherited one. A module (not an alias on the class) keeps it out of the class's own source.
|
|
68
|
+
module ResultAlias
|
|
69
|
+
def result(...)
|
|
70
|
+
squishling_result(...)
|
|
71
|
+
end
|
|
72
|
+
end
|
|
57
73
|
|
|
58
74
|
# Hand the squished method currently executing to the LLM, e.g. from a `rescue` when the Ruby path can't
|
|
59
75
|
# handle this input. Returns the typed result (squished? true, or false when the declared fallback supplied
|
|
60
|
-
# it); return it from the method. The overrides apply to this call only:
|
|
76
|
+
# it); return it from the method. The overrides apply to this call only: append_to_purpose adds to (or,
|
|
61
77
|
# with false, replaces) the declared sections, context is sent alongside the declared squish_context,
|
|
62
|
-
# model/escalation (one or the other)/provider/
|
|
63
|
-
# by key over them.
|
|
64
|
-
def squish!(
|
|
65
|
-
params: nil)
|
|
66
|
-
Router.hand_off(self, {
|
|
78
|
+
# model/escalation (one or the other)/provider/purpose/harness replace the declared ones, and params merge
|
|
79
|
+
# key by key over them.
|
|
80
|
+
def squish!(append_to_purpose: nil, context: nil, purpose: nil, model: nil, escalation: nil, provider: nil,
|
|
81
|
+
params: nil, harness: nil)
|
|
82
|
+
Router.hand_off(self, { append_to_purpose:, context:, purpose:, model:, escalation:, provider:, params:,
|
|
83
|
+
harness: })
|
|
67
84
|
end
|
|
68
85
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: squishling
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Michael Carroll
|
|
@@ -29,14 +29,14 @@ dependencies:
|
|
|
29
29
|
requirements:
|
|
30
30
|
- - "~>"
|
|
31
31
|
- !ruby/object:Gem::Version
|
|
32
|
-
version: '2.
|
|
32
|
+
version: '2.1'
|
|
33
33
|
type: :runtime
|
|
34
34
|
prerelease: false
|
|
35
35
|
version_requirements: !ruby/object:Gem::Requirement
|
|
36
36
|
requirements:
|
|
37
37
|
- - "~>"
|
|
38
38
|
- !ruby/object:Gem::Version
|
|
39
|
-
version: '2.
|
|
39
|
+
version: '2.1'
|
|
40
40
|
- !ruby/object:Gem::Dependency
|
|
41
41
|
name: schematist
|
|
42
42
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -72,21 +72,30 @@ files:
|
|
|
72
72
|
- SECURITY.md
|
|
73
73
|
- docs/configuration.md
|
|
74
74
|
- docs/failures.md
|
|
75
|
+
- docs/harnesses.md
|
|
76
|
+
- docs/measuring-tokens.md
|
|
77
|
+
- docs/naming.md
|
|
75
78
|
- docs/routing.md
|
|
76
79
|
- docs/schemas.md
|
|
77
80
|
- lib/squishling.rb
|
|
78
81
|
- lib/squishling/appendices.rb
|
|
79
82
|
- lib/squishling/class_methods.rb
|
|
83
|
+
- lib/squishling/collisions.rb
|
|
80
84
|
- lib/squishling/configuration.rb
|
|
81
85
|
- lib/squishling/definition.rb
|
|
82
86
|
- lib/squishling/errors.rb
|
|
87
|
+
- lib/squishling/harness.rb
|
|
83
88
|
- lib/squishling/invoker.rb
|
|
89
|
+
- lib/squishling/judge.rb
|
|
90
|
+
- lib/squishling/llm_client.rb
|
|
84
91
|
- lib/squishling/model_path.rb
|
|
92
|
+
- lib/squishling/output_check.rb
|
|
85
93
|
- lib/squishling/params.rb
|
|
86
94
|
- lib/squishling/result.rb
|
|
87
95
|
- lib/squishling/router.rb
|
|
88
96
|
- lib/squishling/schema.rb
|
|
89
97
|
- lib/squishling/source.rb
|
|
98
|
+
- lib/squishling/squawk.rb
|
|
90
99
|
- lib/squishling/version.rb
|
|
91
100
|
- lib/squishling/wrapper.rb
|
|
92
101
|
homepage: https://github.com/Coolhand-Labs/squishling
|