agentcat 0.0.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE +2 -11
- data/README.md +265 -20
- data/lib/agentcat/adapters/base.rb +98 -0
- data/lib/agentcat/adapters/official.rb +277 -0
- data/lib/agentcat/client.rb +189 -0
- data/lib/agentcat/clock.rb +16 -0
- data/lib/agentcat/custom_event.rb +215 -0
- data/lib/agentcat/delivery.rb +58 -0
- data/lib/agentcat/diagnostics.rb +434 -0
- data/lib/agentcat/error_capture.rb +117 -0
- data/lib/agentcat/event_id.rb +19 -0
- data/lib/agentcat/event_pipeline.rb +82 -0
- data/lib/agentcat/event_queue.rb +151 -0
- data/lib/agentcat/exporters/base.rb +135 -0
- data/lib/agentcat/exporters/datadog.rb +137 -0
- data/lib/agentcat/exporters/otlp.rb +157 -0
- data/lib/agentcat/exporters/posthog.rb +185 -0
- data/lib/agentcat/exporters/sentry.rb +242 -0
- data/lib/agentcat/exporters/trace_context.rb +79 -0
- data/lib/agentcat/exporters.rb +108 -0
- data/lib/agentcat/fork_safety.rb +30 -0
- data/lib/agentcat/hooks.rb +190 -0
- data/lib/agentcat/http_post.rb +61 -0
- data/lib/agentcat/interceptor.rb +728 -0
- data/lib/agentcat/log.rb +80 -0
- data/lib/agentcat/options.rb +313 -0
- data/lib/agentcat/payload_limits.rb +71 -0
- data/lib/agentcat/private_members.rb +286 -0
- data/lib/agentcat/redaction.rb +176 -0
- data/lib/agentcat/rejected_calls.rb +147 -0
- data/lib/agentcat/request_context.rb +266 -0
- data/lib/agentcat/sanitization.rb +191 -0
- data/lib/agentcat/schema/prompts.rb +44 -0
- data/lib/agentcat/server_info.rb +280 -0
- data/lib/agentcat/session_manager.rb +83 -0
- data/lib/agentcat/tags.rb +61 -0
- data/lib/agentcat/test_mode.rb +86 -0
- data/lib/agentcat/testing.rb +153 -0
- data/lib/agentcat/text.rb +68 -0
- data/lib/agentcat/token_estimate.rb +98 -0
- data/lib/agentcat/tool_call.rb +129 -0
- data/lib/agentcat/tool_handle.rb +289 -0
- data/lib/agentcat/tracker.rb +204 -0
- data/lib/agentcat/truncation.rb +265 -0
- data/lib/agentcat/unredacted_event.rb +155 -0
- data/lib/agentcat/user_identity.rb +11 -0
- data/lib/agentcat/version.rb +2 -2
- data/lib/agentcat.rb +393 -21
- metadata +194 -16
- data/agentcat.gemspec +0 -30
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
# typed: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require_relative "clock"
|
|
5
|
+
require_relative "client"
|
|
6
|
+
require_relative "redaction"
|
|
7
|
+
require_relative "test_mode"
|
|
8
|
+
|
|
9
|
+
module AgentCat
|
|
10
|
+
# The server info heartbeat: what an agent sees on a tracked server (its title,
|
|
11
|
+
# instructions and lists of tools, resources, resource templates and prompts), read
|
|
12
|
+
# through the server's own list handlers and sent to AgentCat about once an hour for
|
|
13
|
+
# each process, project and server name.
|
|
14
|
+
#
|
|
15
|
+
# A request only runs `check`, which reads and writes two in-memory values: the time of
|
|
16
|
+
# the first check in this process and the time of the last attempt for each project and
|
|
17
|
+
# server name. When a heartbeat is due, a short-lived thread does the list calls, the
|
|
18
|
+
# redaction and the POST, and the request goes on without it. The heartbeat writes no
|
|
19
|
+
# file and nothing to standard output or standard error: its log lines are debug only.
|
|
20
|
+
# A process that keeps no memory (a new one, or a fork after the hour) just sends again;
|
|
21
|
+
# AgentCat keeps the newest heartbeat.
|
|
22
|
+
module ServerInfo
|
|
23
|
+
extend T::Sig
|
|
24
|
+
|
|
25
|
+
INTERVAL = 3600
|
|
26
|
+
# A tool call starts a heartbeat only in a memory this old. A tool list request and a
|
|
27
|
+
# `get_more_tools` call start one at any age.
|
|
28
|
+
MIN_MEMORY_AGE = 60
|
|
29
|
+
MAX_KEYS = 1000
|
|
30
|
+
MAX_PAGES = 100
|
|
31
|
+
LIST_TIMEOUT = 5
|
|
32
|
+
BUILD_TIMEOUT = 30
|
|
33
|
+
THREAD_NAME = "agentcat-server-info"
|
|
34
|
+
LIST_THREAD_NAME = "agentcat-server-info-list"
|
|
35
|
+
# [list request, key in `features` and in the handler's result]
|
|
36
|
+
LISTS = T.let([
|
|
37
|
+
["tools/list", "tools"],
|
|
38
|
+
["resources/list", "resources"],
|
|
39
|
+
["resources/templates/list", "resourceTemplates"],
|
|
40
|
+
["prompts/list", "prompts"]
|
|
41
|
+
].freeze, T::Array[[String, String]])
|
|
42
|
+
|
|
43
|
+
# A list that cannot be read whole.
|
|
44
|
+
class ListFailed < StandardError; end
|
|
45
|
+
|
|
46
|
+
@mutex = T.let(Mutex.new, Mutex)
|
|
47
|
+
@first_check_at = T.let(nil, T.nilable(Float))
|
|
48
|
+
# [project_id, server_name] => monotonic time of the last attempt
|
|
49
|
+
@attempts = T.let({}, T::Hash[[String, String], Float])
|
|
50
|
+
@sender = T.let(nil, T.untyped)
|
|
51
|
+
|
|
52
|
+
class << self
|
|
53
|
+
extend T::Sig
|
|
54
|
+
|
|
55
|
+
# Anything with `post_server_info(body) -> Integer | nil`, used in place of the
|
|
56
|
+
# tracked server's client. Tests set it to a recorder.
|
|
57
|
+
sig { returns(T.untyped) }
|
|
58
|
+
attr_accessor :sender
|
|
59
|
+
|
|
60
|
+
# Runs on the request thread. `kind` is :tools_list, :get_more_tools or :tool_call.
|
|
61
|
+
# Never raises and never waits.
|
|
62
|
+
sig { params(tracker: T.untyped, kind: Symbol, server_context: T.untyped).void }
|
|
63
|
+
def check(tracker, kind, server_context = nil)
|
|
64
|
+
return unless due?(tracker, kind)
|
|
65
|
+
|
|
66
|
+
start(tracker, server_context)
|
|
67
|
+
rescue => e
|
|
68
|
+
log("server info check failed: #{e.class}")
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Monotonic seconds; the clock of the guard.
|
|
72
|
+
sig { returns(Float) }
|
|
73
|
+
def now
|
|
74
|
+
Clock.monotonic_now
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
sig { returns(Float) }
|
|
78
|
+
def list_timeout
|
|
79
|
+
LIST_TIMEOUT.to_f
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
sig { returns(Float) }
|
|
83
|
+
def build_timeout
|
|
84
|
+
BUILD_TIMEOUT.to_f
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Forgets the guard and the first check, as a new process would.
|
|
88
|
+
sig { void }
|
|
89
|
+
def reset!
|
|
90
|
+
@mutex.synchronize do
|
|
91
|
+
@first_check_at = nil
|
|
92
|
+
@attempts = {}
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Reads every page of one list through `handler` (`(params, server_context) ->
|
|
97
|
+
# result`). Raises ListFailed when a cursor repeats, when the list has more than
|
|
98
|
+
# MAX_PAGES pages or when a page is not a list. Each item is the JSON the server
|
|
99
|
+
# would put on the wire.
|
|
100
|
+
sig { params(handler: T.untyped, key: String, server_context: T.untyped).returns(T::Array[T.untyped]) }
|
|
101
|
+
def read_pages(handler, key, server_context)
|
|
102
|
+
items = []
|
|
103
|
+
cursors = Set.new
|
|
104
|
+
cursor = T.let(nil, T.untyped)
|
|
105
|
+
MAX_PAGES.times do
|
|
106
|
+
result = handler.call(cursor.nil? ? {} : {cursor: cursor}, server_context)
|
|
107
|
+
raise ListFailed, "the #{key} result is not an object" unless result.is_a?(Hash)
|
|
108
|
+
|
|
109
|
+
page = result[key.to_sym] || result[key]
|
|
110
|
+
raise ListFailed, "the #{key} result has no list" unless page.is_a?(Array)
|
|
111
|
+
|
|
112
|
+
items.concat(JSON.parse(JSON.generate(page)))
|
|
113
|
+
cursor = result[:nextCursor] || result["nextCursor"]
|
|
114
|
+
return items if cursor.nil?
|
|
115
|
+
raise ListFailed, "the #{key} list repeated a cursor" unless cursors.add?(cursor)
|
|
116
|
+
end
|
|
117
|
+
raise ListFailed, "the #{key} list has more than #{MAX_PAGES} pages"
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
private
|
|
121
|
+
|
|
122
|
+
sig { params(tracker: T.untyped, kind: Symbol).returns(T::Boolean) }
|
|
123
|
+
def due?(tracker, kind)
|
|
124
|
+
project_id = tracker.project_id
|
|
125
|
+
return false if project_id.nil? || project_id.empty?
|
|
126
|
+
return false unless tracker.options.enable_tracing
|
|
127
|
+
return false unless AgentCat.active_fake.nil? && TestMode.sdk_enabled?
|
|
128
|
+
|
|
129
|
+
key = [project_id, tracker.server_info[:name].to_s]
|
|
130
|
+
current = now
|
|
131
|
+
@mutex.synchronize do
|
|
132
|
+
first = (@first_check_at ||= current)
|
|
133
|
+
next false if kind == :tool_call && current - first < MIN_MEMORY_AGE
|
|
134
|
+
|
|
135
|
+
last = @attempts[key]
|
|
136
|
+
next false if last && current - last < INTERVAL
|
|
137
|
+
|
|
138
|
+
@attempts = {} if @attempts.size >= MAX_KEYS
|
|
139
|
+
@attempts[key] = current
|
|
140
|
+
true
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
sig { params(tracker: T.untyped, server_context: T.untyped).void }
|
|
145
|
+
def start(tracker, server_context)
|
|
146
|
+
sender = @sender || tracker.options.client
|
|
147
|
+
epoch = AgentCat.event_epoch
|
|
148
|
+
thread = Thread.new do
|
|
149
|
+
Thread.current.report_on_exception = false
|
|
150
|
+
run(tracker, server_context, sender, epoch)
|
|
151
|
+
# Every error stays on this thread, also one that is not a StandardError: with
|
|
152
|
+
# `Thread.abort_on_exception` on, an error that leaves it would reach the main thread.
|
|
153
|
+
rescue Exception => e # standard:disable Lint/RescueException
|
|
154
|
+
log("server info failed: #{e.class}")
|
|
155
|
+
end
|
|
156
|
+
thread.name = THREAD_NAME
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
sig { params(tracker: T.untyped, server_context: T.untyped, sender: T.untyped, epoch: Integer).void }
|
|
160
|
+
def run(tracker, server_context, sender, epoch)
|
|
161
|
+
return log("server info skipped: the client cannot send it") unless sender.respond_to?(:post_server_info)
|
|
162
|
+
|
|
163
|
+
deadline = Clock.monotonic_now + build_timeout
|
|
164
|
+
fields = build(tracker, server_context, deadline)
|
|
165
|
+
return if fields.nil?
|
|
166
|
+
|
|
167
|
+
fields = Redaction.apply_server_info_redaction(fields, tracker.options, deadline: deadline)
|
|
168
|
+
return log("server info skipped: redact_sensitive_information raised, timed out or returned a non-String") if fields.nil?
|
|
169
|
+
return log("server info skipped: it took more than #{build_timeout} seconds to build") if Clock.monotonic_now > deadline
|
|
170
|
+
|
|
171
|
+
info = tracker.server_info
|
|
172
|
+
body = {project_id: tracker.project_id, server_name: info[:name], server_version: info[:version]}.merge(fields)
|
|
173
|
+
deliver(sender, body, epoch)
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
sig { params(tracker: T.untyped, server_context: T.untyped, deadline: Float).returns(T.nilable(T::Hash[Symbol, T.untyped])) }
|
|
177
|
+
def build(tracker, server_context, deadline)
|
|
178
|
+
# Tools defined since the last request get AgentCat's parameters first, so the list
|
|
179
|
+
# is the one a client gets.
|
|
180
|
+
tracker.sync!
|
|
181
|
+
adapter = tracker.adapter
|
|
182
|
+
features = {}
|
|
183
|
+
LISTS.each do |method, key|
|
|
184
|
+
handler = adapter.list_handler(method)
|
|
185
|
+
next if handler.nil?
|
|
186
|
+
|
|
187
|
+
items = collect(handler, key, server_context, deadline)
|
|
188
|
+
if items
|
|
189
|
+
features[key] = items
|
|
190
|
+
elsif key == "tools"
|
|
191
|
+
return log("server info skipped: the tool list failed")
|
|
192
|
+
end
|
|
193
|
+
end
|
|
194
|
+
return log("server info skipped: the server has no tool list") unless features.key?("tools")
|
|
195
|
+
|
|
196
|
+
{server_title: adapter.server_title, instructions: adapter.instructions, features: features}
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
sig { params(handler: T.untyped, key: String, server_context: T.untyped, deadline: Float).returns(T.nilable(T::Array[T.untyped])) }
|
|
200
|
+
def collect(handler, key, server_context, deadline)
|
|
201
|
+
seconds = [list_timeout, deadline - Clock.monotonic_now].min
|
|
202
|
+
status, value = within(seconds) { read_pages(handler, key, server_context) }
|
|
203
|
+
return value if status == :ok
|
|
204
|
+
|
|
205
|
+
reason = (status == :timeout) ? "it took more than #{seconds.round(1)} seconds" : value.class.to_s
|
|
206
|
+
log("server info: the #{key} list failed (#{reason})")
|
|
207
|
+
nil
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# Runs the block on a thread of its own and waits at most `seconds` for it: a list
|
|
211
|
+
# handler that never returns costs its list, not the heartbeat. [:ok, value],
|
|
212
|
+
# [:error, exception] or [:timeout].
|
|
213
|
+
sig { params(seconds: Numeric, block: T.proc.returns(T.untyped)).returns([Symbol, T.untyped]) }
|
|
214
|
+
def within(seconds, &block)
|
|
215
|
+
outcome = T.let([:timeout, nil], [Symbol, T.untyped])
|
|
216
|
+
thread = Thread.new do
|
|
217
|
+
Thread.current.report_on_exception = false
|
|
218
|
+
outcome = [:ok, block.call]
|
|
219
|
+
rescue Exception => e # standard:disable Lint/RescueException
|
|
220
|
+
outcome = [:error, e]
|
|
221
|
+
end
|
|
222
|
+
thread.name = LIST_THREAD_NAME
|
|
223
|
+
return outcome if thread.join([seconds, 0].max)
|
|
224
|
+
|
|
225
|
+
thread.kill
|
|
226
|
+
[:timeout, nil]
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
sig { params(sender: T.untyped, body: T::Hash[Symbol, T.untyped], epoch: Integer).void }
|
|
230
|
+
def deliver(sender, body, epoch)
|
|
231
|
+
status = post(sender, body, epoch)
|
|
232
|
+
if status == 413
|
|
233
|
+
body = body.merge(features: without_schemas(body[:features]))
|
|
234
|
+
status = post(sender, body, epoch)
|
|
235
|
+
end
|
|
236
|
+
return if status == :stale
|
|
237
|
+
|
|
238
|
+
if status.is_a?(Integer) && status.between?(200, 299)
|
|
239
|
+
log("server info sent: #{body[:features]["tools"].size} tools, #{byte_size(body)} bytes")
|
|
240
|
+
else
|
|
241
|
+
log("server info not accepted: #{status.nil? ? "no answer" : "HTTP #{status}"}")
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# The answer's status, nil for no answer, :stale after `AgentCat.reset!`.
|
|
246
|
+
sig { params(sender: T.untyped, body: T::Hash[Symbol, T.untyped], epoch: Integer).returns(T.untyped) }
|
|
247
|
+
def post(sender, body, epoch)
|
|
248
|
+
return :stale if epoch != AgentCat.event_epoch
|
|
249
|
+
|
|
250
|
+
sender.post_server_info(body)
|
|
251
|
+
rescue => e
|
|
252
|
+
log("server info send failed: #{e.class}")
|
|
253
|
+
nil
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
# Each tool keeps its name, title and description, for a body under the size limit.
|
|
257
|
+
sig { params(features: T::Hash[String, T.untyped]).returns(T::Hash[String, T.untyped]) }
|
|
258
|
+
def without_schemas(features)
|
|
259
|
+
features.merge("tools" => features.fetch("tools").map { |tool| tool.is_a?(Hash) ? tool.except("inputSchema", "outputSchema") : tool })
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
sig { params(body: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
|
|
263
|
+
def byte_size(body)
|
|
264
|
+
JSON.generate(body).bytesize
|
|
265
|
+
rescue
|
|
266
|
+
"unknown"
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
# Debug only: the default logger writes warnings to standard error, which a stdio
|
|
270
|
+
# server must keep clean. Returns nil, so a step can `return log(...)`.
|
|
271
|
+
sig { params(line: String).returns(NilClass) }
|
|
272
|
+
def log(line)
|
|
273
|
+
AgentCat.logger.debug(line)
|
|
274
|
+
nil
|
|
275
|
+
rescue
|
|
276
|
+
nil
|
|
277
|
+
end
|
|
278
|
+
end
|
|
279
|
+
end
|
|
280
|
+
end
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# typed: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "securerandom"
|
|
5
|
+
|
|
6
|
+
module AgentCat
|
|
7
|
+
# Mints/validates "ses_<27-char base62 KSUID>" ids, same format the backend uses.
|
|
8
|
+
# No signature needed — 128 random bits already make these unguessable.
|
|
9
|
+
class SessionManager
|
|
10
|
+
PREFIX = "ses_"
|
|
11
|
+
START_SENTINEL = "start"
|
|
12
|
+
KSUID_EPOCH = 1_400_000_000 # 2017-05-13T16:53:20Z, the standard KSUID epoch (seconds)
|
|
13
|
+
# 2024-01-01T00:00:00Z in Unix milliseconds — the fixed epoch used for
|
|
14
|
+
# deterministic (derived) session IDs, matching the Go/TypeScript SDKs.
|
|
15
|
+
EPOCH_2024_MS = 1_704_067_200_000
|
|
16
|
+
# Caps the deterministic timestamp offset at one year (same as the Go SDK).
|
|
17
|
+
MAX_TIMESTAMP_OFFSET_MS = 365 * 24 * 60 * 60 * 1000
|
|
18
|
+
BASE62_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
|
|
19
|
+
BODY_PATTERN = /\A[0-9A-Za-z]{27}\z/
|
|
20
|
+
|
|
21
|
+
class << self
|
|
22
|
+
extend T::Sig
|
|
23
|
+
|
|
24
|
+
sig { returns(String).checked(:tests) }
|
|
25
|
+
def mint_session_id
|
|
26
|
+
mint_id(T.must(PREFIX))
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Shared by SessionManager (ses_) and EventId (evt_) — same KSUID body, different prefix.
|
|
30
|
+
sig { params(prefix: String).returns(String) }
|
|
31
|
+
def mint_id(prefix)
|
|
32
|
+
"#{prefix}#{generate_ksuid}"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
sig { params(session_id: T.untyped).returns(T::Boolean).checked(:tests) }
|
|
36
|
+
def valid?(session_id)
|
|
37
|
+
return false unless session_id.is_a?(String)
|
|
38
|
+
|
|
39
|
+
session_id.start_with?(PREFIX) && BODY_PATTERN.match?(session_id.delete_prefix(PREFIX))
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Deterministically turns a host's own correlation id into a session id. Same
|
|
43
|
+
# input always produces the same output, across processes and restarts, so
|
|
44
|
+
# workers never split a session. Has to match the Go SDK's DeriveSessionID
|
|
45
|
+
# byte-for-byte — if you touch this, check against the golden vectors.
|
|
46
|
+
sig { params(customer_id: T.nilable(String), project_id: T.nilable(String)).returns(String) }
|
|
47
|
+
def derive_session_id(customer_id, project_id = nil)
|
|
48
|
+
input = project_id.to_s.empty? ? customer_id.to_s : "#{customer_id}:#{project_id}"
|
|
49
|
+
hash = Digest::SHA256.digest(input)
|
|
50
|
+
offset_ms = hash.byteslice(0, 4).unpack1("N") % MAX_TIMESTAMP_OFFSET_MS
|
|
51
|
+
timestamp_seconds = (EPOCH_2024_MS + offset_ms) / 1000
|
|
52
|
+
ksuid_seconds = timestamp_seconds - KSUID_EPOCH
|
|
53
|
+
raw = [ksuid_seconds].pack("N") + hash.byteslice(4, 16)
|
|
54
|
+
"#{PREFIX}#{base62_encode(raw)}"
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
sig { params(value: T.untyped).returns(T::Boolean).checked(:tests) }
|
|
58
|
+
def start_sentinel?(value)
|
|
59
|
+
value.is_a?(String) && value.strip.casecmp?(START_SENTINEL)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
private
|
|
63
|
+
|
|
64
|
+
sig { returns(String) }
|
|
65
|
+
def generate_ksuid
|
|
66
|
+
timestamp = Time.now.to_i - KSUID_EPOCH
|
|
67
|
+
raw = [timestamp].pack("N") + SecureRandom.random_bytes(16)
|
|
68
|
+
base62_encode(raw)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
sig { params(bytes: String).returns(String) }
|
|
72
|
+
def base62_encode(bytes)
|
|
73
|
+
number = bytes.unpack1("H*").to_i(16)
|
|
74
|
+
chars = T.let([], T::Array[String])
|
|
75
|
+
while number.positive?
|
|
76
|
+
number, remainder = number.divmod(62)
|
|
77
|
+
chars.unshift(T.must(BASE62_ALPHABET[remainder]))
|
|
78
|
+
end
|
|
79
|
+
chars.join.rjust(27, "0")
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# typed: true
|
|
2
|
+
|
|
3
|
+
require_relative "payload_limits"
|
|
4
|
+
require_relative "text"
|
|
5
|
+
|
|
6
|
+
module AgentCat
|
|
7
|
+
# Validates `event_tags` hook output and `publish_custom_event`'s `tags:` the same
|
|
8
|
+
# way: matches agentcat-api's generated limits (keys up to 32 chars from a fixed
|
|
9
|
+
# charset, values up to 200 with no newlines, 50 entries max). Lengths are UTF-16
|
|
10
|
+
# code units, like the TS SDK. Bad entries are dropped one at a time, not the whole hash.
|
|
11
|
+
# Dropping anything logs one WARN line (rate-limited) with the count, never the tags.
|
|
12
|
+
module Tags
|
|
13
|
+
extend T::Sig
|
|
14
|
+
|
|
15
|
+
# `source` names where the tags came from, for the WARN line.
|
|
16
|
+
sig { params(value: T.untyped, source: String).returns(T::Hash[String, String]) }
|
|
17
|
+
def self.sanitize(value, source = "tags")
|
|
18
|
+
return {} if value.nil?
|
|
19
|
+
|
|
20
|
+
unless value.is_a?(Hash)
|
|
21
|
+
Log.warn("#{source}: expected a Hash of tags, got #{value.class}; ignored")
|
|
22
|
+
return {}
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
invalid = 0
|
|
26
|
+
over_limit = 0
|
|
27
|
+
kept = value.each_with_object({}) do |(key, val), acc|
|
|
28
|
+
if !valid_key?(key) || !valid_value?(val)
|
|
29
|
+
invalid += 1
|
|
30
|
+
elsif acc.size >= PayloadLimits::MAX_TAGS
|
|
31
|
+
over_limit += 1
|
|
32
|
+
else
|
|
33
|
+
acc[key] = val
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
warn_dropped(source, invalid, over_limit)
|
|
37
|
+
kept
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
sig { params(source: String, invalid: Integer, over_limit: Integer).void }
|
|
41
|
+
def self.warn_dropped(source, invalid, over_limit)
|
|
42
|
+
reasons = []
|
|
43
|
+
reasons << "#{invalid} invalid (a key is a String of up to #{PayloadLimits::MAX_TAG_KEY_LENGTH} letters, digits, spaces or _.:$-; a value is a String of up to #{PayloadLimits::MAX_TAG_VALUE_LENGTH} characters with no line break)" if invalid.positive?
|
|
44
|
+
reasons << "#{over_limit} over the limit of #{PayloadLimits::MAX_TAGS} tags" if over_limit.positive?
|
|
45
|
+
Log.warn("#{source}: dropped #{reasons.join(" and ")}") unless reasons.empty?
|
|
46
|
+
end
|
|
47
|
+
private_class_method :warn_dropped
|
|
48
|
+
|
|
49
|
+
sig { params(key: T.untyped).returns(T::Boolean) }
|
|
50
|
+
def self.valid_key?(key)
|
|
51
|
+
!!(key.is_a?(String) && !Text.exceeds_utf16?(key, PayloadLimits::MAX_TAG_KEY_LENGTH) && PayloadLimits::TAG_KEY_PATTERN.match?(key))
|
|
52
|
+
end
|
|
53
|
+
private_class_method :valid_key?
|
|
54
|
+
|
|
55
|
+
sig { params(value: T.untyped).returns(T::Boolean) }
|
|
56
|
+
def self.valid_value?(value)
|
|
57
|
+
!!(value.is_a?(String) && !Text.exceeds_utf16?(value, PayloadLimits::MAX_TAG_VALUE_LENGTH) && !value.match?(/[\r\n]/))
|
|
58
|
+
end
|
|
59
|
+
private_class_method :valid_value?
|
|
60
|
+
end
|
|
61
|
+
end
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# typed: true
|
|
2
|
+
|
|
3
|
+
module AgentCat
|
|
4
|
+
# When AgentCat instruments and delivers. `AgentCat.fake` decides first:
|
|
5
|
+
# a fake always records. Then `AGENTCAT_ENABLED`: a false value turns AgentCat off
|
|
6
|
+
# everywhere (an operator's kill switch), a true value turns it on everywhere, test
|
|
7
|
+
# environments included. Otherwise AgentCat is off while a test framework runs and on
|
|
8
|
+
# everywhere else, so a customer's test suite never sends events or diagnostics.
|
|
9
|
+
module TestMode
|
|
10
|
+
TRUE_WORDS = %w[true 1 yes on].freeze
|
|
11
|
+
FALSE_WORDS = %w[false 0 no off].freeze
|
|
12
|
+
TEST_ENV_NAMES = %w[RAILS_ENV RACK_ENV APP_ENV].freeze
|
|
13
|
+
# By name: neither framework is a dependency of the gem.
|
|
14
|
+
RUNNER_NAMES = %w[Minitest::Test RSpec::Core].freeze
|
|
15
|
+
TEST_ENV_VALUES = %w[test testing].freeze
|
|
16
|
+
|
|
17
|
+
@mutex = Mutex.new
|
|
18
|
+
@reported = T.let(false, T::Boolean)
|
|
19
|
+
|
|
20
|
+
class << self
|
|
21
|
+
extend T::Sig
|
|
22
|
+
|
|
23
|
+
# True when Minitest or RSpec is loaded, or RAILS_ENV / RACK_ENV / APP_ENV is
|
|
24
|
+
# `test` or `testing` (trimmed, any case). `runner_loaded:` is a test seam.
|
|
25
|
+
sig { params(runner_loaded: T::Boolean).returns(T::Boolean) }
|
|
26
|
+
def test_environment?(runner_loaded: runner_loaded?)
|
|
27
|
+
return true if runner_loaded
|
|
28
|
+
|
|
29
|
+
TEST_ENV_NAMES.any? { |name| TEST_ENV_VALUES.include?(ENV[name].to_s.strip.downcase) }
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Whether `track` instruments and `publish_custom_event` publishes. An unknown
|
|
33
|
+
# AGENTCAT_ENABLED value counts as unset and logs one WARN line per process.
|
|
34
|
+
sig { params(runner_loaded: T::Boolean).returns(T::Boolean) }
|
|
35
|
+
def sdk_enabled?(runner_loaded: runner_loaded?)
|
|
36
|
+
return true if AgentCat.active_fake
|
|
37
|
+
|
|
38
|
+
word = enabled_word
|
|
39
|
+
return word unless word.nil?
|
|
40
|
+
|
|
41
|
+
!test_environment?(runner_loaded: runner_loaded)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# The AGENTCAT_ENABLED word: true, false, or nil when unset, blank or unknown.
|
|
45
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
46
|
+
def enabled_word
|
|
47
|
+
raw = ENV["AGENTCAT_ENABLED"].to_s
|
|
48
|
+
value = raw.strip.downcase
|
|
49
|
+
return true if TRUE_WORDS.include?(value)
|
|
50
|
+
return false if FALSE_WORDS.include?(value)
|
|
51
|
+
|
|
52
|
+
report_unknown(raw) unless value.empty?
|
|
53
|
+
nil
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Tests only: forgets the logged line.
|
|
57
|
+
sig { void }
|
|
58
|
+
def reset!
|
|
59
|
+
@mutex.synchronize { @reported = false }
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
private
|
|
63
|
+
|
|
64
|
+
# A loaded runner, not an installed gem: `minitest` sits in many bundles as a
|
|
65
|
+
# dependency, but only a test process loads Minitest::Test or RSpec::Core.
|
|
66
|
+
sig { returns(T::Boolean) }
|
|
67
|
+
def runner_loaded?
|
|
68
|
+
RUNNER_NAMES.any? { |name| Object.const_defined?(name) }
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
sig { params(raw: String).void }
|
|
72
|
+
def report_unknown(raw)
|
|
73
|
+
first = @mutex.synchronize do
|
|
74
|
+
next false if @reported
|
|
75
|
+
|
|
76
|
+
@reported = true
|
|
77
|
+
end
|
|
78
|
+
return unless first
|
|
79
|
+
|
|
80
|
+
AgentCat.logger.warn(
|
|
81
|
+
"AGENTCAT_ENABLED #{raw.inspect} is not one of #{(TRUE_WORDS + FALSE_WORDS).join(", ")}; the default is used"
|
|
82
|
+
)
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# typed: true
|
|
2
|
+
|
|
3
|
+
module AgentCat
|
|
4
|
+
# Test doubles for your own test suite. Loaded on first use of `AgentCat.fake` (or
|
|
5
|
+
# `require "agentcat/testing"`), never by `require "agentcat"` alone.
|
|
6
|
+
module Testing
|
|
7
|
+
# Raised by a failed `Fake` assertion when neither Minitest nor RSpec is loaded.
|
|
8
|
+
class AssertionError < StandardError; end
|
|
9
|
+
|
|
10
|
+
# What `AgentCat.fake` returns. While it is active, every
|
|
11
|
+
# event is recorded here - after hooks, redaction, sanitization and truncation -
|
|
12
|
+
# and nothing goes to AgentCat, to the exporters, or to diagnostics. It records
|
|
13
|
+
# whatever AGENTCAT_ENABLED says, so a CI that sets AGENTCAT_ENABLED=false still
|
|
14
|
+
# works. `AgentCat.reset!` (in your teardown) removes it.
|
|
15
|
+
#
|
|
16
|
+
# Events are the wire hashes AgentCat would send (`event[:event_type]`,
|
|
17
|
+
# `event[:resource_name]`, `event[:parameters]`, ...). Assertions raise
|
|
18
|
+
# Minitest::Assertion under Minitest, RSpec::Expectations::ExpectationNotMetError
|
|
19
|
+
# under RSpec, and AgentCat::Testing::AssertionError otherwise.
|
|
20
|
+
class Fake
|
|
21
|
+
extend T::Sig
|
|
22
|
+
|
|
23
|
+
TOOL_CALL = "mcp:tools/call"
|
|
24
|
+
CUSTOM_EVENT = "agentcat:custom"
|
|
25
|
+
FLUSH_TIMEOUT = 5
|
|
26
|
+
|
|
27
|
+
sig { void }
|
|
28
|
+
def initialize
|
|
29
|
+
@mutex = T.let(Mutex.new, Mutex)
|
|
30
|
+
@events = T.let([], T::Array[T::Hash[Symbol, T.untyped]])
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Every recorded event, in publish order. Waits for queued events first, so a
|
|
34
|
+
# tool call that just returned is in the list.
|
|
35
|
+
sig { returns(T::Array[T::Hash[Symbol, T.untyped]]) }
|
|
36
|
+
def events
|
|
37
|
+
AgentCat.flush(timeout: FLUSH_TIMEOUT)
|
|
38
|
+
@mutex.synchronize { @events.dup }
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Called by AgentCat's pipeline; not part of the API.
|
|
42
|
+
sig { params(event: T::Hash[Symbol, T.untyped]).void }
|
|
43
|
+
def record(event)
|
|
44
|
+
@mutex.synchronize { @events << event }
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Asserts that a tool call (`mcp:tools/call`) was tracked: of `tool_name` when
|
|
48
|
+
# given, matching the block when given, and exactly `count:` of them when given
|
|
49
|
+
# (else at least one).
|
|
50
|
+
sig do
|
|
51
|
+
params(
|
|
52
|
+
tool_name: T.nilable(String),
|
|
53
|
+
count: T.nilable(Integer),
|
|
54
|
+
filter: T.nilable(T.proc.params(event: T::Hash[Symbol, T.untyped]).returns(T.untyped))
|
|
55
|
+
).void
|
|
56
|
+
end
|
|
57
|
+
def assert_tracked(tool_name = nil, count: nil, &filter)
|
|
58
|
+
tool_calls = of_type(TOOL_CALL)
|
|
59
|
+
matched = matching(tool_calls, tool_name, filter)
|
|
60
|
+
what = describe("a tool call", tool_name, filter)
|
|
61
|
+
if count.nil?
|
|
62
|
+
check(!matched.empty?, hinted("Expected #{what} to be tracked, but none was. Tracked: #{names(tool_calls)}.", tool_calls))
|
|
63
|
+
else
|
|
64
|
+
check(matched.size == count, hinted("Expected #{count} of #{what} to be tracked, got #{matched.size}. Tracked: #{names(tool_calls)}.", tool_calls))
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Asserts the number of tracked tool calls; custom events do not count.
|
|
69
|
+
sig { params(count: Integer).void }
|
|
70
|
+
def assert_tracked_count(count)
|
|
71
|
+
tool_calls = of_type(TOOL_CALL)
|
|
72
|
+
check(tool_calls.size == count, hinted("Expected #{count} tool call(s) to be tracked, got #{tool_calls.size}. Tracked: #{names(tool_calls)}.", tool_calls))
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Asserts that no event of any type was recorded.
|
|
76
|
+
sig { void }
|
|
77
|
+
def assert_nothing_tracked
|
|
78
|
+
recorded = events
|
|
79
|
+
check(recorded.empty?, "Expected nothing to be tracked, but #{recorded.size} event(s) were: #{names(recorded)}.")
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Asserts that a custom event (`agentcat:custom`) was published: with
|
|
83
|
+
# `resource_name` when given, and matching the block when given.
|
|
84
|
+
sig do
|
|
85
|
+
params(
|
|
86
|
+
resource_name: T.nilable(String),
|
|
87
|
+
filter: T.nilable(T.proc.params(event: T::Hash[Symbol, T.untyped]).returns(T.untyped))
|
|
88
|
+
).void
|
|
89
|
+
end
|
|
90
|
+
def assert_custom_event_published(resource_name = nil, &filter)
|
|
91
|
+
custom = of_type(CUSTOM_EVENT)
|
|
92
|
+
what = describe("a custom event", resource_name, filter)
|
|
93
|
+
check(!matching(custom, resource_name, filter).empty?, "Expected #{what} to be published, but none was. Published: #{names(custom)}.")
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
private
|
|
97
|
+
|
|
98
|
+
sig { params(event_type: String).returns(T::Array[T::Hash[Symbol, T.untyped]]) }
|
|
99
|
+
def of_type(event_type)
|
|
100
|
+
events.select { |event| event[:event_type] == event_type }
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
sig do
|
|
104
|
+
params(
|
|
105
|
+
events: T::Array[T::Hash[Symbol, T.untyped]],
|
|
106
|
+
name: T.nilable(String),
|
|
107
|
+
filter: T.nilable(T.proc.params(event: T::Hash[Symbol, T.untyped]).returns(T.untyped))
|
|
108
|
+
).returns(T::Array[T::Hash[Symbol, T.untyped]])
|
|
109
|
+
end
|
|
110
|
+
def matching(events, name, filter)
|
|
111
|
+
events.select { |event| (name.nil? || event[:resource_name] == name) && (filter.nil? || filter.call(event)) }
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
sig { params(kind: String, name: T.nilable(String), filter: T.untyped).returns(String) }
|
|
115
|
+
def describe(kind, name, filter)
|
|
116
|
+
text = name ? "#{kind} named #{name.inspect}" : kind
|
|
117
|
+
filter ? "#{text} matching the block" : text
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
sig { params(events: T::Array[T::Hash[Symbol, T.untyped]]).returns(String) }
|
|
121
|
+
def names(events)
|
|
122
|
+
return "none" if events.empty?
|
|
123
|
+
|
|
124
|
+
events.map { |event| event[:resource_name].inspect }.join(", ")
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# Says why nothing may have been recorded: a server tracked while AgentCat was
|
|
128
|
+
# off is not tracked at all.
|
|
129
|
+
sig { params(message: String, tool_calls: T::Array[T.untyped]).returns(String) }
|
|
130
|
+
def hinted(message, tool_calls)
|
|
131
|
+
return message unless tool_calls.empty? && AgentCat.skipped_track?
|
|
132
|
+
|
|
133
|
+
"#{message} A server was tracked while AgentCat was off: call AgentCat.fake before AgentCat.track."
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
sig { params(condition: T::Boolean, message: String).void }
|
|
137
|
+
def check(condition, message)
|
|
138
|
+
return if condition
|
|
139
|
+
|
|
140
|
+
raise failure_class, message
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Looked up by name: neither framework is a dependency of the gem.
|
|
144
|
+
FAILURE_CLASS_NAMES = %w[Minitest::Assertion RSpec::Expectations::ExpectationNotMetError].freeze
|
|
145
|
+
|
|
146
|
+
sig { returns(T.class_of(Exception)) }
|
|
147
|
+
def failure_class
|
|
148
|
+
name = FAILURE_CLASS_NAMES.find { |candidate| Object.const_defined?(candidate) }
|
|
149
|
+
name ? Object.const_get(name) : AssertionError
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
end
|