@theoremjs/agents 0.3.5-next.953.gf255d6a → 0.3.5-next.959.g351c307
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/src/guardrails/labels.d.ts +1 -1
- package/esm/src/guardrails/labels.js +5 -1
- package/esm/src/guardrails/sanitize.d.ts +7 -1
- package/esm/src/guardrails/sanitize.js +11 -5
- package/esm/src/kernel/engine/runner/mod.js +8 -1
- package/esm/src/kernel/engine/runner/nested.js +7 -4
- package/esm/src/kernel/engine/runner/refused-media.d.ts +8 -1
- package/esm/src/kernel/engine/runner/refused-media.js +32 -1
- package/esm/src/kernel/engine/runner/state.d.ts +2 -0
- package/esm/src/kernel/registry/attachments.d.ts +25 -7
- package/esm/src/kernel/registry/attachments.js +45 -12
- package/esm/src/kernel/registry/catalog.d.ts +6 -2
- package/esm/src/kernel/registry/catalog.js +20 -3
- package/esm/src/kernel/registry/ingress.js +3 -3
- package/esm/src/kernel/schema.d.ts +3 -3
- package/esm/src/kernel/schema.js +5 -5
- package/esm/src/kernel/tools/stage-run.js +9 -3
- package/esm/src/kernel/turn-events.d.ts +8 -3
- package/esm/src/providers/google/api-error.js +3 -3
- package/esm/src/providers/openrouter/openai/compat.d.ts +2 -2
- package/esm/src/providers/openrouter/openai/compat.js +3 -2
- package/esm/src/providers/openrouter/protocol.js +35 -0
- package/package.json +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/** Values that fill a label template's `{param}` placeholders. */
|
|
2
2
|
export type LabelParams = Record<string, string | number>;
|
|
3
3
|
/** Keys are stable API. */
|
|
4
|
-
export declare const LABEL_KEYS: readonly ["provider.decision_failed", "provider.decision_unavailable", "provider.decision_model_missing", "provider.decision_ended", "provider.decision_images_off", "provider.decision_box_model", "provider.budget_exhausted", "provider.adapter_version", "provider.adapter_methods", "provider.adapter_schemas", "provider.fallback_slot", "provider.model_fallback_slot", "provider.host_model", "turn.media_refused_image", "turn.media_refused_video", "turn.media_refused_file", "turn.media_refused_tool", "provider.turn_profile", "provider.model_binding_missing", "provider.session_closed", "provider.settled_call", "provider.session_unavailable", "provider.live_input", "provider.tool_cancel_missing", "provider.kernel_gate", "provider.event_invalid", "provider.response_identity", "provider.credential_resolution", "provider.credential_shape", "provider.stored_continuation", "provider.continuation_mismatch", "provider.continuation_contract", "provider.stateless_history", "provider.events_after_completion", "provider.duplicate_call", "provider.tool_cancel_live", "provider.complete_unavailable", "provider.parallel_tools", "provider.binding_identity", "provider.profile_type", "provider.event_type", "provider.feature", "provider.input_kind", "provider.output_kind", "provider.builtin", "provider.credential_missing", "provider.profile_configuration", "continue.instruction", "canary.bind_note", "user_data.note", "context.note", "taint.blocked", "taint.reason_steered", "taint.reason_tainted", "taint.destination_blocked", "taint.destination_confirm", "advisory.notice_elevated", "advisory.notice_high", "advisory.guidance", "attachments.too_many_files", "attachments.too_many_of_kind", "attachments.file_too_large", "attachments.turn_too_large", "attachments.not_accepted", "attachments.mime_not_allowed", "attachments.limits_unconfigured", "quota.exhausted", "quota.busy", "error.config", "error.request", "error.conflict", "error.not_found", "error.input", "error.action", "error.auth", "error.rate_limit", "error.busy", "error.unsupported", "error.unavailable", "error.bad_response", "error.network", "error.timeout", "error.safety", "error.blocked", "error.declined", "error.failed", "error.cancelled", "error.internal", "repair.default_guidance", "repair.items_guidance", "repair.cut_off_guidance", "repair.reply_cut_off", "repair.call_cut_off", "repair.call_cut_off_guidance", "repair.nothing_written", "repair.nothing_written_guidance", "repair.prompt_header", "repair.prompt_intro", "repair.prompt_instructions", "repair.history_heading", "repair.section_previous_output", "repair.section_validator_rejection", "repair.section_repair_guidance", "repair.section_instructions", "structured.answer_tool", "structured.answer_missing", "structured.cut_off", "structured.not_json", "compaction.request", "compaction.tool_call", "compaction.tool_result", "compaction.tool_files", "decision.box_image", "detect.blocked", "detect.call_blocked", "detect.output_blocked", "detect.shown_blocked", "detect.hint.ids", "detect.hint.financial", "detect.hint.network", "detect.hint.credentials", "detect.hint.email", "detect.hint.phone", "detect.hint.bank_account", "detect.hint.injection", "detect.hint.canary_leak", "detect.hint.prompt_leak", "detect.hint.marker_leak", "detect.hint.ungiven_images", "detect.hint.ungiven_links", "detect.hint.tool_leak", "detect.hint.own", "egress.default_repair_guidance", "egress.refusal", "egress.refusal_cut", "egress.rejection", "egress.rejection_found", "egress.hint_unscannable", "egress.hint_provider_tool_leak", "thought.omitted_image", "thought.omitted_link", "thought.omitted_instructions", "session.abandon_gated", "session.tool_denied", "session.tool_aborted", "session.sign_in", "sign_in.link", "sign_in.pending", "sign_in.done", "sign_in.declined", "sign_in.expired", "sign_in.out_of_scope", "session.gate_expired", "session.provider_state_unowned", "session.conversation_unknown", "session.full", "session.busy", "session.turn_ended", "session.gate_pending", "session.part_skipped", "live.session_ended", "voice.unsupported", "voice.permission", "voice.unavailable", "voice.failed", "voice.empty", "stop.cancelled", "stop.interrupted", "stop.length", "stop.filtered", "stop.provider_error", "stop.stream_incomplete", "stop.step_limit", "stop.answer_missing", "composer.model", "composer.effort", "tool.awaiting_user", "tool.completed_hidden", "tool.t2_loader_needs_snapshot", "tool.t2_loader_shape", "tool.t2_loader_output_invalid", "tool.execution_uncertain", "tool.input_invalid", "tool.input_host_fields", "tool.hook_failed", "tool.input_invalid_after_mutate", "tool.output_invalid_after_mutate", "tool.handler_no_output", "tool.did_not_run", "tool.unreachable", "tool.page_no_answer", "tool.page_timed_out", "tool.output_invalid", "tool.not_loaded_t2", "tool.wait_unknown", "tool.wait_cycle", "tool.after_field", "tool.background_field", "tool.background_started", "tool.late_result", "tool.not_visible", "tool.builtin_not_enabled", "tool.provider_native", "tool.not_registered", "tool.builtin_needs_snapshot", "tool.not_allowed", "tool.not_eligible", "tool.unsupported_type", "tool.agent_call_limit", "tool.agent_failed", "tool.activity_failed", "protocol.not_served", "protocol.profile_type", "protocol.options", "protocol.option_clash", "protocol.base_options", "protocol.id_clash", "protocol.ws_headers", "protocol.builtin_unwritten", "protocol.provider_missing", "protocol.connection", "protocol.part_schema", "protocol.part_invalid", "protocol.key_invalid", "protocol.no_key", "protocol.reply_shape", "protocol.endpoint_url", "protocol.endpoint_host", "protocol.http", "protocol.no_operation", "protocol.definition", "protocol.provider_id_clash", "protocol.history_needed"];
|
|
4
|
+
export declare const LABEL_KEYS: readonly ["provider.decision_failed", "provider.decision_unavailable", "provider.decision_model_missing", "provider.decision_ended", "provider.decision_images_off", "provider.decision_box_model", "provider.budget_exhausted", "provider.adapter_version", "provider.adapter_methods", "provider.adapter_schemas", "provider.fallback_slot", "provider.model_fallback_slot", "provider.host_model", "turn.media_refused_image", "turn.media_refused_video", "turn.media_refused_file", "turn.media_refused_tool", "provider.turn_profile", "provider.model_binding_missing", "provider.session_closed", "provider.settled_call", "provider.session_unavailable", "provider.live_input", "provider.tool_cancel_missing", "provider.kernel_gate", "provider.event_invalid", "provider.response_identity", "provider.credential_resolution", "provider.credential_shape", "provider.stored_continuation", "provider.continuation_mismatch", "provider.continuation_contract", "provider.stateless_history", "provider.events_after_completion", "provider.duplicate_call", "provider.tool_cancel_live", "provider.complete_unavailable", "provider.parallel_tools", "provider.binding_identity", "provider.profile_type", "provider.event_type", "provider.feature", "provider.input_kind", "provider.output_kind", "provider.builtin", "provider.credential_missing", "provider.profile_configuration", "continue.instruction", "canary.bind_note", "user_data.note", "context.note", "taint.blocked", "taint.reason_steered", "taint.reason_tainted", "taint.destination_blocked", "taint.destination_confirm", "advisory.notice_elevated", "advisory.notice_high", "advisory.guidance", "attachments.too_many_files", "attachments.too_many_of_kind", "attachments.file_too_large", "attachments.turn_too_large", "attachments.not_accepted", "attachments.mime_not_allowed", "attachments.limits_unconfigured", "quota.exhausted", "quota.busy", "error.config", "error.request", "error.conflict", "error.not_found", "error.input", "error.action", "error.auth", "error.rate_limit", "error.busy", "error.unsupported", "error.unavailable", "error.bad_response", "error.network", "error.timeout", "error.safety", "error.blocked", "error.declined", "error.failed", "error.cancelled", "error.internal", "repair.default_guidance", "repair.items_guidance", "repair.cut_off_guidance", "repair.reply_cut_off", "repair.call_cut_off", "repair.call_cut_off_guidance", "repair.nothing_written", "repair.nothing_written_guidance", "repair.prompt_header", "repair.prompt_intro", "repair.prompt_instructions", "repair.history_heading", "repair.section_previous_output", "repair.section_validator_rejection", "repair.section_repair_guidance", "repair.section_instructions", "structured.answer_tool", "structured.answer_missing", "structured.cut_off", "structured.not_json", "compaction.request", "compaction.tool_call", "compaction.tool_result", "compaction.tool_files", "decision.box_image", "detect.blocked", "detect.call_blocked", "detect.output_blocked", "detect.shown_blocked", "detect.hint.ids", "detect.hint.financial", "detect.hint.network", "detect.hint.credentials", "detect.hint.email", "detect.hint.phone", "detect.hint.bank_account", "detect.hint.injection", "detect.hint.canary_leak", "detect.hint.prompt_leak", "detect.hint.marker_leak", "detect.hint.ungiven_images", "detect.hint.ungiven_links", "detect.hint.tool_leak", "detect.hint.own", "egress.default_repair_guidance", "egress.refusal", "egress.refusal_cut", "egress.rejection", "egress.rejection_found", "egress.hint_unscannable", "egress.hint_provider_tool_leak", "thought.omitted_image", "thought.omitted_link", "thought.omitted_instructions", "session.abandon_gated", "session.tool_denied", "session.tool_aborted", "session.sign_in", "sign_in.link", "sign_in.pending", "sign_in.done", "sign_in.declined", "sign_in.expired", "sign_in.out_of_scope", "session.gate_expired", "session.provider_state_unowned", "session.conversation_unknown", "session.full", "session.busy", "session.turn_ended", "session.gate_pending", "session.part_skipped", "live.session_ended", "voice.unsupported", "voice.permission", "voice.unavailable", "voice.failed", "voice.empty", "stop.cancelled", "stop.interrupted", "stop.length", "stop.filtered", "stop.provider_error", "stop.stream_incomplete", "stop.step_limit", "stop.answer_missing", "composer.model", "composer.effort", "tool.awaiting_user", "tool.completed_hidden", "tool.t2_loader_needs_snapshot", "tool.t2_loader_shape", "tool.t2_loader_output_invalid", "tool.execution_uncertain", "tool.input_invalid", "tool.input_host_fields", "tool.hook_failed", "tool.input_invalid_after_mutate", "tool.output_invalid_after_mutate", "tool.handler_no_output", "tool.did_not_run", "tool.unreachable", "tool.page_no_answer", "tool.page_timed_out", "tool.output_invalid", "tool.not_loaded_t2", "tool.wait_unknown", "tool.after_unknown", "tool.wait_cycle", "tool.after_field", "tool.background_field", "tool.background_started", "tool.late_result", "tool.not_visible", "tool.builtin_not_enabled", "tool.provider_native", "tool.not_registered", "tool.builtin_needs_snapshot", "tool.not_allowed", "tool.not_eligible", "tool.unsupported_type", "tool.agent_call_limit", "tool.agent_failed", "tool.activity_failed", "protocol.not_served", "protocol.profile_type", "protocol.options", "protocol.option_clash", "protocol.base_options", "protocol.id_clash", "protocol.ws_headers", "protocol.builtin_unwritten", "protocol.provider_missing", "protocol.connection", "protocol.part_schema", "protocol.part_invalid", "protocol.key_invalid", "protocol.no_key", "protocol.reply_shape", "protocol.endpoint_url", "protocol.endpoint_host", "protocol.http", "protocol.no_operation", "protocol.definition", "protocol.provider_id_clash", "protocol.history_needed"];
|
|
5
5
|
/** One of {@linkcode LABEL_KEYS}. */
|
|
6
6
|
export type LabelKey = (typeof LABEL_KEYS)[number];
|
|
7
7
|
/**
|
|
@@ -199,6 +199,7 @@ export const LABEL_KEYS = [
|
|
|
199
199
|
'tool.output_invalid',
|
|
200
200
|
'tool.not_loaded_t2',
|
|
201
201
|
'tool.wait_unknown',
|
|
202
|
+
'tool.after_unknown',
|
|
202
203
|
'tool.wait_cycle',
|
|
203
204
|
'tool.after_field',
|
|
204
205
|
'tool.background_field',
|
|
@@ -554,11 +555,12 @@ const DEFAULTS = {
|
|
|
554
555
|
'tool.output_invalid': 'Tool output validation failed',
|
|
555
556
|
'tool.not_loaded_t2': "Tool '{tool}' is not loaded — call {loader} to load it first",
|
|
556
557
|
'tool.wait_unknown': "Tool '{tool}' waits for calls that are not in this step: {calls}",
|
|
558
|
+
'tool.after_unknown': "Tool '{tool}' waits for tools you did not call in this step: {calls}. after takes the names of other tools you call in this same step, not call ids.",
|
|
557
559
|
'tool.wait_cycle': "Tool '{tool}' waits for calls that cannot finish before it: {calls}",
|
|
558
560
|
'tool.background_field': 'Set to true to get a reply now and let this call finish after the turn. Leave out to wait for its result.',
|
|
559
561
|
'tool.background_started': '{tool} has started and is still running. You do not have its result, so do not state or guess one, and do not call {tool} again. Tell the person it is under way and that the result comes later.',
|
|
560
562
|
'tool.late_result': 'The {tool} call you started earlier has finished. This is what it returned:',
|
|
561
|
-
'tool.after_field': '
|
|
563
|
+
'tool.after_field': 'Names of other tools you call in this same step, as search_web, that must finish before this call starts. Tool names only, not call ids. Leave out to set no order.',
|
|
562
564
|
'tool.not_visible': "Tool '{tool}' is not visible this turn",
|
|
563
565
|
'tool.builtin_not_enabled': "Builtin '{tool}' is not enabled this turn",
|
|
564
566
|
'tool.provider_native': "Tool '{tool}' is a provider builtin — execution is handled by the model provider, not the kernel",
|
|
@@ -786,6 +788,7 @@ export const LABEL_NOTES = {
|
|
|
786
788
|
'tool.output_invalid': "Told to the model when a tool's output fails the tool's output schema.",
|
|
787
789
|
'tool.not_loaded_t2': 'Told to the model when it calls a T2 tool the T2 loader has not loaded yet. Takes {tool} and {loader}, the loader tool.',
|
|
788
790
|
'tool.wait_unknown': "Told to the model when a call's pre_tool waitFor names calls that are not in its step. Takes {tool} and {calls}.",
|
|
791
|
+
'tool.after_unknown': "Told to the model when a call's after field names tools with no other call in its step, as a call id sent in place of a tool name. Takes {tool} and {calls}.",
|
|
789
792
|
'tool.wait_cycle': "Told to the model when a call's pre_tool waitFor names calls that cannot settle before it, because they wait for it or run after it. Takes {tool} and {calls}.",
|
|
790
793
|
'tool.background_field': "What the model reads on the background field of a tool that sets registerTool.background to ask. Read when the tool registers, so a profile's labels do not change it.",
|
|
791
794
|
'tool.background_started': 'Told to the model when a call goes on after its reply: the call runs in the background and its result comes later. Takes {tool}.',
|
|
@@ -874,6 +877,7 @@ const LABEL_PLACEHOLDERS = {
|
|
|
874
877
|
'tool.page_no_answer': TOOL,
|
|
875
878
|
'tool.not_loaded_t2': ['tool', 'loader'],
|
|
876
879
|
'tool.wait_unknown': ['tool', 'calls'],
|
|
880
|
+
'tool.after_unknown': ['tool', 'calls'],
|
|
877
881
|
'tool.wait_cycle': ['tool', 'calls'],
|
|
878
882
|
'tool.not_visible': TOOL,
|
|
879
883
|
'tool.builtin_not_enabled': TOOL,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type LeftOutFile } from '../kernel/registry/attachments.js';
|
|
1
2
|
import type { NormalizedTurnRequest, Profile, TurnContext, TurnEvent, TurnHistoryMessage, TurnRequest } from '../kernel/types.js';
|
|
2
3
|
import { type Boundary } from './boundaries.js';
|
|
3
4
|
import { type BoundaryReader } from './detect-at.js';
|
|
@@ -27,6 +28,8 @@ interface SanitizedTurnRequest {
|
|
|
27
28
|
events: TurnEvent[];
|
|
28
29
|
/** Set when a match blocks: the request does not reach the model, and the turn ends on this. */
|
|
29
30
|
refusal?: TheoremError;
|
|
31
|
+
/** The files the profile does not take, left out of `request`; the turn goes on without them. */
|
|
32
|
+
leftOut: LeftOutFile[];
|
|
30
33
|
}
|
|
31
34
|
/**
|
|
32
35
|
* `req.system` is host-assembled per turn — it interpolates retrieval and user
|
|
@@ -34,7 +37,10 @@ interface SanitizedTurnRequest {
|
|
|
34
37
|
* path and stays verbatim.
|
|
35
38
|
*/
|
|
36
39
|
declare function sanitizeTurnRequestWithEvents(req: TurnRequest, profile: Profile): SanitizedTurnRequest;
|
|
37
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* The turn request with its input cleaned under the profile's guardrails. Throws when a match
|
|
42
|
+
* blocks it, and when the profile does not take one of its files: nothing here goes on without it.
|
|
43
|
+
*/
|
|
38
44
|
declare function sanitizeTurnRequest(req: TurnRequest, profile: Profile): NormalizedTurnRequest;
|
|
39
45
|
export type { SanitizedTurnRequest };
|
|
40
46
|
export { contextText, requestRefused, sanitizeContext, sanitizeHistory, sanitizeProjectId, sanitizeTurnRequest, sanitizeTurnRequestWithEvents, };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { sanitizeTurnBlobs } from '../kernel/registry/attachments.js';
|
|
1
|
+
import { attachmentsRefused, sanitizeTurnBlobs, } from '../kernel/registry/attachments.js';
|
|
2
2
|
import { CONTEXT_SENDERS } from '../kernel/schema.js';
|
|
3
3
|
import { mapSystemPrompt } from '../kernel/system-parts.js';
|
|
4
4
|
import { recordOf } from './boundaries.js';
|
|
@@ -124,9 +124,9 @@ function sanitizeTurnRequestWithEvents(req, profile) {
|
|
|
124
124
|
const told = sanitizeContext(input.context, () => boundaryReader('context', detect));
|
|
125
125
|
const repair = sanitizeRepair(input.repair, at.repair);
|
|
126
126
|
const history = input.history ? sanitizeHistory(input.history, at.history) : undefined;
|
|
127
|
-
const { attachments, voice } = input.attachments?.length || input.voice?.length
|
|
127
|
+
const { attachments, voice, leftOut } = input.attachments?.length || input.voice?.length
|
|
128
128
|
? sanitizeTurnBlobs(profile, input.attachments, input.voice, at)
|
|
129
|
-
: input;
|
|
129
|
+
: { ...input, leftOut: [] };
|
|
130
130
|
const events = [];
|
|
131
131
|
let refusal;
|
|
132
132
|
for (const boundary of REQUEST_BOUNDARIES) {
|
|
@@ -159,13 +159,19 @@ function sanitizeTurnRequestWithEvents(req, profile) {
|
|
|
159
159
|
},
|
|
160
160
|
events,
|
|
161
161
|
...(refusal ? { refusal } : {}),
|
|
162
|
+
leftOut,
|
|
162
163
|
};
|
|
163
164
|
}
|
|
164
|
-
/**
|
|
165
|
+
/**
|
|
166
|
+
* The turn request with its input cleaned under the profile's guardrails. Throws when a match
|
|
167
|
+
* blocks it, and when the profile does not take one of its files: nothing here goes on without it.
|
|
168
|
+
*/
|
|
165
169
|
function sanitizeTurnRequest(req, profile) {
|
|
166
|
-
const { request, refusal } = sanitizeTurnRequestWithEvents(req, profile);
|
|
170
|
+
const { request, refusal, leftOut } = sanitizeTurnRequestWithEvents(req, profile);
|
|
167
171
|
if (refusal)
|
|
168
172
|
throw refusal;
|
|
173
|
+
if (leftOut.length > 0)
|
|
174
|
+
throw attachmentsRefused(leftOut.flatMap((file) => file.issues));
|
|
169
175
|
return request;
|
|
170
176
|
}
|
|
171
177
|
export { contextText, requestRefused, sanitizeContext, sanitizeHistory, sanitizeProjectId, sanitizeTurnRequest, sanitizeTurnRequestWithEvents, };
|
|
@@ -24,6 +24,7 @@ import { endThrownSpan, endTurnSpan, guardrailAttributes, guardrailCheckAttribut
|
|
|
24
24
|
import { sumEventTokens, sumTurnTokens } from '../usage.js';
|
|
25
25
|
import { runAttemptsWithValidation } from './gates.js';
|
|
26
26
|
import { createAgentCaller, NESTED_THROWS, turnError } from './nested.js';
|
|
27
|
+
import { leftOutNotes, leftOutReport, turnMediaRefused } from './refused-media.js';
|
|
27
28
|
import { applyTurnStage, landLateResultsAt, unreadLateResultsAt, } from './stages.js';
|
|
28
29
|
import { cutConversation, openTurnState, } from './state.js';
|
|
29
30
|
import { shouldSkipStreamEvent } from './stream.js';
|
|
@@ -265,6 +266,8 @@ async function* emitTurn(args) {
|
|
|
265
266
|
...(args.steers ? { steers: args.steers } : {}),
|
|
266
267
|
});
|
|
267
268
|
args.outState.state = state;
|
|
269
|
+
if (args.leftOut)
|
|
270
|
+
state.leftOut = args.leftOut;
|
|
268
271
|
const opening = { stage: 'pre_turn', step: 0 };
|
|
269
272
|
const late = yield* unreadLateResultsAt(state, profile, {
|
|
270
273
|
...opening,
|
|
@@ -305,10 +308,11 @@ async function* emitTurn(args) {
|
|
|
305
308
|
}
|
|
306
309
|
yield* runAttemptsWithValidation(safe, profile, generation, system, provider, state);
|
|
307
310
|
const tokens = sumTurnTokens(state.allEmittedEvents);
|
|
311
|
+
const mediaRefused = turnMediaRefused(state.leftOut, state.mediaRefused);
|
|
308
312
|
const done = turnDoneOf({
|
|
309
313
|
stop: state.lastStop ?? { kind: 'completed' },
|
|
310
314
|
steps: state.stepCount,
|
|
311
|
-
...(
|
|
315
|
+
...(mediaRefused ? { mediaRefused } : {}),
|
|
312
316
|
...(state.providerState &&
|
|
313
317
|
!cutConversation(state) &&
|
|
314
318
|
!('providerContinuation' in profile && profile.providerContinuation?.history === 'stateless') &&
|
|
@@ -336,6 +340,7 @@ async function* streamTurnEvents(ctx, profile, gen, provider, compactionSpec) {
|
|
|
336
340
|
background: ctx.background,
|
|
337
341
|
steers: ctx.steers,
|
|
338
342
|
outState: ctx,
|
|
343
|
+
leftOut: ctx.leftOut,
|
|
339
344
|
})) {
|
|
340
345
|
const crashes = [];
|
|
341
346
|
const attached = await maybeAttachAfter(event, ctx, gen, compactionSpec, crashes);
|
|
@@ -524,6 +529,8 @@ async function* runTurnBody(ctx, provider) {
|
|
|
524
529
|
if (sanitized.refusal)
|
|
525
530
|
throw sanitized.refusal;
|
|
526
531
|
const { profile, generation: gen } = resolveTurnInRegistry(ctx.registry, ctx.safe);
|
|
532
|
+
gen.input.push(...leftOutNotes(sanitized.leftOut, profile.labels));
|
|
533
|
+
ctx.leftOut = leftOutReport(sanitized.leftOut);
|
|
527
534
|
const t1Policy = await applyT1Policy(ctx.registry.tools, gen.tools, profile, ctx.safe);
|
|
528
535
|
if (t1Policy) {
|
|
529
536
|
yield deliver(ctx, hookFailed('tools.t1Policy', t1Policy.crashed, { span: ctx.root }));
|
|
@@ -2,6 +2,7 @@ import { errorCopyText, isAbortError, TheoremError, throwIfAborted, toErrorEvent
|
|
|
2
2
|
import { labelText } from '../../../guardrails/labels.js';
|
|
3
3
|
import { hookCrashFailure, hookFailed } from '../../hook-failure.js';
|
|
4
4
|
import { nestedTurnModel, resolveRegisteredTurnModel } from '../../provider-dispatch.js';
|
|
5
|
+
import { assertTurnAttachments } from '../../registry/attachments.js';
|
|
5
6
|
import { profileAllowsInject } from '../../stop.js';
|
|
6
7
|
import { agentProfileProblem, agentToolOutputOf, splitHostInput, } from '../../tools/agent.js';
|
|
7
8
|
import { findLast } from '../../util/find-last.js';
|
|
@@ -73,14 +74,15 @@ function createAgentCaller(args) {
|
|
|
73
74
|
throw new TheoremError('internal', `Agent tool "${tool.name}" ran without a span`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
|
|
74
75
|
}
|
|
75
76
|
const { speech, ...sent } = modelInput;
|
|
76
|
-
const
|
|
77
|
+
const turnInput = fields.input ?? sent;
|
|
78
|
+
const events = yield* nestedEvents(() => assertTurnAttachments(profile, turnInput.attachments, turnInput.voice), args.runNested({
|
|
77
79
|
req: {
|
|
78
80
|
...(speech ? { speech } : {}),
|
|
79
81
|
...fields,
|
|
80
82
|
profile: tool.profile,
|
|
81
83
|
execution: req.execution,
|
|
82
84
|
pageExecution: req.pageExecution,
|
|
83
|
-
input:
|
|
85
|
+
input: turnInput,
|
|
84
86
|
...(signal ? { signal } : {}),
|
|
85
87
|
...(req.onAgentCall ? { onAgentCall: req.onAgentCall } : {}),
|
|
86
88
|
},
|
|
@@ -95,16 +97,17 @@ function createAgentCaller(args) {
|
|
|
95
97
|
};
|
|
96
98
|
}
|
|
97
99
|
/** Relays an agent's turn and keeps its events; a turn refused before it starts ends in its error event. */
|
|
98
|
-
async function* nestedEvents(turn) {
|
|
100
|
+
async function* nestedEvents(check, turn) {
|
|
99
101
|
const events = [];
|
|
100
102
|
try {
|
|
103
|
+
check();
|
|
101
104
|
for await (const event of turn) {
|
|
102
105
|
events.push(event);
|
|
103
106
|
yield event;
|
|
104
107
|
}
|
|
105
108
|
}
|
|
106
109
|
catch (err) {
|
|
107
|
-
// why: A turn
|
|
110
|
+
// why: A file the agent does not take refuses its turn before it starts, where a person's turn goes on without it: the caller's model reads why and can correct the call.
|
|
108
111
|
if (!(err instanceof TheoremError) || NESTED_THROWS.has(err.kind) || isAbortError(err)) {
|
|
109
112
|
throw err;
|
|
110
113
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import type { LeftOutFile } from '../../registry/attachments.js';
|
|
1
2
|
import type { MediaRefused, RefusedMedia } from '../../turn-events.js';
|
|
2
|
-
import type { Profile, TurnHistoryMessage } from '../../types.js';
|
|
3
|
+
import type { InteractionPart, Profile, TurnHistoryMessage } from '../../types.js';
|
|
3
4
|
import type { StepExecutionState } from './state.js';
|
|
4
5
|
/** The files a retry took out, and how to put them back if the retry fails too. */
|
|
5
6
|
export interface RefusedMediaTry {
|
|
@@ -24,3 +25,9 @@ export declare function dropMedia(state: StepExecutionState, said: string, label
|
|
|
24
25
|
export declare function restoreMedia(state: StepExecutionState, tried: RefusedMediaTry): void;
|
|
25
26
|
/** What the host reads on `done`. */
|
|
26
27
|
export declare function refusedReport(tried: RefusedMediaTry): MediaRefused;
|
|
28
|
+
/** The note the model reads in place of each file the profile does not take. */
|
|
29
|
+
export declare function leftOutNotes(leftOut: readonly LeftOutFile[], labels: Profile['labels']): InteractionPart[];
|
|
30
|
+
/** The files the profile does not take, as `done.mediaRefused` reports them; `said` names each check that failed. */
|
|
31
|
+
export declare function leftOutReport(leftOut: readonly LeftOutFile[]): MediaRefused | undefined;
|
|
32
|
+
/** The turn's report: the files left out at ingress, then the ones the provider refused. */
|
|
33
|
+
export declare function turnMediaRefused(leftOut: MediaRefused | undefined, refused: MediaRefused | undefined): MediaRefused | undefined;
|
|
@@ -9,8 +9,12 @@ function holdsMedia(message) {
|
|
|
9
9
|
function typeKey(mimeType) {
|
|
10
10
|
return (mimeType.split(';')[0] ?? '').trim().toLowerCase();
|
|
11
11
|
}
|
|
12
|
+
/** invariant: a named `image/*` is every image, as a MIME wildcard is. */
|
|
12
13
|
function isNamed(part, named) {
|
|
13
|
-
|
|
14
|
+
if (!isMedia(part) || !('mimeType' in part))
|
|
15
|
+
return false;
|
|
16
|
+
const type = typeKey(part.mimeType);
|
|
17
|
+
return named.has(type) || named.has(`${type.split('/')[0] ?? ''}/*`);
|
|
14
18
|
}
|
|
15
19
|
function noteKey(mimeType, fromTool) {
|
|
16
20
|
if (fromTool)
|
|
@@ -88,3 +92,30 @@ export function restoreMedia(state, tried) {
|
|
|
88
92
|
export function refusedReport(tried) {
|
|
89
93
|
return { dropped: tried.dropped, said: tried.said };
|
|
90
94
|
}
|
|
95
|
+
/** The note the model reads in place of each file the profile does not take. */
|
|
96
|
+
export function leftOutNotes(leftOut, labels) {
|
|
97
|
+
return leftOut.map((file) => ({
|
|
98
|
+
type: 'text',
|
|
99
|
+
text: labelText(noteKey(file.mimeType.toLowerCase(), false), {}, labels),
|
|
100
|
+
}));
|
|
101
|
+
}
|
|
102
|
+
/** The files the profile does not take, as `done.mediaRefused` reports them; `said` names each check that failed. */
|
|
103
|
+
export function leftOutReport(leftOut) {
|
|
104
|
+
if (leftOut.length === 0)
|
|
105
|
+
return undefined;
|
|
106
|
+
return {
|
|
107
|
+
dropped: leftOut.map((file) => ({ mimeType: file.mimeType, from: 'input' })),
|
|
108
|
+
said: leftOut
|
|
109
|
+
.map((file) => `${file.mimeType}: ${file.issues.map((issue) => issue.code).join(', ')}`)
|
|
110
|
+
.join('\n'),
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
/** The turn's report: the files left out at ingress, then the ones the provider refused. */
|
|
114
|
+
export function turnMediaRefused(leftOut, refused) {
|
|
115
|
+
if (!leftOut || !refused)
|
|
116
|
+
return leftOut ?? refused;
|
|
117
|
+
return {
|
|
118
|
+
dropped: [...leftOut.dropped, ...refused.dropped],
|
|
119
|
+
said: `${leftOut.said}\n${refused.said}`,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
@@ -45,6 +45,8 @@ interface StepExecutionState {
|
|
|
45
45
|
/** The files this turn's refused calls went without: each type the provider named once, or every file once. */
|
|
46
46
|
mediaTry?: RefusedMediaTry;
|
|
47
47
|
mediaRefused?: MediaRefused;
|
|
48
|
+
/** The files the profile does not take, left out before the turn began. */
|
|
49
|
+
leftOut?: MediaRefused;
|
|
48
50
|
allEmittedEvents: TurnEvent[];
|
|
49
51
|
attemptEvents: TurnEvent[];
|
|
50
52
|
/**
|
|
@@ -36,7 +36,10 @@ export interface AttachmentRules {
|
|
|
36
36
|
voice?: string[];
|
|
37
37
|
limits?: MediaLimits;
|
|
38
38
|
}
|
|
39
|
-
/**
|
|
39
|
+
/**
|
|
40
|
+
* The one attachment check. The headless interface runs it before a send; at ingress the kernel
|
|
41
|
+
* runs it on each file, leaves out the files it names, and throws only on what names no file.
|
|
42
|
+
*/
|
|
40
43
|
declare function attachmentIssues(rules: AttachmentRules, files: AttachmentFacts[], clips: AttachmentFacts[]): AttachmentValidationIssue[];
|
|
41
44
|
/** The user-facing wording key and params for an attachment issue. */
|
|
42
45
|
declare function attachmentIssueCopy(issue: AttachmentValidationIssue): ErrorCopy;
|
|
@@ -46,14 +49,29 @@ declare function attachmentIssueText(issue: AttachmentValidationIssue, labels?:
|
|
|
46
49
|
declare function attachmentsRefused(issues: readonly AttachmentValidationIssue[]): TheoremError;
|
|
47
50
|
/** The decoded size of inline base64 data; throws when it is not base64. */
|
|
48
51
|
declare function base64Bytes(data: string): number;
|
|
49
|
-
/**
|
|
52
|
+
/** Throws on every issue the check finds: a host's own check before a send. */
|
|
50
53
|
declare function assertTurnAttachments(profile: Profile, attachments: Array<TurnBlob | TurnMediaRef> | undefined, voice: TurnBlob[] | undefined): void;
|
|
54
|
+
/** A file the turn goes without, and what the check found wrong with it. */
|
|
55
|
+
interface LeftOutFile {
|
|
56
|
+
mimeType: string;
|
|
57
|
+
channel: 'attachment' | 'voice';
|
|
58
|
+
issues: AttachmentValidationIssue[];
|
|
59
|
+
}
|
|
60
|
+
interface TurnFiles {
|
|
61
|
+
attachments?: Array<TurnBlob | TurnMediaRef>;
|
|
62
|
+
voice?: TurnBlob[];
|
|
63
|
+
leftOut: LeftOutFile[];
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* invariant: a file the profile does not take is left out and the turn goes on; what names no file
|
|
67
|
+
* (too many files, too many bytes in all) refuses the turn, since no one file is at fault.
|
|
68
|
+
* Provider references are checked for MIME and count; byte limits apply to inline blobs only.
|
|
69
|
+
*/
|
|
70
|
+
declare function splitTurnAttachments(profile: Profile, attachments: Array<TurnBlob | TurnMediaRef> | undefined, voice: TurnBlob[] | undefined): TurnFiles;
|
|
51
71
|
/** Provider file references pass through unsanitized: the kernel has no bytes to scan. */
|
|
52
72
|
declare function sanitizeTurnBlobs(profile: Profile, attachments: Array<TurnBlob | TurnMediaRef> | undefined, voice: TurnBlob[] | undefined, at: {
|
|
53
73
|
attachment: BoundaryReader;
|
|
54
74
|
voice: BoundaryReader;
|
|
55
|
-
}):
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
};
|
|
59
|
-
export { assertTurnAttachments, attachmentIssueCopy, attachmentIssues, attachmentIssueText, attachmentsRefused, base64Bytes, isTextFile, isTextFileData, isTurnMediaRef, maxBytesForMime, modelTextFile, requireMediaLimits, resolveMediaLimits, rewriteTextFile, sanitizeCsvText, sanitizeTurnBlobs, textFileText, };
|
|
75
|
+
}): TurnFiles;
|
|
76
|
+
export type { LeftOutFile, TurnFiles };
|
|
77
|
+
export { assertTurnAttachments, attachmentIssueCopy, attachmentIssues, attachmentIssueText, attachmentsRefused, base64Bytes, isTextFile, isTextFileData, isTurnMediaRef, maxBytesForMime, modelTextFile, requireMediaLimits, resolveMediaLimits, rewriteTextFile, sanitizeCsvText, sanitizeTurnBlobs, splitTurnAttachments, textFileText, };
|
|
@@ -2,7 +2,7 @@ import { TheoremError } from '../../guardrails/error.js';
|
|
|
2
2
|
import { labelText, } from '../../guardrails/labels.js';
|
|
3
3
|
import { base64ToBytes, bytesToBase64 } from '../util/base64.js';
|
|
4
4
|
import { mimeEssence } from '../util/mime.js';
|
|
5
|
-
import { mimeAllowed, profileAccept } from './catalog.js';
|
|
5
|
+
import { mediaKindForMime, mimeAllowed, profileAccept } from './catalog.js';
|
|
6
6
|
const B64_PAD = 2;
|
|
7
7
|
const B64_WORD = 4;
|
|
8
8
|
const B64_TRIPLET = 3;
|
|
@@ -254,7 +254,7 @@ function named(issue, name) {
|
|
|
254
254
|
}
|
|
255
255
|
function mimeIssues(accept, files, channel) {
|
|
256
256
|
return files
|
|
257
|
-
.filter((file) => !mimeAllowed(accept, file.mimeType))
|
|
257
|
+
.filter((file) => !mediaKindForMime(file.mimeType) || !mimeAllowed(accept, file.mimeType))
|
|
258
258
|
.map((file) => named({ code: 'mime_not_allowed', params: { mimeType: file.mimeType, channel } }, file.name));
|
|
259
259
|
}
|
|
260
260
|
function limitIssues(files, limits) {
|
|
@@ -277,7 +277,10 @@ function limitIssues(files, limits) {
|
|
|
277
277
|
}
|
|
278
278
|
return issues;
|
|
279
279
|
}
|
|
280
|
-
/**
|
|
280
|
+
/**
|
|
281
|
+
* The one attachment check. The headless interface runs it before a send; at ingress the kernel
|
|
282
|
+
* runs it on each file, leaves out the files it names, and throws only on what names no file.
|
|
283
|
+
*/
|
|
281
284
|
function attachmentIssues(rules, files, clips) {
|
|
282
285
|
const issues = [];
|
|
283
286
|
if (files.length > 0 && !rules.attachments) {
|
|
@@ -345,19 +348,46 @@ function factsOf(item) {
|
|
|
345
348
|
facts.sizeBytes = base64Bytes(item.data);
|
|
346
349
|
return facts;
|
|
347
350
|
}
|
|
348
|
-
/**
|
|
351
|
+
/** Throws on every issue the check finds: a host's own check before a send. */
|
|
349
352
|
function assertTurnAttachments(profile, attachments, voice) {
|
|
350
353
|
if (!hasTurnBlobs(attachments, voice))
|
|
351
354
|
return;
|
|
352
|
-
const limits = requireMediaLimits(profile);
|
|
353
355
|
const issues = attachmentIssues({
|
|
354
356
|
attachments: profileAccept(profile, 'attachments'),
|
|
355
357
|
voice: profileAccept(profile, 'voice'),
|
|
356
|
-
limits,
|
|
358
|
+
limits: requireMediaLimits(profile),
|
|
357
359
|
}, (attachments ?? []).map(factsOf), (voice ?? []).map(factsOf));
|
|
358
360
|
if (issues.length > 0)
|
|
359
361
|
throw attachmentsRefused(issues);
|
|
360
362
|
}
|
|
363
|
+
/**
|
|
364
|
+
* invariant: a file the profile does not take is left out and the turn goes on; what names no file
|
|
365
|
+
* (too many files, too many bytes in all) refuses the turn, since no one file is at fault.
|
|
366
|
+
* Provider references are checked for MIME and count; byte limits apply to inline blobs only.
|
|
367
|
+
*/
|
|
368
|
+
function splitTurnAttachments(profile, attachments, voice) {
|
|
369
|
+
if (!hasTurnBlobs(attachments, voice))
|
|
370
|
+
return { attachments, voice, leftOut: [] };
|
|
371
|
+
const rules = {
|
|
372
|
+
attachments: profileAccept(profile, 'attachments'),
|
|
373
|
+
voice: profileAccept(profile, 'voice'),
|
|
374
|
+
limits: requireMediaLimits(profile),
|
|
375
|
+
};
|
|
376
|
+
const leftOut = [];
|
|
377
|
+
const keep = (items, channel) => items?.filter((item) => {
|
|
378
|
+
const one = [factsOf(item)];
|
|
379
|
+
const issues = channel === 'voice' ? attachmentIssues(rules, [], one) : attachmentIssues(rules, one, []);
|
|
380
|
+
if (issues.length === 0)
|
|
381
|
+
return true;
|
|
382
|
+
leftOut.push({ mimeType: item.mimeType, channel, issues });
|
|
383
|
+
return false;
|
|
384
|
+
});
|
|
385
|
+
const kept = { attachments: keep(attachments, 'attachment'), voice: keep(voice, 'voice') };
|
|
386
|
+
const issues = attachmentIssues(rules, (kept.attachments ?? []).map(factsOf), (kept.voice ?? []).map(factsOf));
|
|
387
|
+
if (issues.length > 0)
|
|
388
|
+
throw attachmentsRefused(issues);
|
|
389
|
+
return { ...kept, leftOut };
|
|
390
|
+
}
|
|
361
391
|
function sanitizeAttachment(blob, reader) {
|
|
362
392
|
if (isTurnMediaRef(blob)) {
|
|
363
393
|
return blob;
|
|
@@ -380,12 +410,15 @@ function hasTurnBlobs(attachments, voice) {
|
|
|
380
410
|
}
|
|
381
411
|
/** Provider file references pass through unsanitized: the kernel has no bytes to scan. */
|
|
382
412
|
function sanitizeTurnBlobs(profile, attachments, voice, at) {
|
|
383
|
-
|
|
413
|
+
const kept = splitTurnAttachments(profile, attachments, voice);
|
|
384
414
|
return {
|
|
385
|
-
attachments: attachments?.length
|
|
386
|
-
? attachments.map((blob) => sanitizeAttachment(blob, at.attachment))
|
|
387
|
-
: attachments,
|
|
388
|
-
voice: voice?.length
|
|
415
|
+
attachments: kept.attachments?.length
|
|
416
|
+
? kept.attachments.map((blob) => sanitizeAttachment(blob, at.attachment))
|
|
417
|
+
: kept.attachments,
|
|
418
|
+
voice: kept.voice?.length
|
|
419
|
+
? kept.voice.map((blob) => sanitizeAttachment(blob, at.voice))
|
|
420
|
+
: kept.voice,
|
|
421
|
+
leftOut: kept.leftOut,
|
|
389
422
|
};
|
|
390
423
|
}
|
|
391
|
-
export { assertTurnAttachments, attachmentIssueCopy, attachmentIssues, attachmentIssueText, attachmentsRefused, base64Bytes, isTextFile, isTextFileData, isTurnMediaRef, maxBytesForMime, modelTextFile, requireMediaLimits, resolveMediaLimits, rewriteTextFile, sanitizeCsvText, sanitizeTurnBlobs, textFileText, };
|
|
424
|
+
export { assertTurnAttachments, attachmentIssueCopy, attachmentIssues, attachmentIssueText, attachmentsRefused, base64Bytes, isTextFile, isTextFileData, isTurnMediaRef, maxBytesForMime, modelTextFile, requireMediaLimits, resolveMediaLimits, rewriteTextFile, sanitizeCsvText, sanitizeTurnBlobs, splitTurnAttachments, textFileText, };
|
|
@@ -4,12 +4,16 @@ import { mimeEssence } from '../util/mime.js';
|
|
|
4
4
|
type MediaInputChannel = 'attachments' | 'voice';
|
|
5
5
|
/** Rules support a subtype wildcard such as `image/*`; parameter values are ignored. */
|
|
6
6
|
declare function mimeAllowed(accept: readonly string[], mime: string): boolean;
|
|
7
|
-
/**
|
|
7
|
+
/**
|
|
8
|
+
* The kind of media a MIME type is: its own row in `MEDIA_INPUT_KINDS`, else its top-level type
|
|
9
|
+
* (`image`, `audio`, `video`), else `document`. `undefined` only for a string that is not a MIME
|
|
10
|
+
* type. invariant: the profile's `accept` lists decide which types a turn takes, not this.
|
|
11
|
+
*/
|
|
8
12
|
declare function mediaKindForMime(mime: string): MediaInputKind | undefined;
|
|
9
13
|
declare function profileInputs(profile: Profile): ProfileInputsSpec | undefined;
|
|
10
14
|
declare function profileAccept(profile: Profile, channel: MediaInputChannel): string[] | undefined;
|
|
11
15
|
/**
|
|
12
|
-
* `undefined` when the profile accepts it nowhere or
|
|
16
|
+
* `undefined` when the profile accepts it nowhere or it is not a MIME type. Hosts route
|
|
13
17
|
* files with this instead of their own MIME table: the profile's `accept` lists are the whole declaration.
|
|
14
18
|
*/
|
|
15
19
|
declare function mediaChannelForMime(profile: Profile, mime: string): MediaInputChannel | undefined;
|
|
@@ -12,9 +12,26 @@ function mimeAllowed(accept, mime) {
|
|
|
12
12
|
return allowed === actual;
|
|
13
13
|
});
|
|
14
14
|
}
|
|
15
|
-
/**
|
|
15
|
+
/** RFC 6838: a type and a subtype, each a run of the characters a MIME name may use. */
|
|
16
|
+
const MIME_NAME = /^[a-z0-9][a-z0-9!#$&^_.+-]*\/[a-z0-9][a-z0-9!#$&^_.+-]*$/;
|
|
17
|
+
const KIND_OF_TYPE = {
|
|
18
|
+
image: 'image',
|
|
19
|
+
audio: 'audio',
|
|
20
|
+
video: 'video',
|
|
21
|
+
};
|
|
22
|
+
/**
|
|
23
|
+
* The kind of media a MIME type is: its own row in `MEDIA_INPUT_KINDS`, else its top-level type
|
|
24
|
+
* (`image`, `audio`, `video`), else `document`. `undefined` only for a string that is not a MIME
|
|
25
|
+
* type. invariant: the profile's `accept` lists decide which types a turn takes, not this.
|
|
26
|
+
*/
|
|
16
27
|
function mediaKindForMime(mime) {
|
|
17
|
-
|
|
28
|
+
const essence = mimeEssence(mime);
|
|
29
|
+
const known = MEDIA_INPUT_KINDS[essence];
|
|
30
|
+
if (known)
|
|
31
|
+
return known;
|
|
32
|
+
if (!MIME_NAME.test(essence))
|
|
33
|
+
return undefined;
|
|
34
|
+
return KIND_OF_TYPE[essence.split('/')[0] ?? ''] ?? 'document';
|
|
18
35
|
}
|
|
19
36
|
function profileInputs(profile) {
|
|
20
37
|
return profile.type === 'text' ||
|
|
@@ -29,7 +46,7 @@ function profileAccept(profile, channel) {
|
|
|
29
46
|
return channel === 'voice' ? inputs?.voice?.accept : inputs?.attachments?.accept;
|
|
30
47
|
}
|
|
31
48
|
/**
|
|
32
|
-
* `undefined` when the profile accepts it nowhere or
|
|
49
|
+
* `undefined` when the profile accepts it nowhere or it is not a MIME type. Hosts route
|
|
33
50
|
* files with this instead of their own MIME table: the profile's `accept` lists are the whole declaration.
|
|
34
51
|
*/
|
|
35
52
|
function mediaChannelForMime(profile, mime) {
|
|
@@ -81,11 +81,11 @@ function resolveImageFormat(profile) {
|
|
|
81
81
|
function assertMediaMime(mime) {
|
|
82
82
|
const kind = mediaKindForMime(mime);
|
|
83
83
|
if (!kind) {
|
|
84
|
-
throw new TheoremError('input', `
|
|
84
|
+
throw new TheoremError('input', `'${mime}' is not a MIME type`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
|
|
85
85
|
}
|
|
86
86
|
return kind;
|
|
87
87
|
}
|
|
88
|
-
/** Runs after `
|
|
88
|
+
/** Runs after `splitTurnAttachments`. A reference's uri passes untouched: the host owns the upload. */
|
|
89
89
|
function mediaParts(blobs) {
|
|
90
90
|
return blobs.map((blob) => {
|
|
91
91
|
const kind = assertMediaMime(blob.mimeType);
|
|
@@ -142,7 +142,7 @@ function pinnedReferenceParts(profile) {
|
|
|
142
142
|
? mediaParts(profile.image.references)
|
|
143
143
|
: [];
|
|
144
144
|
}
|
|
145
|
-
/** `sanitizeTurnRequest` has already
|
|
145
|
+
/** `sanitizeTurnRequest` has already left out the blobs the profile's limits and accept lists refuse. */
|
|
146
146
|
function extractMediaParts(profile, req) {
|
|
147
147
|
const files = req.input?.attachments ?? [];
|
|
148
148
|
const clips = req.input?.voice ?? [];
|
|
@@ -154,9 +154,9 @@ export type ToolType = (typeof TOOL_TYPES)[number];
|
|
|
154
154
|
/** A tool type the host defines, which is every type but `builtin`. */
|
|
155
155
|
export type CustomToolType = Exclude<ToolType, 'builtin'>;
|
|
156
156
|
/**
|
|
157
|
-
* The
|
|
158
|
-
*
|
|
159
|
-
*
|
|
157
|
+
* The MIME types the kernel names: the widest provider's documented lists (verified
|
|
158
|
+
* 2026-09-12) and their alias essences (`image/jpg`, `video/mov`, …). `mediaKindForMime`
|
|
159
|
+
* reads a type's kind here first; a type not here takes its kind from its top-level type.
|
|
160
160
|
* `resolveInputParts` canonicalizes `image/jpg` to `image/jpeg` on the wire.
|
|
161
161
|
*/
|
|
162
162
|
export declare const MEDIA_INPUT_KINDS: Record<string, MediaInputKind>;
|
package/esm/src/kernel/schema.js
CHANGED
|
@@ -334,9 +334,9 @@ export const STUDIO_AUTH_TYPES = [
|
|
|
334
334
|
...TOOL_AUTH_TYPES,
|
|
335
335
|
];
|
|
336
336
|
/**
|
|
337
|
-
* The
|
|
338
|
-
*
|
|
339
|
-
*
|
|
337
|
+
* The MIME types the kernel names: the widest provider's documented lists (verified
|
|
338
|
+
* 2026-09-12) and their alias essences (`image/jpg`, `video/mov`, …). `mediaKindForMime`
|
|
339
|
+
* reads a type's kind here first; a type not here takes its kind from its top-level type.
|
|
340
340
|
* `resolveInputParts` canonicalizes `image/jpg` to `image/jpeg` on the wire.
|
|
341
341
|
*/
|
|
342
342
|
export const MEDIA_INPUT_KINDS = {
|
|
@@ -718,9 +718,9 @@ export const PROFILE_FIELDS = withScopeAndPresence({
|
|
|
718
718
|
'inputs.maxStateBytes': field('number'),
|
|
719
719
|
'inputs.text': field('boolean'),
|
|
720
720
|
'inputs.attachments': field('{ accept: string[] }'),
|
|
721
|
-
'inputs.attachments.accept': field('string[]', ATTACHMENT_ACCEPT_MIMES,
|
|
721
|
+
'inputs.attachments.accept': field('string[]', ATTACHMENT_ACCEPT_MIMES, 'Common types; any MIME type can be listed.'),
|
|
722
722
|
'inputs.voice': field('{ accept: string[] }'),
|
|
723
|
-
'inputs.voice.accept': field('string[]', VOICE_ACCEPT_MIMES, '
|
|
723
|
+
'inputs.voice.accept': field('string[]', VOICE_ACCEPT_MIMES, 'Common audio types; any MIME type can be listed.'),
|
|
724
724
|
'inputs.maxFiles': field('number'),
|
|
725
725
|
'inputs.maxBytes': field('number'),
|
|
726
726
|
'inputs.maxTurnBytes': field('number'),
|
|
@@ -103,7 +103,7 @@ export async function* runPreToolPipeline(args) {
|
|
|
103
103
|
const reparsed = pre.mutated ? tool.input.safeParse(plainToolInput(pre.input)) : undefined;
|
|
104
104
|
if (reparsed && !reparsed.success)
|
|
105
105
|
return invalidAfterMutate(reparsed.error, ctx);
|
|
106
|
-
const waitFailure = await waitForStepCalls(stages, base.callId, tool.name, [...(pre.waitFor ?? []), ...after], ctx.profile.labels);
|
|
106
|
+
const waitFailure = await waitForStepCalls(stages, base.callId, tool.name, [...(pre.waitFor ?? []), ...after], ctx.profile.labels, names);
|
|
107
107
|
if (waitFailure)
|
|
108
108
|
return { ok: false, kind: 'failed', failure: waitFailure };
|
|
109
109
|
return { ok: true, input: reparsed ? reparsed.data : pre.input };
|
|
@@ -169,7 +169,7 @@ async function* runPreToolStages(args) {
|
|
|
169
169
|
return { ok: true, input, mutated: false, ...waitFor };
|
|
170
170
|
}
|
|
171
171
|
/** Holds the call for the calls its `pre_tool` named; a wait that cannot end fails the call. */
|
|
172
|
-
async function waitForStepCalls(stages, callId, toolName, waitFor, labels) {
|
|
172
|
+
async function waitForStepCalls(stages, callId, toolName, waitFor, labels, afterNames) {
|
|
173
173
|
if (waitFor.length === 0)
|
|
174
174
|
return undefined;
|
|
175
175
|
const outcome = stages?.wait
|
|
@@ -177,10 +177,16 @@ async function waitForStepCalls(stages, callId, toolName, waitFor, labels) {
|
|
|
177
177
|
: { ok: false, reason: 'unknown', callIds: [...waitFor] };
|
|
178
178
|
if (outcome.ok)
|
|
179
179
|
return undefined;
|
|
180
|
+
// why: An after name with no call in the step stays a name, so the model reads that after takes tool names.
|
|
181
|
+
const afterOnly = outcome.reason === 'unknown' && outcome.callIds.every((id) => afterNames.includes(id));
|
|
180
182
|
return {
|
|
181
183
|
code: outcome.reason === 'unknown' ? 'wait_unknown' : 'wait_cycle',
|
|
182
184
|
kind: 'request',
|
|
183
|
-
message: labelText(
|
|
185
|
+
message: labelText(afterOnly
|
|
186
|
+
? 'tool.after_unknown'
|
|
187
|
+
: outcome.reason === 'unknown'
|
|
188
|
+
? 'tool.wait_unknown'
|
|
189
|
+
: 'tool.wait_cycle', { tool: toolName, calls: outcome.callIds.join(', ') }, labels),
|
|
184
190
|
};
|
|
185
191
|
}
|
|
186
192
|
export async function* runPostToolStages(args) {
|
|
@@ -591,12 +591,17 @@ export interface RefusedMedia {
|
|
|
591
591
|
from: 'input' | 'history' | 'tool';
|
|
592
592
|
}
|
|
593
593
|
/**
|
|
594
|
-
* The turn went on without files the
|
|
595
|
-
*
|
|
594
|
+
* The turn went on without files: first those the profile does not take (its `accept` lists and
|
|
595
|
+
* byte limits), left out before any call, then those the provider refused. A refused file is
|
|
596
|
+
* listed only when the same call, sent without it, was accepted: that is the proof it was at fault.
|
|
596
597
|
*/
|
|
597
598
|
export interface MediaRefused {
|
|
598
599
|
dropped: RefusedMedia[];
|
|
599
|
-
/**
|
|
600
|
+
/**
|
|
601
|
+
* Why, for the builder only: `forClient` strips the whole report. A line per file left out
|
|
602
|
+
* names its type and each check it failed (`image/gif: mime_not_allowed`); then the provider's
|
|
603
|
+
* own reason.
|
|
604
|
+
*/
|
|
600
605
|
said: string;
|
|
601
606
|
}
|
|
602
607
|
export interface DoneBase {
|
|
@@ -82,10 +82,10 @@ function namedCodeError(code, message, options) {
|
|
|
82
82
|
? new TheoremError('bad_response', message)
|
|
83
83
|
: new TheoremError(kindOfHttpStatus(status), message, options);
|
|
84
84
|
}
|
|
85
|
-
/** probed 14/11/2026: a refused document names its type, and only the first refused one: "Unsupported mime_type 'text/plain' in document content." */
|
|
86
|
-
const
|
|
85
|
+
/** probed 14/11/2026: a refused document or image names its type, and only the first refused one: "Unsupported mime_type 'text/plain' in document content.", "… 'image/x-icon' in image content.". A refused audio or video part names none. */
|
|
86
|
+
const REFUSED_TYPE = /Unsupported mime_type '([^']+)' in \w+ content/;
|
|
87
87
|
function refusedMedia(message) {
|
|
88
|
-
const type = typeof message === 'string' ?
|
|
88
|
+
const type = typeof message === 'string' ? REFUSED_TYPE.exec(message)?.[1] : undefined;
|
|
89
89
|
return type ? { refusedMedia: [type] } : undefined;
|
|
90
90
|
}
|
|
91
91
|
/**
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type ProducedError, TheoremError } from '../../../guardrails/error.js';
|
|
1
|
+
import { type ProducedError, TheoremError, type TheoremErrorOptions } from '../../../guardrails/error.js';
|
|
2
2
|
import type { InteractionPart, ModelRequest, ResolvedStructured, WireFunctionTool } from '../../../kernel/types.js';
|
|
3
3
|
/**
|
|
4
4
|
* Why a Chat Completions body says the call failed: its `error.message`, then the provider's own
|
|
@@ -9,7 +9,7 @@ declare function statedError(body: unknown): string | undefined;
|
|
|
9
9
|
declare function httpError(failure: {
|
|
10
10
|
status: number;
|
|
11
11
|
body: unknown;
|
|
12
|
-
}, label: string): TheoremError;
|
|
12
|
+
}, label: string, options?: TheoremErrorOptions): TheoremError;
|
|
13
13
|
declare function httpErrorEvent(res: Response, label: string): Promise<ProducedError>;
|
|
14
14
|
interface GatewayHeaderConfig {
|
|
15
15
|
siteUrl?: string;
|
|
@@ -21,10 +21,10 @@ function statedError(body) {
|
|
|
21
21
|
return [...said, ...more].join(' ') || JSON.stringify(error);
|
|
22
22
|
}
|
|
23
23
|
/** A refused Chat Completions call, its kind from the status and its stated reason in the message. */
|
|
24
|
-
function httpError(failure, label) {
|
|
24
|
+
function httpError(failure, label, options) {
|
|
25
25
|
const detail = statedError(failure.body);
|
|
26
26
|
const head = `${label} HTTP ${String(failure.status)}`;
|
|
27
|
-
return new TheoremError(kindOfHttpStatus(failure.status), detail ? `${head}: ${detail}` : head);
|
|
27
|
+
return new TheoremError(kindOfHttpStatus(failure.status), detail ? `${head}: ${detail}` : head, options);
|
|
28
28
|
}
|
|
29
29
|
async function httpErrorEvent(res, label) {
|
|
30
30
|
const text = (await res.text()).trim();
|
|
@@ -42,6 +42,7 @@ function rejectMediaRef(part) {
|
|
|
42
42
|
throw new TheoremError('unsupported', 'media references are not supported on openAi');
|
|
43
43
|
}
|
|
44
44
|
}
|
|
45
|
+
/** probed 14/11/2026: Chat Completions takes only `wav` and `mp3` as `format`, and refuses `flac` or `ogg`; an upstream reads the clip's real format from its bytes, so flac and ogg sent as `mp3` are heard. */
|
|
45
46
|
function wireAudioPart(part) {
|
|
46
47
|
let format = 'mp3';
|
|
47
48
|
if (part.mimeType.includes('wav')) {
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
+
import { kindOfHttpStatus, TheoremError } from '../../guardrails/error.js';
|
|
2
3
|
import { asRecord } from '../../kernel/engine/record.js';
|
|
3
4
|
import { jsonObjectSchema } from '../../kernel/json.js';
|
|
4
5
|
import { extendProtocol } from '../../kernel/protocols/mod.js';
|
|
5
6
|
import { openAi } from '../open-ai/protocol.js';
|
|
7
|
+
import { httpError, statedError } from './openai/compat.js';
|
|
6
8
|
import { collectReasoning } from './openai/reasoning-state.js';
|
|
7
9
|
/** What is cached: the whole request (`automatic`) or the system instruction only (`system`). */
|
|
8
10
|
export const CACHE_MODES = ['automatic', 'system'];
|
|
@@ -78,6 +80,33 @@ const reasoning = z.strictObject({
|
|
|
78
80
|
details: z.array(jsonObjectSchema),
|
|
79
81
|
content: z.string().optional(),
|
|
80
82
|
});
|
|
83
|
+
/**
|
|
84
|
+
* probed 14/11/2026: with `require_parameters`, a model that cannot read a kind of file answers
|
|
85
|
+
* 404 "No endpoints found that support image input", its `metadata.failed_routing_step` naming
|
|
86
|
+
* the kind: "Filter by Image Support", "Filter by Input Audio Support", "Filter by Input Video
|
|
87
|
+
* Support". Every file of that kind is refused, whatever its type.
|
|
88
|
+
*/
|
|
89
|
+
const REFUSED_STEP = /^Filter by (?:Input )?(Image|Audio|Video) Support$/;
|
|
90
|
+
/** probed 14/11/2026: an upstream that cannot decode a clip says so in a 200 stream's error row: "Upstream error from Nvidia: Unsupported audio format: send WAV, MP3, FLAC or Ogg (Vorbis/Opus) audio." */
|
|
91
|
+
const REFUSED_FORMAT = /\bUnsupported (audio|image|video) format\b/i;
|
|
92
|
+
function refusedKind(body) {
|
|
93
|
+
const error = asRecord(asRecord(body)?.error);
|
|
94
|
+
const step = asRecord(error?.metadata)?.failed_routing_step;
|
|
95
|
+
const kind = (typeof step === 'string' ? REFUSED_STEP.exec(step)?.[1] : undefined) ??
|
|
96
|
+
(typeof error?.message === 'string' ? REFUSED_FORMAT.exec(error.message)?.[1] : undefined);
|
|
97
|
+
return kind ? { refusedMedia: [`${kind.toLowerCase()}/*`] } : undefined;
|
|
98
|
+
}
|
|
99
|
+
/** invariant: an error row is typed as the base types it; this only names the kind of file it refused. */
|
|
100
|
+
function refusedRow(row) {
|
|
101
|
+
const error = asRecord(row.error);
|
|
102
|
+
if (!error)
|
|
103
|
+
return undefined;
|
|
104
|
+
const kind = kindOfHttpStatus(typeof error.code === 'number' ? error.code : 502);
|
|
105
|
+
const refused = kind === 'unsupported' ? refusedKind(row) : undefined;
|
|
106
|
+
return refused
|
|
107
|
+
? new TheoremError(kind, statedError(row) ?? JSON.stringify(error), refused)
|
|
108
|
+
: undefined;
|
|
109
|
+
}
|
|
81
110
|
function cacheControl(spec) {
|
|
82
111
|
return spec.ttl ? { type: 'ephemeral', ttl: spec.ttl } : { type: 'ephemeral' };
|
|
83
112
|
}
|
|
@@ -155,6 +184,7 @@ export const openRouterOpenAi = extendProtocol(openAi, {
|
|
|
155
184
|
description: "Chat Completions with OpenRouter's caching, routing, web search, citations and reasoning.", // labels-exempt: builder field description, read by the studio and docs
|
|
156
185
|
options: z.strictObject({ cache: cache.optional(), provider: routing.optional() }),
|
|
157
186
|
reasoning,
|
|
187
|
+
error: (failure) => httpError(failure, 'Chat Completions', refusedKind(failure.body)),
|
|
158
188
|
body(body, at) {
|
|
159
189
|
// why: its fields are Chat Completions ones; a speech request goes as openAi sends it.
|
|
160
190
|
if (at.profileType !== 'text')
|
|
@@ -192,8 +222,13 @@ export const openRouterOpenAi = extendProtocol(openAi, {
|
|
|
192
222
|
const cited = new Set();
|
|
193
223
|
const reported = new Set();
|
|
194
224
|
let content = '';
|
|
225
|
+
let begun = false;
|
|
195
226
|
return {
|
|
196
227
|
row(row) {
|
|
228
|
+
const refused = begun ? undefined : refusedRow(row);
|
|
229
|
+
if (refused)
|
|
230
|
+
throw refused;
|
|
231
|
+
begun = true;
|
|
197
232
|
content += collectReasoning(row, details);
|
|
198
233
|
const events = [];
|
|
199
234
|
const thought = thinking(row);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@theoremjs/agents",
|
|
3
|
-
"version": "0.3.5-next.
|
|
3
|
+
"version": "0.3.5-next.959.g351c307",
|
|
4
4
|
"description": "A flat TypeScript agent kernel for typed profiles, deterministic turn execution, registered tools, provider adapters, guardrails, and host-injected traces.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"agent",
|