@theoremjs/agents 0.3.4-next.679.gcdf1a3a → 0.3.4-next.711.g78e504a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -4
- package/esm/mod.d.ts +6 -3
- package/esm/mod.js +3 -1
- package/esm/src/cli/commands/bench.js +4 -10
- package/esm/src/cli/commands/profile.js +4 -1
- package/esm/src/guardrails/labels.d.ts +1 -1
- package/esm/src/guardrails/labels.js +9 -0
- package/esm/src/interface/blocks.js +19 -0
- package/esm/src/interface/from-profile.d.ts +2 -2
- package/esm/src/interface/from-profile.js +7 -0
- package/esm/src/interface/history.d.ts +2 -2
- package/esm/src/interface/history.js +7 -4
- package/esm/src/interface/mod.d.ts +2 -2
- package/esm/src/interface/mod.js +1 -1
- package/esm/src/interface/profile-interface.js +14 -4
- package/esm/src/interface/tool-calls.d.ts +3 -3
- package/esm/src/interface/tool-calls.js +5 -8
- package/esm/src/interface/tool-invoke.d.ts +3 -1
- package/esm/src/interface/tool-invoke.js +6 -1
- package/esm/src/interface/types.d.ts +7 -5
- package/esm/src/kernel/api-catalog.js +13 -5
- package/esm/src/kernel/auth/oauth.js +3 -1
- package/esm/src/kernel/engine/background.d.ts +7 -0
- package/esm/src/kernel/engine/background.js +18 -0
- package/esm/src/kernel/engine/runner/mod.js +24 -1
- package/esm/src/kernel/engine/runner/state.d.ts +4 -0
- package/esm/src/kernel/engine/runner/state.js +1 -0
- package/esm/src/kernel/engine/runner/step-order.d.ts +8 -0
- package/esm/src/kernel/engine/runner/step-order.js +1 -1
- package/esm/src/kernel/engine/runner/steps.js +1 -0
- package/esm/src/kernel/engine/session/mod.d.ts +2 -0
- package/esm/src/kernel/engine/session/mod.js +20 -2
- package/esm/src/kernel/engine/session/session-trace.js +1 -0
- package/esm/src/kernel/mod.d.ts +6 -3
- package/esm/src/kernel/mod.js +3 -1
- package/esm/src/kernel/profile-presence.js +1 -0
- package/esm/src/kernel/profile-scope.js +23 -21
- package/esm/src/kernel/protocols/protocol.d.ts +1 -1
- package/esm/src/kernel/protocols/protocol.js +1 -0
- package/esm/src/kernel/registry/catalog.js +4 -1
- package/esm/src/kernel/registry/ingress.d.ts +3 -1
- package/esm/src/kernel/registry/ingress.js +12 -1
- package/esm/src/kernel/registry/profile-outputs.js +2 -1
- package/esm/src/kernel/registry/profiles.d.ts +12 -2
- package/esm/src/kernel/registry/profiles.js +37 -7
- package/esm/src/kernel/registry/resolve.js +9 -5
- package/esm/src/kernel/registry/system-prompt.d.ts +6 -1
- package/esm/src/kernel/registry/system-prompt.js +20 -13
- package/esm/src/kernel/schema.d.ts +1 -1
- package/esm/src/kernel/schema.js +24 -8
- package/esm/src/kernel/system-lines.d.ts +22 -0
- package/esm/src/kernel/system-lines.js +162 -0
- package/esm/src/kernel/tools/activity-label.d.ts +15 -0
- package/esm/src/kernel/tools/activity-label.js +12 -7
- package/esm/src/kernel/tools/agent.js +1 -1
- package/esm/src/kernel/tools/background.d.ts +81 -0
- package/esm/src/kernel/tools/background.js +70 -0
- package/esm/src/kernel/tools/call-order.d.ts +18 -0
- package/esm/src/kernel/tools/call-order.js +52 -0
- package/esm/src/kernel/tools/coordinate.d.ts +8 -1
- package/esm/src/kernel/tools/coordinate.js +274 -34
- package/esm/src/kernel/tools/execute.d.ts +2 -0
- package/esm/src/kernel/tools/execute.js +17 -4
- package/esm/src/kernel/tools/invoke.js +40 -13
- package/esm/src/kernel/tools/memory-execution.js +10 -1
- package/esm/src/kernel/tools/operation.d.ts +21 -0
- package/esm/src/kernel/tools/operation.js +19 -10
- package/esm/src/kernel/tools/registry.js +28 -6
- package/esm/src/kernel/tools/resolve.d.ts +4 -2
- package/esm/src/kernel/tools/resolve.js +34 -7
- package/esm/src/kernel/tools/run.d.ts +2 -2
- package/esm/src/kernel/tools/run.js +4 -1
- package/esm/src/kernel/tools/stage-run.d.ts +3 -1
- package/esm/src/kernel/tools/stage-run.js +58 -55
- package/esm/src/kernel/tools/types.d.ts +10 -0
- package/esm/src/kernel/turn-events.d.ts +8 -1
- package/esm/src/kernel/turn-events.js +2 -0
- package/esm/src/kernel/types.d.ts +39 -4
- package/esm/src/observability/trace-catalog.js +3 -0
- package/esm/src/providers/google/interactions/steps.js +14 -3
- package/esm/src/providers/mod.d.ts +2 -0
- package/esm/src/providers/mod.js +1 -0
- package/esm/src/providers/open-ai/protocol.js +8 -4
- package/esm/src/providers/openrouter/protocol.d.ts +26 -0
- package/esm/src/providers/openrouter/protocol.js +58 -2
- package/esm/src/providers/openrouter/provider.d.ts +1 -1
- package/esm/src/providers/openrouter/provider.js +3 -1
- package/esm/src/providers/openrouter/transcriptions-protocol.d.ts +10 -0
- package/esm/src/providers/openrouter/transcriptions-protocol.js +87 -0
- package/package.json +1 -1
- package/esm/src/kernel/registry/system-role.d.ts +0 -3
- package/esm/src/kernel/registry/system-role.js +0 -9
|
@@ -8,6 +8,7 @@ import type { AuthUnauthenticatedPolicy, HttpMethod, ToolAccess, ToolAuthType, T
|
|
|
8
8
|
import type { RemovedItem, Source, ToolCallEdit, ToolCallEvent, ToolCallRequest, ToolFailure, ToolGate, ToolPhaseEvent, ToolTraceStep, ToolWarning, TurnToolSnapshot, WireFunctionTool } from '../turn-events.js';
|
|
9
9
|
import type { InteractionPart, ModelId, Profile, ProfileId, ToolId, TurnInput, TurnRequest, TurnSpeech, TurnTraceLink } from '../types.js';
|
|
10
10
|
import type { LabelTemplate } from './activity-label.js';
|
|
11
|
+
import type { ToolBackground, ToolBackgroundService } from './background.js';
|
|
11
12
|
import type { PageToolExecution, ToolExecutionCoordinator } from './operation.js';
|
|
12
13
|
import type { UncheckedOutput } from './unchecked-output.js';
|
|
13
14
|
export type { AuthUnauthenticatedPolicy, HttpMethod, ToolAccess, ToolAuthType, ToolCallEdit, ToolCallEvent, ToolCallRequest, ToolFailure, ToolGate, ToolPermission, ToolPhaseEvent, ToolTraceStep, ToolWarning, TurnToolSnapshot, WireFunctionTool, };
|
|
@@ -155,6 +156,12 @@ export interface ToolHostHooks<TIn = unknown> {
|
|
|
155
156
|
/** Runs before the host's `pre_tool` `onStage`. */
|
|
156
157
|
preTool?: (input: TIn, ctx: ToolContext) => import('../stages.js').StageResult | undefined | Promise<import('../stages.js').StageResult | undefined>;
|
|
157
158
|
exposeToModel?: boolean;
|
|
159
|
+
/** The model gets an `after` field: the tools in the same step that must settle before this call starts. */
|
|
160
|
+
after?: boolean;
|
|
161
|
+
/** When the model gets its reply. Unset is `never`: when the body ends. A turn without a background service runs inline. */
|
|
162
|
+
background?: ToolBackground;
|
|
163
|
+
/** `inputSchema` with the `background` argument. The registry sets it on an `ask` tool. */
|
|
164
|
+
askInputSchema?: Record<string, unknown>;
|
|
158
165
|
}
|
|
159
166
|
export interface ToolOutputHooks<TOut = unknown> {
|
|
160
167
|
/** A source that fails `sourceSchema`, or a throw, is a `sources_invalid` warning and is not cited. */
|
|
@@ -316,6 +323,8 @@ export interface AgentToolDef extends ToolBase {
|
|
|
316
323
|
/** Channels the profile takes that the calling model can't fill; only `onAgentCall` sends them. */
|
|
317
324
|
hostOnly?: AgentToolChannel[];
|
|
318
325
|
preTool?: ToolHostHooks<AgentToolInput>['preTool'];
|
|
326
|
+
after?: ToolHostHooks<AgentToolInput>['after'];
|
|
327
|
+
background?: ToolHostHooks<AgentToolInput>['background'];
|
|
319
328
|
exposeToModel?: boolean;
|
|
320
329
|
}
|
|
321
330
|
export interface RegisteredAgentTool extends AgentToolDef, ToolHostHooks<AgentToolInput>, ToolOutputHooks<AgentToolOutput> {
|
|
@@ -397,6 +406,7 @@ export type ToolPolicy = (ctx: ToolLoadContext) => ToolId[] | Promise<ToolId[]>;
|
|
|
397
406
|
/** A request to run a held tool call outside a turn. */
|
|
398
407
|
export interface InvokeToolRequest {
|
|
399
408
|
execution?: ToolExecutionCoordinator;
|
|
409
|
+
background?: ToolBackgroundService;
|
|
400
410
|
/** invariant: Only the trusted host restores the kernel-issued identity from saved authority. */
|
|
401
411
|
operationId?: string;
|
|
402
412
|
profile: string;
|
|
@@ -368,6 +368,8 @@ export interface TurnToolSnapshot {
|
|
|
368
368
|
path?: string;
|
|
369
369
|
sessionPermissions?: string[];
|
|
370
370
|
wire: WireFunctionTool[];
|
|
371
|
+
/** The request has a background service, so `ask` tools offer the `background` argument. */
|
|
372
|
+
modelBackground?: boolean;
|
|
371
373
|
}
|
|
372
374
|
/** The schema a tool snapshot parses against. */
|
|
373
375
|
export declare const turnToolSnapshotSchema: z.ZodType<TurnToolSnapshot>;
|
|
@@ -473,12 +475,17 @@ export type ToolPhaseEvent = ToolPhaseBase & ({
|
|
|
473
475
|
readBack?: string;
|
|
474
476
|
parts?: InteractionPart[];
|
|
475
477
|
activityPast?: string;
|
|
476
|
-
/** An agent tool's call: the called agent's own usage
|
|
478
|
+
/** An agent tool's call: the called agent's own usage. The caller's `done.tokens` includes it, unless the call settled after `done`. */
|
|
477
479
|
tokens?: TurnTokens;
|
|
478
480
|
} | {
|
|
479
481
|
phase: 'gate';
|
|
480
482
|
gate: ToolGate;
|
|
481
483
|
readBack?: string;
|
|
484
|
+
}
|
|
485
|
+
/** The model reads `readBack` now and the call goes on; its real end reaches the host alone. */
|
|
486
|
+
| {
|
|
487
|
+
phase: 'background';
|
|
488
|
+
readBack: string;
|
|
482
489
|
} | {
|
|
483
490
|
phase: 'error';
|
|
484
491
|
failure: ToolFailure;
|
|
@@ -246,6 +246,7 @@ const turnToolSnapshot = z.object({
|
|
|
246
246
|
path: z.string().optional(),
|
|
247
247
|
sessionPermissions: z.array(z.string()).optional(),
|
|
248
248
|
wire: z.array(wireFunctionTool),
|
|
249
|
+
modelBackground: z.boolean().optional(),
|
|
249
250
|
});
|
|
250
251
|
true;
|
|
251
252
|
/** The schema a tool snapshot parses against. */
|
|
@@ -299,6 +300,7 @@ const toolPhaseEvent = z.discriminatedUnion('phase', [
|
|
|
299
300
|
gate: toolGate,
|
|
300
301
|
readBack: z.string().optional(),
|
|
301
302
|
}),
|
|
303
|
+
z.object({ ...toolPhaseBase, phase: z.literal('background'), readBack: z.string() }),
|
|
302
304
|
z.object({
|
|
303
305
|
...toolPhaseBase,
|
|
304
306
|
phase: z.literal('error'),
|
|
@@ -2,6 +2,7 @@ import type { ProviderCheckpoint, ProviderContinuationPolicy } from './checkpoin
|
|
|
2
2
|
import type { JsonObject } from './json.js';
|
|
3
3
|
import type { CacheMode, CacheTtl, CompactionMeter, CompactionOutcome, CompactionTiming, ContextSender, ContinueStopKind, FieldMeta, KeySlot, KeyVault, LiveActivityHandling, LiveEndSensitivity, LiveStartSensitivity, MediaInputKind, ProfileType, SpeechAudioFormat, StreamMode, SummaryMode, ThinkingLevel, ToolLoadTier, TurnStage, TurnStopKind } from './schema.js';
|
|
4
4
|
import type { StageHandler } from './stages.js';
|
|
5
|
+
import type { ToolBackgroundService } from './tools/background.js';
|
|
5
6
|
import type { GateDecision } from './tools/gate-answer.js';
|
|
6
7
|
import type { PageToolExecution, ToolExecutionCoordinator } from './tools/operation.js';
|
|
7
8
|
import type { AgentCall, AgentCallHook, AgentCallRequest, AgentToolDef, AgentToolInput, AgentToolOutput, HostProfileToolsSpec, InvokeToolRequest, InvokeToolResume, InvokeToolStepCall, InvokeToolStepRequest, LiveProfileToolsSpec, ModelToolResult, PageAnswer, ProfileToolsSpec, RegisteredTool, ToolCallEvent, ToolFailure, ToolGate, ToolLoadContext, ToolPolicy, TurnToolSnapshot, WireFunctionTool } from './tools/types.js';
|
|
@@ -375,6 +376,12 @@ export type SystemPart = string | PrivateSystemPart;
|
|
|
375
376
|
* part is `{ private }`, the plain strings beside it are shareable.
|
|
376
377
|
*/
|
|
377
378
|
export type SystemPrompt = string | readonly SystemPart[];
|
|
379
|
+
/** One line of a profile's prompt: text, or parts kept together. A leading `when` decides it. */
|
|
380
|
+
export type SystemLine = string | readonly SystemPart[];
|
|
381
|
+
/** A profile's prompt: a string, or lines. Every line that fits is sent, a blank line apart. */
|
|
382
|
+
export type ProfileSystem = string | readonly SystemLine[];
|
|
383
|
+
/** The facts a host states on each turn, by name, each with the values it can take. */
|
|
384
|
+
export type ProfileConditions = Readonly<Record<string, readonly string[]>>;
|
|
378
385
|
/** A stretch of the system prompt as sent, and whether a reply may repeat it. */
|
|
379
386
|
export interface SystemPiece {
|
|
380
387
|
text: string;
|
|
@@ -383,8 +390,9 @@ export interface SystemPiece {
|
|
|
383
390
|
/** A profile's handle and system prompt. */
|
|
384
391
|
export interface ProfileIdentity {
|
|
385
392
|
handle: string;
|
|
386
|
-
|
|
387
|
-
|
|
393
|
+
/** The facts this agent's turns state, by name, with the values each can take. */
|
|
394
|
+
conditions?: ProfileConditions;
|
|
395
|
+
system?: ProfileSystem;
|
|
388
396
|
}
|
|
389
397
|
/** The fields every profile type shares. */
|
|
390
398
|
export interface ProfileCommon {
|
|
@@ -631,6 +639,21 @@ export interface SpeechProfile extends Omit<ProfileCommon, 'identity'> {
|
|
|
631
639
|
speech: ProfileSpeechSpec;
|
|
632
640
|
turnBehaviour?: MediaTurnBehaviourSpec;
|
|
633
641
|
}
|
|
642
|
+
/** A transcription profile's inputs: voice clips within byte limits, and no text. */
|
|
643
|
+
export type TranscriptionInputsSpec = Required<Pick<ProfileInputsSpec, 'voice'>> & Pick<ProfileInputsSpec, 'maxFiles' | 'maxBytes' | 'maxTurnBytes' | 'limitsByMime'>;
|
|
644
|
+
/**
|
|
645
|
+
* Speech to text: a voice clip in, what was said out as the turn's text. No tools, no system
|
|
646
|
+
* prompt, and no canary, because the model follows no instruction and has nothing of the profile's to leak.
|
|
647
|
+
*/
|
|
648
|
+
export interface TranscriptionProfile extends Omit<ProfileCommon, 'identity' | 'outputs' | 'maxSteps'> {
|
|
649
|
+
type: 'transcription';
|
|
650
|
+
identity: Pick<ProfileIdentity, 'handle'>;
|
|
651
|
+
inputs: TranscriptionInputsSpec & {
|
|
652
|
+
text: false;
|
|
653
|
+
};
|
|
654
|
+
outputs?: undefined;
|
|
655
|
+
maxSteps?: undefined;
|
|
656
|
+
}
|
|
634
657
|
/** A profile that holds a live audio session. */
|
|
635
658
|
export interface LiveProfile extends Omit<ProfileCommon, 'outputs'> {
|
|
636
659
|
type: 'live';
|
|
@@ -656,7 +679,7 @@ export interface HostProfile {
|
|
|
656
679
|
labels?: LabelOverrides;
|
|
657
680
|
}
|
|
658
681
|
/** The contract that defines an agent’s capabilities and the rules for its interactions. Its type determines which settings it accepts. */
|
|
659
|
-
export type Profile = TextProfile | ImageProfile | SpeechProfile | LiveProfile | DecisionProfile | EmbeddingProfile | HostProfile;
|
|
682
|
+
export type Profile = TextProfile | ImageProfile | SpeechProfile | TranscriptionProfile | LiveProfile | DecisionProfile | EmbeddingProfile | HostProfile;
|
|
660
683
|
/** Profiles that run a model turn — every type except `host` and `decision` (which runs through `runDecision`). */
|
|
661
684
|
export type ModelProfile = Exclude<Profile, HostProfile | DecisionProfile | EmbeddingProfile>;
|
|
662
685
|
/** The image a profile asks the model to return. */
|
|
@@ -699,7 +722,6 @@ export interface TurnRepairRequest {
|
|
|
699
722
|
/** What the user sent for a turn: text, slots, attachments, voice, and history. */
|
|
700
723
|
export interface TurnInput {
|
|
701
724
|
text?: string;
|
|
702
|
-
role?: string;
|
|
703
725
|
slots?: Record<string, string>;
|
|
704
726
|
/** What the page or the host wants the model to know, by sender; the profile's `inputs.context` allows it. */
|
|
705
727
|
context?: TurnContext;
|
|
@@ -738,6 +760,7 @@ export interface TurnTraceLink {
|
|
|
738
760
|
/** A request to `runTurn`: the profile, the input, and the host's hooks and keys. */
|
|
739
761
|
export interface TurnRequest {
|
|
740
762
|
execution?: ToolExecutionCoordinator;
|
|
763
|
+
background?: ToolBackgroundService;
|
|
741
764
|
pageExecution?: PageToolExecution;
|
|
742
765
|
providerState?: ProviderCheckpoint;
|
|
743
766
|
profile: ProfileId;
|
|
@@ -763,6 +786,8 @@ export interface TurnRequest {
|
|
|
763
786
|
sessionPermissions?: string[];
|
|
764
787
|
/** Host channel/path for catalog `paths` filtering. */
|
|
765
788
|
path?: string;
|
|
789
|
+
/** This turn's facts, one value for each of `profile.identity.conditions` but `path`. */
|
|
790
|
+
conditions?: Readonly<Record<string, string>>;
|
|
766
791
|
/** Host-owned metadata preserved for traces; the kernel does not interpret it. */
|
|
767
792
|
metadata?: Record<string, unknown>;
|
|
768
793
|
/**
|
|
@@ -797,6 +822,12 @@ export interface TurnRequest {
|
|
|
797
822
|
* Compared to `profile.turnBehaviour.resumption.maxContinues` when that cap is set.
|
|
798
823
|
*/
|
|
799
824
|
continuation?: number;
|
|
825
|
+
/**
|
|
826
|
+
* Tools this reply already had when an earlier run of it paused. The turn opens with them on its
|
|
827
|
+
* list, so an answered call does not take back what the model loaded. An id `tools.allow` or the
|
|
828
|
+
* path does not grant is left out, as for `tools.t1Policy`.
|
|
829
|
+
*/
|
|
830
|
+
promoted?: ToolId[];
|
|
800
831
|
input?: TurnInput;
|
|
801
832
|
/** Credentials for authenticated HTTP / MCP tools, read by auth slot when a tool needs one. */
|
|
802
833
|
credentials?: ToolCredentialSource;
|
|
@@ -893,6 +924,8 @@ export interface ResolvedGeneration extends ProviderGenerationConfig {
|
|
|
893
924
|
canary: string;
|
|
894
925
|
/** Snapshotted synchronously before any async work; the runner binds the canary on top. */
|
|
895
926
|
resolvedSystem: readonly SystemPiece[];
|
|
927
|
+
/** Which of the profile's prompt lines fit this turn, by number from 1. */
|
|
928
|
+
systemLines: readonly number[];
|
|
896
929
|
/** `TurnRequest.host`, carried to tool contexts only. Never sent to providers or traces. */
|
|
897
930
|
host?: unknown;
|
|
898
931
|
}
|
|
@@ -935,6 +968,8 @@ export interface SessionRequest {
|
|
|
935
968
|
/** As `TurnRequest.builtInArgs`. */
|
|
936
969
|
builtInArgs?: BuiltInArgs;
|
|
937
970
|
path?: string;
|
|
971
|
+
/** As `TurnRequest.conditions`. */
|
|
972
|
+
conditions?: Readonly<Record<string, string>>;
|
|
938
973
|
sessionPermissions?: string[];
|
|
939
974
|
history?: TurnHistoryMessage[];
|
|
940
975
|
/** The value chosen for each of the profile's `inputs.slots`. */
|
|
@@ -184,6 +184,7 @@ const OPERATIONS = {
|
|
|
184
184
|
label: 'Generate content',
|
|
185
185
|
doc: 'A model call that generates content other than over chat completions: Gemini, speech, or images.',
|
|
186
186
|
},
|
|
187
|
+
transcribe: { label: 'Transcribe', doc: 'One transcription call: a voice clip in, text out.' },
|
|
187
188
|
execute_tool: { label: 'Run tool', doc: 'A tool call theorem ran.' },
|
|
188
189
|
decide: {
|
|
189
190
|
label: 'Decide',
|
|
@@ -481,6 +482,7 @@ const SPAN_ATTRIBUTES = {
|
|
|
481
482
|
'theorem.steps': attr('agent', 'Model calls', 'number', 'Model calls made (Live: responses).'),
|
|
482
483
|
'theorem.turn.time_to_first_text': attr('agent', 'Time to first visible text', 'seconds', 'From the start of the turn to the first reply text the host received, after every guardrail and held tail.'),
|
|
483
484
|
'theorem.guardrail.stream_ms': attr('agent', 'Stream guardrail time', 'milliseconds', "Time every check on this call's streamed output took, together; each check's own time is on its `theorem.guardrail` event."),
|
|
485
|
+
'theorem.turn.system_lines': attr('agent', 'Prompt lines sent', 'list', "Which of the profile's prompt lines fit this turn, by number from 1."),
|
|
484
486
|
'theorem.step': attr('agent', 'Step', 'number', 'Which model call of the turn this is, from 1.'),
|
|
485
487
|
'theorem.attempt': attr('agent', 'Attempt', 'number', 'Which attempt of the turn made this call.'),
|
|
486
488
|
'theorem.request.effort': attr('agent', 'Effort', 'text', 'The effort the host asked for.'),
|
|
@@ -898,6 +900,7 @@ function traceSpanMeta(span) {
|
|
|
898
900
|
return withSubject(span.events.some((event) => event.name === 'theorem.session') ? 'session' : 'turn', stringAttribute(span, 'gen_ai.agent.name'));
|
|
899
901
|
case 'chat':
|
|
900
902
|
case 'generate_content':
|
|
903
|
+
case 'transcribe':
|
|
901
904
|
return withSubject('theorem.request.live' in span.attributes ? 'response' : 'call', stringAttribute(span, 'gen_ai.request.model'));
|
|
902
905
|
case 'execute_tool':
|
|
903
906
|
return withSubject('tool', stringAttribute(span, 'gen_ai.tool.name'));
|
|
@@ -15,6 +15,8 @@ import { byModality, groundingCounts, modalityCounts } from '../usage.js';
|
|
|
15
15
|
* `mime_type` + `data`. Audio also reports `sample_rate` / `channels`; the
|
|
16
16
|
* buffered mime already states them (`audio/l16; rate=24000; channels=1`),
|
|
17
17
|
* the streamed one does not (`audio/l16`), so they are folded into the mime.
|
|
18
|
+
* Probed 30/10/2026: gemini-3.1-flash-lite-image streams `image` deltas with
|
|
19
|
+
* `data` alone, no `mime_type`, so the bytes name their own type.
|
|
18
20
|
* - `code_execution_call` carries `id` and `arguments: { language, code }`;
|
|
19
21
|
* `code_execution_result` carries `call_id`, `result`, and `is_error`. Each
|
|
20
22
|
* arrives whole, in one delta (5 577 characters of code in one probe).
|
|
@@ -38,13 +40,22 @@ function interactionsMime(rec) {
|
|
|
38
40
|
}
|
|
39
41
|
return extra.length > 0 ? `${mime}; ${extra.join('; ')}` : mime;
|
|
40
42
|
}
|
|
43
|
+
/** How each image type Google returns begins, in base64. */
|
|
44
|
+
const IMAGE_MIME_BY_BASE64_START = [
|
|
45
|
+
['/9j/', 'image/jpeg'],
|
|
46
|
+
['iVBORw0KGgo', 'image/png'],
|
|
47
|
+
['UklGR', 'image/webp'],
|
|
48
|
+
];
|
|
49
|
+
function imageMimeOfBytes(data) {
|
|
50
|
+
return IMAGE_MIME_BY_BASE64_START.find(([start]) => data.startsWith(start))?.[1];
|
|
51
|
+
}
|
|
41
52
|
function interactionsMedia(rec) {
|
|
42
|
-
const mimeType = interactionsMime(rec);
|
|
43
53
|
const { data } = rec;
|
|
44
|
-
if (
|
|
54
|
+
if (typeof data !== 'string' || !data) {
|
|
45
55
|
return [];
|
|
46
56
|
}
|
|
47
|
-
|
|
57
|
+
const mimeType = interactionsMime(rec) ?? (rec.type === 'image' ? imageMimeOfBytes(data) : undefined);
|
|
58
|
+
return mimeType ? [{ type: 'media', media: { mimeType, data } }] : [];
|
|
48
59
|
}
|
|
49
60
|
function textEvent(type, text) {
|
|
50
61
|
return text ? [{ type, text }] : [];
|
|
@@ -29,5 +29,7 @@ export { openRouterEmbeddings } from './openrouter/embeddings-protocol.js';
|
|
|
29
29
|
export { openRouterImages } from './openrouter/images-protocol.js';
|
|
30
30
|
export { openRouterOpenAi } from './openrouter/protocol.js';
|
|
31
31
|
export { openRouterProvider } from './openrouter/provider.js';
|
|
32
|
+
export type { OpenRouterTranscriptionsOptions } from './openrouter/transcriptions-protocol.js';
|
|
33
|
+
export { openRouterTranscriptions } from './openrouter/transcriptions-protocol.js';
|
|
32
34
|
export { systemOne } from './system-one/protocol.js';
|
|
33
35
|
export { typesafeProvider } from './typesafe/provider.js';
|
package/esm/src/providers/mod.js
CHANGED
|
@@ -20,5 +20,6 @@ export { openRouterEmbeddings } from './openrouter/embeddings-protocol.js';
|
|
|
20
20
|
export { openRouterImages } from './openrouter/images-protocol.js';
|
|
21
21
|
export { openRouterOpenAi } from './openrouter/protocol.js';
|
|
22
22
|
export { openRouterProvider } from './openrouter/provider.js';
|
|
23
|
+
export { openRouterTranscriptions } from './openrouter/transcriptions-protocol.js';
|
|
23
24
|
export { systemOne } from './system-one/protocol.js';
|
|
24
25
|
export { typesafeProvider } from './typesafe/provider.js';
|
|
@@ -36,13 +36,17 @@ function requestBody(context, request) {
|
|
|
36
36
|
};
|
|
37
37
|
return JSON.parse(JSON.stringify(body));
|
|
38
38
|
}
|
|
39
|
-
/**
|
|
40
|
-
|
|
39
|
+
/**
|
|
40
|
+
* An error row inside a 200 reply: thrown, so a quota refusal before any output can still fall back.
|
|
41
|
+
* why: a reply the provider already began was accepted, so a 4xx in it is the stream breaking, not a refusal.
|
|
42
|
+
*/
|
|
43
|
+
function errorRow(row, reply) {
|
|
41
44
|
const error = asRecord(row.error);
|
|
42
45
|
if (!error)
|
|
43
46
|
return;
|
|
44
47
|
const status = typeof error.code === 'number' ? error.code : 502;
|
|
45
|
-
|
|
48
|
+
const kind = kindOfHttpStatus(status);
|
|
49
|
+
throw new TheoremError(reply.response && kind === 'unsupported' ? 'unavailable' : kind, statedError(row) ?? JSON.stringify(error));
|
|
46
50
|
}
|
|
47
51
|
function citations(part, reply) {
|
|
48
52
|
const sources = (Array.isArray(part.annotations) ? part.annotations : []).flatMap((item) => {
|
|
@@ -87,7 +91,7 @@ function partEvents(part, reply, request) {
|
|
|
87
91
|
return events;
|
|
88
92
|
}
|
|
89
93
|
function rowEvents(row, reply, request, stream) {
|
|
90
|
-
errorRow(row);
|
|
94
|
+
errorRow(row, reply);
|
|
91
95
|
const events = [];
|
|
92
96
|
const identity = foldResponse(reply.response, openAiResponse(row));
|
|
93
97
|
reply.response = identity.known;
|
|
@@ -1,8 +1,34 @@
|
|
|
1
|
+
import { type JsonObject } from '../../kernel/json.js';
|
|
1
2
|
import { type Protocol } from '../../kernel/protocols/mod.js';
|
|
2
3
|
import type { CacheSpec } from '../../kernel/types.js';
|
|
3
4
|
/** What an `openRouterOpenAi` binding sets in `options`. */
|
|
4
5
|
export interface OpenRouterOptions {
|
|
5
6
|
cache?: CacheSpec;
|
|
7
|
+
provider?: OpenRouterRouting;
|
|
8
|
+
}
|
|
9
|
+
/** OpenRouter's own `provider` request field, under its own names. */
|
|
10
|
+
export interface OpenRouterRouting {
|
|
11
|
+
only?: string[];
|
|
12
|
+
order?: string[];
|
|
13
|
+
ignore?: string[];
|
|
14
|
+
allow_fallbacks?: boolean;
|
|
15
|
+
require_parameters?: boolean;
|
|
16
|
+
data_collection?: 'allow' | 'deny';
|
|
17
|
+
zdr?: boolean;
|
|
18
|
+
enforce_distillable_text?: boolean;
|
|
19
|
+
quantizations?: string[];
|
|
20
|
+
sort?: string | {
|
|
21
|
+
by: string;
|
|
22
|
+
partition?: string;
|
|
23
|
+
};
|
|
24
|
+
preferred_min_throughput?: number | JsonObject;
|
|
25
|
+
preferred_max_latency?: number | JsonObject;
|
|
26
|
+
max_price?: {
|
|
27
|
+
prompt?: number;
|
|
28
|
+
completion?: number;
|
|
29
|
+
request?: number;
|
|
30
|
+
image?: number;
|
|
31
|
+
};
|
|
6
32
|
}
|
|
7
33
|
/** Chat Completions with OpenRouter's caching, routing, web search, citations, and reasoning. */
|
|
8
34
|
export declare const openRouterOpenAi: Protocol<OpenRouterOptions>;
|
|
@@ -17,6 +17,59 @@ const cache = z
|
|
|
17
17
|
.describe('How long the cache is kept. Absent: the upstream default.'),
|
|
18
18
|
})
|
|
19
19
|
.describe('Prompt caching, for models that support it.');
|
|
20
|
+
// labels-exempt: builder field description, read by the studio and docs
|
|
21
|
+
const routing = z
|
|
22
|
+
.strictObject({
|
|
23
|
+
only: z
|
|
24
|
+
.array(z.string())
|
|
25
|
+
.optional()
|
|
26
|
+
.describe('Upstream endpoints to allow, by slug, such as `gmicloud/fp8`.'),
|
|
27
|
+
order: z.array(z.string()).optional().describe('Upstream endpoints to try first, in order.'),
|
|
28
|
+
ignore: z.array(z.string()).optional().describe('Upstream endpoints to skip.'),
|
|
29
|
+
allow_fallbacks: z
|
|
30
|
+
.boolean()
|
|
31
|
+
.optional()
|
|
32
|
+
.describe('Whether another upstream may answer when the listed ones cannot.'),
|
|
33
|
+
require_parameters: z
|
|
34
|
+
.boolean()
|
|
35
|
+
.optional()
|
|
36
|
+
.describe('Whether every upstream must support every supplied parameter. Unset: on.'),
|
|
37
|
+
data_collection: z
|
|
38
|
+
.enum(['allow', 'deny'])
|
|
39
|
+
.optional()
|
|
40
|
+
.describe('Whether an upstream that may store and train on data can answer.'),
|
|
41
|
+
zdr: z.boolean().optional().describe('Use only endpoints with zero data retention.'),
|
|
42
|
+
enforce_distillable_text: z
|
|
43
|
+
.boolean()
|
|
44
|
+
.optional()
|
|
45
|
+
.describe('Use only models whose authors allow text distillation.'),
|
|
46
|
+
quantizations: z
|
|
47
|
+
.array(z.string())
|
|
48
|
+
.optional()
|
|
49
|
+
.describe('Quantization levels to allow, such as `fp8`.'),
|
|
50
|
+
sort: z
|
|
51
|
+
.union([z.string(), z.strictObject({ by: z.string(), partition: z.string().optional() })])
|
|
52
|
+
.optional()
|
|
53
|
+
.describe('What to order upstreams by: `price`, `throughput` or `latency`.'),
|
|
54
|
+
preferred_min_throughput: z
|
|
55
|
+
.union([z.number(), jsonObjectSchema])
|
|
56
|
+
.optional()
|
|
57
|
+
.describe('Tokens per second below which an upstream is tried later.'),
|
|
58
|
+
preferred_max_latency: z
|
|
59
|
+
.union([z.number(), jsonObjectSchema])
|
|
60
|
+
.optional()
|
|
61
|
+
.describe('Seconds above which an upstream is tried later.'),
|
|
62
|
+
max_price: z
|
|
63
|
+
.strictObject({
|
|
64
|
+
prompt: z.number().optional().describe('The most to pay per million input tokens.'),
|
|
65
|
+
completion: z.number().optional().describe('The most to pay per million output tokens.'),
|
|
66
|
+
request: z.number().optional().describe('The most to pay per request.'),
|
|
67
|
+
image: z.number().optional().describe('The most to pay per image.'),
|
|
68
|
+
})
|
|
69
|
+
.optional()
|
|
70
|
+
.describe('The prices above which an upstream is refused.'),
|
|
71
|
+
})
|
|
72
|
+
.describe('Which upstreams OpenRouter may route to. Absent: its own choice.');
|
|
20
73
|
/** What OpenRouter's reasoning comes back as, to be sent back word for word. */
|
|
21
74
|
const reasoning = z.strictObject({
|
|
22
75
|
details: z.array(jsonObjectSchema),
|
|
@@ -97,13 +150,16 @@ export const openRouterOpenAi = extendProtocol(openAi, {
|
|
|
97
150
|
id: 'openRouterOpenAi',
|
|
98
151
|
name: 'OpenRouter Chat Completions',
|
|
99
152
|
description: "Chat Completions with OpenRouter's caching, routing, web search, citations and reasoning.", // labels-exempt: builder field description, read by the studio and docs
|
|
100
|
-
options: z.strictObject({ cache: cache.optional() }),
|
|
153
|
+
options: z.strictObject({ cache: cache.optional(), provider: routing.optional() }),
|
|
101
154
|
reasoning,
|
|
102
155
|
body(body, at) {
|
|
103
156
|
// why: its fields are Chat Completions ones; a speech request goes as openAi sends it.
|
|
104
157
|
if (at.profileType !== 'text')
|
|
105
158
|
return { body };
|
|
106
|
-
const out = {
|
|
159
|
+
const out = {
|
|
160
|
+
...body,
|
|
161
|
+
provider: { require_parameters: true, ...at.options.provider },
|
|
162
|
+
};
|
|
107
163
|
if (out.reasoning_effort !== undefined) {
|
|
108
164
|
out.reasoning = { effort: out.reasoning_effort };
|
|
109
165
|
delete out.reasoning_effort;
|
|
@@ -6,5 +6,5 @@ export interface OpenRouterConnection {
|
|
|
6
6
|
siteUrl?: string;
|
|
7
7
|
siteName?: string;
|
|
8
8
|
}
|
|
9
|
-
/** OpenRouter: many labs' models behind one key. Chat, speech, images, and decisions. */
|
|
9
|
+
/** OpenRouter: many labs' models behind one key. Chat, speech, transcription, images, and decisions. */
|
|
10
10
|
export declare const openRouterProvider: Provider<OpenRouterConnection, string>;
|
|
@@ -4,6 +4,7 @@ import { openRouterDecisions } from './decisions-protocol.js';
|
|
|
4
4
|
import { openRouterEmbeddings } from './embeddings-protocol.js';
|
|
5
5
|
import { openRouterImages } from './images-protocol.js';
|
|
6
6
|
import { openRouterOpenAi } from './protocol.js';
|
|
7
|
+
import { openRouterTranscriptions } from './transcriptions-protocol.js';
|
|
7
8
|
const connection = z.strictObject({
|
|
8
9
|
baseUrl: endpointUrlSchema
|
|
9
10
|
.optional()
|
|
@@ -24,7 +25,7 @@ const connection = z.strictObject({
|
|
|
24
25
|
.optional()
|
|
25
26
|
.describe('Sent as X-Title, for OpenRouter rankings.'), // labels-exempt: builder field description, read by the studio and docs
|
|
26
27
|
});
|
|
27
|
-
/** OpenRouter: many labs' models behind one key. Chat, speech, images, and decisions. */
|
|
28
|
+
/** OpenRouter: many labs' models behind one key. Chat, speech, transcription, images, and decisions. */
|
|
28
29
|
export const openRouterProvider = defineProvider({
|
|
29
30
|
id: 'openrouter',
|
|
30
31
|
name: 'OpenRouter',
|
|
@@ -46,6 +47,7 @@ export const openRouterProvider = defineProvider({
|
|
|
46
47
|
{ protocol: openRouterOpenAi, baseUrl, headers },
|
|
47
48
|
{ protocol: openRouterEmbeddings, baseUrl, headers },
|
|
48
49
|
{ protocol: openRouterImages, baseUrl, headers },
|
|
50
|
+
{ protocol: openRouterTranscriptions, baseUrl, headers },
|
|
49
51
|
{ protocol: openRouterDecisions, baseUrl: decisionUrl, url: () => decisionUrl, headers },
|
|
50
52
|
];
|
|
51
53
|
},
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { type Protocol } from '../../kernel/protocols/mod.js';
|
|
2
|
+
/** What an `openRouterTranscriptions` binding sets in `options`. */
|
|
3
|
+
export interface OpenRouterTranscriptionsOptions {
|
|
4
|
+
provider?: {
|
|
5
|
+
zdr?: boolean;
|
|
6
|
+
data_collection?: 'allow' | 'deny';
|
|
7
|
+
};
|
|
8
|
+
}
|
|
9
|
+
/** OpenRouter's speech to text. Its body is its own JSON, not OpenAI's multipart upload. */
|
|
10
|
+
export declare const openRouterTranscriptions: Protocol<OpenRouterTranscriptionsOptions>;
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { TheoremError } from '../../guardrails/error.js';
|
|
3
|
+
import { defineProtocol, } from '../../kernel/protocols/mod.js';
|
|
4
|
+
import { openAiResponse, openAiUsageTokens } from './openai/usage.js';
|
|
5
|
+
// why: OpenRouter's transcription endpoint honours these two routing fields and ignores `only`, `order`, and `ignore`, so a binding cannot set one that does nothing.
|
|
6
|
+
// labels-exempt: builder field description, read by the studio and docs
|
|
7
|
+
const options = z.strictObject({
|
|
8
|
+
provider: z
|
|
9
|
+
.strictObject({
|
|
10
|
+
zdr: z.boolean().optional().describe('Use only endpoints with zero data retention.'),
|
|
11
|
+
data_collection: z
|
|
12
|
+
.enum(['allow', 'deny'])
|
|
13
|
+
.optional()
|
|
14
|
+
.describe('Whether an upstream that may store and train on data can answer.'),
|
|
15
|
+
})
|
|
16
|
+
.optional()
|
|
17
|
+
.describe('Which upstreams OpenRouter may route to. Absent: its own choice.'),
|
|
18
|
+
});
|
|
19
|
+
/** OpenRouter's word for each audio type it reads, as `input_audio.format`. */
|
|
20
|
+
const WIRE_FORMATS = {
|
|
21
|
+
'audio/wav': 'wav',
|
|
22
|
+
'audio/x-wav': 'wav',
|
|
23
|
+
'audio/wave': 'wav',
|
|
24
|
+
'audio/mpeg': 'mp3',
|
|
25
|
+
'audio/mp3': 'mp3',
|
|
26
|
+
'audio/mp4': 'm4a',
|
|
27
|
+
'audio/m4a': 'm4a',
|
|
28
|
+
'audio/x-m4a': 'm4a',
|
|
29
|
+
'audio/aac': 'aac',
|
|
30
|
+
'audio/ogg': 'ogg',
|
|
31
|
+
'audio/opus': 'ogg',
|
|
32
|
+
'audio/flac': 'flac',
|
|
33
|
+
'audio/webm': 'webm',
|
|
34
|
+
};
|
|
35
|
+
/** The turn's one clip as OpenRouter takes it: the bytes and their format. */
|
|
36
|
+
function inputAudio(request) {
|
|
37
|
+
const clips = request.input.filter((part) => part.type === 'audio');
|
|
38
|
+
const [clip] = clips;
|
|
39
|
+
if (clips.length !== 1 || !clip || !('data' in clip) || clip.data === undefined)
|
|
40
|
+
throw new TheoremError('request', 'a transcription takes exactly one inline clip'); // labels-exempt: developer contract error
|
|
41
|
+
const format = WIRE_FORMATS[clip.mimeType];
|
|
42
|
+
if (!format)
|
|
43
|
+
throw new TheoremError('request', `OpenRouter cannot transcribe ${clip.mimeType} audio`); // labels-exempt: developer contract error
|
|
44
|
+
return { data: clip.data, format };
|
|
45
|
+
}
|
|
46
|
+
/** `/audio/transcriptions`: one clip in, its text answered whole. */
|
|
47
|
+
async function* transcribe(context, request) {
|
|
48
|
+
const parsed = options.parse(context.options ?? {});
|
|
49
|
+
const body = {
|
|
50
|
+
model: context.apiId,
|
|
51
|
+
input_audio: inputAudio(request),
|
|
52
|
+
...(parsed.provider ? { provider: parsed.provider } : {}),
|
|
53
|
+
};
|
|
54
|
+
for await (const row of context.send(body, { stream: false })) {
|
|
55
|
+
if (typeof row.text !== 'string')
|
|
56
|
+
throw new TheoremError('bad_response', 'no text returned from transcription'); // labels-exempt: upstream contract error
|
|
57
|
+
const response = openAiResponse(row);
|
|
58
|
+
if (response)
|
|
59
|
+
yield { type: 'response', response };
|
|
60
|
+
const tokens = openAiUsageTokens(row.usage);
|
|
61
|
+
if (tokens)
|
|
62
|
+
yield { type: 'tokens', tokens };
|
|
63
|
+
// why: a clip of silence is a true answer with no words, not a failure.
|
|
64
|
+
if (row.text)
|
|
65
|
+
yield { type: 'text', text: row.text };
|
|
66
|
+
yield { type: 'done', stop: { kind: 'completed' } };
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/** OpenRouter's speech to text. Its body is its own JSON, not OpenAI's multipart upload. */
|
|
70
|
+
export const openRouterTranscriptions = defineProtocol({
|
|
71
|
+
id: 'openRouterTranscriptions',
|
|
72
|
+
name: 'OpenRouter Transcriptions',
|
|
73
|
+
description: 'OpenRouter speech-to-text models: a voice clip in, what was said out.', // labels-exempt: builder field description, read by the studio and docs
|
|
74
|
+
endpoints: { transcription: '/audio/transcriptions' },
|
|
75
|
+
operationNames: { transcription: 'transcribe' },
|
|
76
|
+
options,
|
|
77
|
+
// why: the endpoint names its answer only in this header (probed 2026-10-30), and the trace records it.
|
|
78
|
+
async *framing(response) {
|
|
79
|
+
const id = response.headers.get('x-generation-id')?.trim();
|
|
80
|
+
yield { ...(await response.json()), ...(id ? { id } : {}) };
|
|
81
|
+
},
|
|
82
|
+
create(context) {
|
|
83
|
+
return {
|
|
84
|
+
complete: (request) => transcribe(context, request),
|
|
85
|
+
};
|
|
86
|
+
},
|
|
87
|
+
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@theoremjs/agents",
|
|
3
|
-
"version": "0.3.4-next.
|
|
3
|
+
"version": "0.3.4-next.711.g78e504a",
|
|
4
4
|
"description": "A flat TypeScript agent kernel for typed profiles, deterministic turn execution, registered tools, provider adapters, guardrails, and host-injected traces.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"agent",
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
function pickSystemRole(profile, requested) {
|
|
2
|
-
const { handle } = profile.identity;
|
|
3
|
-
const systemByRole = profile.type === 'speech' ? undefined : profile.identity.systemByRole;
|
|
4
|
-
if (requested && systemByRole && Object.hasOwn(systemByRole, requested)) {
|
|
5
|
-
return requested;
|
|
6
|
-
}
|
|
7
|
-
return handle;
|
|
8
|
-
}
|
|
9
|
-
export { pickSystemRole };
|