@open-everywhere/adapters 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent/directory/okta.ts +41 -2
- package/agent/meetings/native-http.ts +72 -0
- package/agent/meetings/native-meet.ts +173 -0
- package/agent/meetings/native-sources.ts +14 -0
- package/agent/meetings/native-teams.ts +119 -0
- package/agent/meetings/native-zoom.ts +98 -0
- package/agent/model/gateway.ts +44 -4
- package/agent/model/meeting-gateway.ts +88 -0
- package/agent/model/transcription-routes.ts +347 -0
- package/agent/oe-oauth/config.ts +71 -3
- package/agent/oe-oauth/errors.ts +17 -0
- package/agent/oe-oauth/oauth-client.ts +118 -2
- package/agent/oe-oauth/token-vault.ts +359 -8
- package/agent/register-defaults.ts +11 -0
- package/agent/sandbox/long-lived-box.ts +464 -0
- package/dist/agent/directory/okta.d.ts +10 -0
- package/dist/agent/directory/okta.d.ts.map +1 -1
- package/dist/agent/directory/okta.js +34 -2
- package/dist/agent/directory/okta.js.map +1 -1
- package/dist/agent/meetings/native-http.d.ts +22 -0
- package/dist/agent/meetings/native-http.d.ts.map +1 -0
- package/dist/agent/meetings/native-http.js +61 -0
- package/dist/agent/meetings/native-http.js.map +1 -0
- package/dist/agent/meetings/native-meet.d.ts +6 -0
- package/dist/agent/meetings/native-meet.d.ts.map +1 -0
- package/dist/agent/meetings/native-meet.js +176 -0
- package/dist/agent/meetings/native-meet.js.map +1 -0
- package/dist/agent/meetings/native-sources.d.ts +2 -0
- package/dist/agent/meetings/native-sources.d.ts.map +1 -0
- package/dist/agent/meetings/native-sources.js +13 -0
- package/dist/agent/meetings/native-sources.js.map +1 -0
- package/dist/agent/meetings/native-teams.d.ts +6 -0
- package/dist/agent/meetings/native-teams.d.ts.map +1 -0
- package/dist/agent/meetings/native-teams.js +118 -0
- package/dist/agent/meetings/native-teams.js.map +1 -0
- package/dist/agent/meetings/native-zoom.d.ts +7 -0
- package/dist/agent/meetings/native-zoom.d.ts.map +1 -0
- package/dist/agent/meetings/native-zoom.js +98 -0
- package/dist/agent/meetings/native-zoom.js.map +1 -0
- package/dist/agent/model/gateway.d.ts +10 -0
- package/dist/agent/model/gateway.d.ts.map +1 -1
- package/dist/agent/model/gateway.js +43 -5
- package/dist/agent/model/gateway.js.map +1 -1
- package/dist/agent/model/meeting-gateway.d.ts +24 -0
- package/dist/agent/model/meeting-gateway.d.ts.map +1 -0
- package/dist/agent/model/meeting-gateway.js +75 -0
- package/dist/agent/model/meeting-gateway.js.map +1 -0
- package/dist/agent/model/transcription-routes.d.ts +89 -0
- package/dist/agent/model/transcription-routes.d.ts.map +1 -0
- package/dist/agent/model/transcription-routes.js +279 -0
- package/dist/agent/model/transcription-routes.js.map +1 -0
- package/dist/agent/oe-oauth/config.d.ts +28 -2
- package/dist/agent/oe-oauth/config.d.ts.map +1 -1
- package/dist/agent/oe-oauth/config.js +40 -1
- package/dist/agent/oe-oauth/config.js.map +1 -1
- package/dist/agent/oe-oauth/errors.d.ts +14 -0
- package/dist/agent/oe-oauth/errors.d.ts.map +1 -1
- package/dist/agent/oe-oauth/errors.js +15 -0
- package/dist/agent/oe-oauth/errors.js.map +1 -1
- package/dist/agent/oe-oauth/oauth-client.d.ts +46 -0
- package/dist/agent/oe-oauth/oauth-client.d.ts.map +1 -1
- package/dist/agent/oe-oauth/oauth-client.js +82 -1
- package/dist/agent/oe-oauth/oauth-client.js.map +1 -1
- package/dist/agent/oe-oauth/token-vault.d.ts +15 -0
- package/dist/agent/oe-oauth/token-vault.d.ts.map +1 -1
- package/dist/agent/oe-oauth/token-vault.js +317 -7
- package/dist/agent/oe-oauth/token-vault.js.map +1 -1
- package/dist/agent/register-defaults.d.ts.map +1 -1
- package/dist/agent/register-defaults.js +9 -0
- package/dist/agent/register-defaults.js.map +1 -1
- package/dist/agent/sandbox/long-lived-box.d.ts +70 -0
- package/dist/agent/sandbox/long-lived-box.d.ts.map +1 -0
- package/dist/agent/sandbox/long-lived-box.js +414 -0
- package/dist/agent/sandbox/long-lived-box.js.map +1 -0
- package/package.json +39 -2
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
// The meeting bot's two model calls (OPE-2459): the `gateway` transcription route, and the recap completion.
|
|
2
|
+
// Core owns both seams (meetings/transcription.ts); this file only fills them, so no meeting code calls a model SDK.
|
|
3
|
+
//
|
|
4
|
+
// TRANSCRIPTION (OPE-2470): one audio chunk per call, along the STACK'S TRANSCRIPTION ROLE — a primary route and a
|
|
5
|
+
// fallback route (transcription-routes.ts), named per stack in the app's `agent/model/stacks.ts`. A provider outage
|
|
6
|
+
// on the primary falls through to the fallback. The policy's route id stays `gateway` (core validates the id), but
|
|
7
|
+
// the role decides where the audio goes, and the approval card's sentence is derived from the role. With a role
|
|
8
|
+
// registered, the policy's `transcription.model` is NOT read.
|
|
9
|
+
//
|
|
10
|
+
// LEGACY: `registerGatewayMeetingModels(defaults)` with no role keeps the one-model route on the policy's
|
|
11
|
+
// `transcription.model`, for an app that has no transcription role yet.
|
|
12
|
+
//
|
|
13
|
+
// RECAP: one tool-less completion on the scope's `utility` route — the same route and call shape the dream
|
|
14
|
+
// judge uses. The prompt carries the fenced, untrusted transcript (core meetings/recap.ts).
|
|
15
|
+
|
|
16
|
+
import { experimental_transcribe as transcribe, generateText, gateway } from "ai";
|
|
17
|
+
import { registerMeetingCompletion, registerTranscriptionRoute, type MeetingCompletion, type TranscriptionInput, type TranscriptionRoute } from "@open-everywhere/core/agent/adapter-api.js";
|
|
18
|
+
import { gatewayTranscriptionModel, type GatewayRoutedDefaults } from "./gateway.js";
|
|
19
|
+
import { resolvedDreamCallSettings } from "./dream-judge.js";
|
|
20
|
+
import { roleDisclosure, transcribeWithFallback, type RoleTranscript, type TranscriptionDeps, type TranscriptionRoleSpec } from "./transcription-routes.js";
|
|
21
|
+
|
|
22
|
+
/** The injectable transcription call, so the legacy route is testable with no network. */
|
|
23
|
+
export type TranscribeCall = (modelId: string, audio: Uint8Array, mediaType: string) => Promise<string>;
|
|
24
|
+
|
|
25
|
+
// The SDK reads the container format from the bytes themselves, so the media type is not forwarded.
|
|
26
|
+
const defaultTranscribe: TranscribeCall = async (modelId, audio) => {
|
|
27
|
+
const result = await transcribe({ model: gatewayTranscriptionModel(modelId), audio });
|
|
28
|
+
return result.text;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
/** The legacy one-model route: the policy's `transcription.model` through the AI Gateway, with no fallback. */
|
|
32
|
+
export function gatewayTranscriptionRoute(call: TranscribeCall = defaultTranscribe): TranscriptionRoute {
|
|
33
|
+
return {
|
|
34
|
+
id: "gateway",
|
|
35
|
+
disclosure: "Audio goes, one minute at a time, through this deployment's own AI Gateway to its transcription model.",
|
|
36
|
+
async transcribe(input, opts) {
|
|
37
|
+
if (input.kind !== "audio") throw new Error("the gateway transcription route transcribes audio — this bot sent text");
|
|
38
|
+
if (opts.model === undefined) throw new Error("the gateway transcription route needs transcription.model in the meeting policy");
|
|
39
|
+
return await call(opts.model, input.data, input.mediaType);
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* The stack route: the core seam's `transcribe` (text only), plus `transcribeTimed`, which keeps the word and segment
|
|
46
|
+
* timestamps for the speaker alignment (OPE-2469) and says which route answered.
|
|
47
|
+
*/
|
|
48
|
+
export interface TimedTranscriptionRoute extends TranscriptionRoute {
|
|
49
|
+
transcribeTimed(input: TranscriptionInput, opts: { readonly scope: string; readonly prompt?: string }): Promise<RoleTranscript>;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function stackTranscriptionRoute(role: TranscriptionRoleSpec, deps: TranscriptionDeps = {}): TimedTranscriptionRoute {
|
|
53
|
+
const run = transcribeWithFallback(role, deps);
|
|
54
|
+
const timed: TimedTranscriptionRoute["transcribeTimed"] = async (input, opts) => {
|
|
55
|
+
if (input.kind !== "audio") throw new Error("the gateway transcription route transcribes audio — this bot sent text");
|
|
56
|
+
return await run({ audio: input.data, mediaType: input.mediaType, ...(opts.prompt !== undefined ? { prompt: opts.prompt } : {}) });
|
|
57
|
+
};
|
|
58
|
+
return {
|
|
59
|
+
id: "gateway",
|
|
60
|
+
disclosure: roleDisclosure(role),
|
|
61
|
+
transcribeTimed: timed,
|
|
62
|
+
async transcribe(input, opts) {
|
|
63
|
+
return (await timed(input, { scope: opts.scope })).text;
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function gatewayMeetingCompletion(defaults: GatewayRoutedDefaults): MeetingCompletion {
|
|
69
|
+
return async (scope, prompt) => {
|
|
70
|
+
const s = resolvedDreamCallSettings(scope, defaults);
|
|
71
|
+
const { text } = await generateText({
|
|
72
|
+
model: gateway(s.id),
|
|
73
|
+
prompt,
|
|
74
|
+
...(s.reasoning !== undefined ? { reasoning: s.reasoning } : {}),
|
|
75
|
+
...(s.providerOptions !== undefined ? { providerOptions: s.providerOptions as Parameters<typeof generateText>[0]["providerOptions"] } : {}),
|
|
76
|
+
});
|
|
77
|
+
return text;
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Boot registration from agent.ts, beside the dream judge: the recap rides the `utility` defaults, and the
|
|
83
|
+
* transcription route rides the stack's transcription role when the app passes one.
|
|
84
|
+
*/
|
|
85
|
+
export function registerGatewayMeetingModels(defaults: GatewayRoutedDefaults, transcription?: TranscriptionRoleSpec): void {
|
|
86
|
+
registerTranscriptionRoute(transcription === undefined ? gatewayTranscriptionRoute() : stackTranscriptionRoute(transcription));
|
|
87
|
+
registerMeetingCompletion(gatewayMeetingCompletion(defaults));
|
|
88
|
+
}
|
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
// THE TRANSCRIPTION ROLE'S TRANSPORTS (OPE-2470) — speech to text as a model-stack role with a primary and a
|
|
2
|
+
// fallback route. The app's `agent/model/stacks.ts` names the routes per stack; this file sends one audio chunk
|
|
3
|
+
// along them and returns the text with its timestamps.
|
|
4
|
+
//
|
|
5
|
+
// THREE TRANSPORTS. Each route names one:
|
|
6
|
+
// • `gateway` — the deployment's own AI Gateway (`gatewayTranscriptionModel`, the same provider and the same
|
|
7
|
+
// host plane as the turn model). The Gateway returns segments with start and end seconds; on
|
|
8
|
+
// `spacexai/grok-stt` each segment is one word (probed 2026-10-11).
|
|
9
|
+
// • `deepinfra` — DeepInfra's native inference endpoint for an open-weights speech model. It returns `segments`
|
|
10
|
+
// and `words`, each with `start` and `end` in seconds (DeepInfra's API page, 2026-10-11). The
|
|
11
|
+
// key is the `DEEPINFRA_API_KEY` entry of the DB secret store (`oe secret set`), and NEVER the
|
|
12
|
+
// environment: see {@link storeOnlySecret}.
|
|
13
|
+
// • `self-hosted` — an OpenAI-compatible `/audio/transcriptions` server that the deployer runs (vLLM serving
|
|
14
|
+
// Qwen3-ASR). Its base URL is config in the stack, not a secret. The call asks for
|
|
15
|
+
// `verbose_json` with word and segment timestamps and keeps whatever the server returns.
|
|
16
|
+
//
|
|
17
|
+
// THE FALLBACK. {@link transcribeWithFallback} tries the primary route, and on ANY failure (an outage, a 4xx from a
|
|
18
|
+
// provider allow-list, a timeout) tries the fallback route. Both fail ⇒ it throws one error that names both. It
|
|
19
|
+
// never tries a third route: the stack names what may hear the meeting, and nothing else may.
|
|
20
|
+
//
|
|
21
|
+
// 🔴 THE DISCLOSURE IS DERIVED. The approval card shows {@link roleDisclosure}, built from the SAME route list the
|
|
22
|
+
// call walks, so the card cannot name one provider while the audio goes to another.
|
|
23
|
+
|
|
24
|
+
import { experimental_transcribe as transcribe } from "ai";
|
|
25
|
+
import { getOperatorSecret } from "@open-everywhere/core/agent/adapter-api.js";
|
|
26
|
+
import { GATEWAY_TARGET_HOST, gatewayTranscriptionModel, sharedHostPlaneFetch } from "./gateway.js";
|
|
27
|
+
|
|
28
|
+
/** How a route reaches its model. */
|
|
29
|
+
export type TranscriptionVia = "gateway" | "deepinfra" | "self-hosted";
|
|
30
|
+
|
|
31
|
+
/** One route of the transcription role. */
|
|
32
|
+
export interface TranscriptionRouteSpec {
|
|
33
|
+
readonly via: TranscriptionVia;
|
|
34
|
+
/** The model: a gateway id (`gateway`), a DeepInfra model path (`deepinfra`), or the served model name (`self-hosted`). */
|
|
35
|
+
readonly id: string;
|
|
36
|
+
/** The server's OpenAI-compatible base URL, e.g. `http://qwen3-asr:8000/v1`. Required for `self-hosted`, refused elsewhere. */
|
|
37
|
+
readonly baseUrl?: string;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** The transcription role of one stack: a primary route and an optional fallback route. */
|
|
41
|
+
export interface TranscriptionRoleSpec {
|
|
42
|
+
readonly primary: TranscriptionRouteSpec;
|
|
43
|
+
readonly fallback?: TranscriptionRouteSpec;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** One timed span of the transcript, in seconds from the start of the chunk. */
|
|
47
|
+
export interface TimedSpan {
|
|
48
|
+
readonly text: string;
|
|
49
|
+
readonly startSecond: number;
|
|
50
|
+
readonly endSecond: number;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** One chunk's transcript. `words` and `segments` are empty when the route returns no timestamps. */
|
|
54
|
+
export interface TimedTranscript {
|
|
55
|
+
readonly text: string;
|
|
56
|
+
readonly segments: readonly TimedSpan[];
|
|
57
|
+
readonly words: readonly TimedSpan[];
|
|
58
|
+
readonly durationSeconds?: number;
|
|
59
|
+
/** The provider's own cost for the call, in USD, when it reports one (the Gateway and DeepInfra do). */
|
|
60
|
+
readonly costUsd?: number;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** What one route call receives. `prompt` is context text (the previous chunk's tail, names) for a model that takes one. */
|
|
64
|
+
export interface TranscribeRequest {
|
|
65
|
+
readonly audio: Uint8Array;
|
|
66
|
+
readonly mediaType: string;
|
|
67
|
+
readonly prompt?: string;
|
|
68
|
+
readonly abortSignal?: AbortSignal;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** The injectable seams, so every transport is testable with no network. */
|
|
72
|
+
export interface TranscriptionDeps {
|
|
73
|
+
/** The `gateway` transport. Default: the AI SDK's `transcribe` on the deployment's gateway provider. */
|
|
74
|
+
readonly gateway?: (modelId: string, req: TranscribeRequest) => Promise<TimedTranscript>;
|
|
75
|
+
/** The HTTP client for `deepinfra` and `self-hosted`. Default: the host-plane fetch on the proxy tier, else `fetch`. */
|
|
76
|
+
readonly fetch?: typeof fetch;
|
|
77
|
+
/** The host-plane seam: declare hosts, get the proxy-tier fetch (undefined off the tier). Default: gateway.ts `sharedHostPlaneFetch`. */
|
|
78
|
+
readonly hostPlane?: (hosts: readonly string[]) => typeof fetch | undefined;
|
|
79
|
+
/** Resolve the DeepInfra key. Default: {@link storeOnlySecret}. */
|
|
80
|
+
readonly secret?: (name: string) => Promise<string>;
|
|
81
|
+
/** One attempt's time limit, ms. Default {@link ATTEMPT_TIMEOUT_MS}. */
|
|
82
|
+
readonly attemptTimeoutMs?: number;
|
|
83
|
+
/** Where a fall-through is reported. Default: `console.warn`. */
|
|
84
|
+
readonly warn?: (message: string) => void;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** The secret-store name of the DeepInfra key. A fixed name in code, so no config line can point it at another secret. */
|
|
88
|
+
export const DEEPINFRA_SECRET = "DEEPINFRA_API_KEY";
|
|
89
|
+
const DEEPINFRA_HOST = "api.deepinfra.com";
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* One attempt's limit. A chunk is at most about 75 seconds of audio, and the probed routes answer a 20-second chunk
|
|
93
|
+
* in about one second. 40 seconds per attempt keeps primary plus fallback inside 80 seconds, so a hung provider
|
|
94
|
+
* falls through instead of holding the chunk.
|
|
95
|
+
*/
|
|
96
|
+
const ATTEMPT_TIMEOUT_MS = 40_000;
|
|
97
|
+
|
|
98
|
+
/** Refuse a malformed route at boot, not at the first chunk of a live meeting. */
|
|
99
|
+
export function assertTranscriptionRole(role: TranscriptionRoleSpec): void {
|
|
100
|
+
for (const [slot, r] of [["primary", role.primary], ["fallback", role.fallback]] as const) {
|
|
101
|
+
if (r === undefined) continue;
|
|
102
|
+
if (r.id.trim() === "") throw new Error(`transcription ${slot} route: the model id is empty`);
|
|
103
|
+
if (r.via === "self-hosted") {
|
|
104
|
+
if (r.baseUrl === undefined) throw new Error(`transcription ${slot} route: a self-hosted route needs baseUrl`);
|
|
105
|
+
const u = new URL(r.baseUrl);
|
|
106
|
+
if (u.protocol !== "http:" && u.protocol !== "https:") throw new Error(`transcription ${slot} route: baseUrl must be http or https`);
|
|
107
|
+
if (u.username !== "" || u.password !== "") throw new Error(`transcription ${slot} route: baseUrl must not carry credentials`);
|
|
108
|
+
} else if (r.baseUrl !== undefined) {
|
|
109
|
+
throw new Error(`transcription ${slot} route: baseUrl is for a self-hosted route only`);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** The host a route's audio reaches. */
|
|
115
|
+
function routeHost(r: TranscriptionRouteSpec): string {
|
|
116
|
+
if (r.via === "deepinfra") return DEEPINFRA_HOST;
|
|
117
|
+
if (r.via === "self-hosted") return new URL(r.baseUrl ?? "").hostname;
|
|
118
|
+
return "the AI Gateway";
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function routePhrase(r: TranscriptionRouteSpec): string {
|
|
122
|
+
if (r.via === "gateway") return `this deployment's own AI Gateway to \`${r.id}\``;
|
|
123
|
+
if (r.via === "deepinfra") return `DeepInfra (${DEEPINFRA_HOST}) to \`${r.id}\``;
|
|
124
|
+
return `this deployment's own speech server at ${routeHost(r)} (\`${r.id}\`)`;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** The approval card's sentence: every place the audio can go, in the order it is tried. */
|
|
128
|
+
export function roleDisclosure(role: TranscriptionRoleSpec): string {
|
|
129
|
+
const head = `Audio goes, in chunks of about a minute, through ${routePhrase(role.primary)}.`;
|
|
130
|
+
return role.fallback === undefined ? head : `${head} If that fails, it goes through ${routePhrase(role.fallback)} instead.`;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Resolve a key from the DB-backed secret store, and from nothing else.
|
|
135
|
+
*
|
|
136
|
+
* `getOperatorSecret` falls back to an env var of the same name (its migration on-ramp). The DeepInfra key is not
|
|
137
|
+
* infrastructure, so it has no env home: a value that resolved from the environment is refused, and the refusal
|
|
138
|
+
* names the fix. The same rule as the SIEM export token (core audit/siem-export.ts).
|
|
139
|
+
*/
|
|
140
|
+
export async function storeOnlySecret(
|
|
141
|
+
name: string,
|
|
142
|
+
resolve: (name: string) => Promise<{ readonly value: string; readonly source: string }> = getOperatorSecret,
|
|
143
|
+
): Promise<string> {
|
|
144
|
+
const resolved = await resolve(name);
|
|
145
|
+
if (resolved.source !== "db") {
|
|
146
|
+
throw new Error(`secret "${name}" resolves only from the environment; the DeepInfra transcription route reads the secret store and nothing else. Store it with \`oe secret set ${name}\``);
|
|
147
|
+
}
|
|
148
|
+
return resolved.value;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// ── wire parsing ────────────────────────────────────────────────────────────────────────────────────
|
|
152
|
+
|
|
153
|
+
function num(v: unknown): number | undefined {
|
|
154
|
+
if (typeof v === "number" && Number.isFinite(v)) return v;
|
|
155
|
+
if (typeof v === "string" && v.trim() !== "" && Number.isFinite(Number(v))) return Number(v);
|
|
156
|
+
return undefined;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** Read `[{text|word, start, end}]` into spans. An entry without a finite start and end is dropped. */
|
|
160
|
+
function parseSpans(raw: unknown): TimedSpan[] {
|
|
161
|
+
if (!Array.isArray(raw)) return [];
|
|
162
|
+
const out: TimedSpan[] = [];
|
|
163
|
+
for (const e of raw as Record<string, unknown>[]) {
|
|
164
|
+
if (e === null || typeof e !== "object") continue;
|
|
165
|
+
const text = typeof e.text === "string" ? e.text : typeof e.word === "string" ? e.word : undefined;
|
|
166
|
+
const start = num(e.start ?? e.startSecond);
|
|
167
|
+
const end = num(e.end ?? e.endSecond);
|
|
168
|
+
if (text === undefined || start === undefined || end === undefined || end < start) continue;
|
|
169
|
+
out.push({ text: text.trim(), startSecond: start, endSecond: end });
|
|
170
|
+
}
|
|
171
|
+
return out;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** True when every span holds one word. Then the segments are word timestamps (the Gateway's grok-stt shape). */
|
|
175
|
+
function spansAreWords(spans: readonly TimedSpan[]): boolean {
|
|
176
|
+
return spans.length > 0 && spans.every((s) => s.text !== "" && !/\s/.test(s.text));
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Parse an OpenAI-compatible `verbose_json` body, or DeepInfra's native body (the same field names). */
|
|
180
|
+
export function parseTranscriptionBody(body: unknown): TimedTranscript {
|
|
181
|
+
if (body === null || typeof body !== "object") throw new Error("transcription response is not a JSON object");
|
|
182
|
+
const b = body as Record<string, unknown>;
|
|
183
|
+
if (typeof b.text !== "string") throw new Error("transcription response has no text");
|
|
184
|
+
const segments = parseSpans(b.segments);
|
|
185
|
+
const words = parseSpans(b.words);
|
|
186
|
+
const status = b.inference_status as Record<string, unknown> | undefined;
|
|
187
|
+
const cost = num(status?.cost);
|
|
188
|
+
const duration = num(b.duration);
|
|
189
|
+
return {
|
|
190
|
+
text: b.text.trim(),
|
|
191
|
+
segments,
|
|
192
|
+
words: words.length > 0 ? words : spansAreWords(segments) ? segments : [],
|
|
193
|
+
...(duration !== undefined ? { durationSeconds: duration } : {}),
|
|
194
|
+
...(cost !== undefined ? { costUsd: cost } : {}),
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** `fetch` looked up at call time, so instrumentation that wraps it after boot still sees the call. */
|
|
199
|
+
const lateFetch: typeof fetch = (input, init) => globalThis.fetch(input, init);
|
|
200
|
+
|
|
201
|
+
// ── transports ──────────────────────────────────────────────────────────────────────────────────────
|
|
202
|
+
|
|
203
|
+
const defaultGateway = async (modelId: string, req: TranscribeRequest): Promise<TimedTranscript> => {
|
|
204
|
+
// The SDK reads the container format from the bytes themselves, so the media type is not forwarded. The prompt is
|
|
205
|
+
// not forwarded either: the Gateway takes no common prompt field for transcription models.
|
|
206
|
+
const r = await transcribe({ model: gatewayTranscriptionModel(modelId), audio: req.audio, ...(req.abortSignal ? { abortSignal: req.abortSignal } : {}) });
|
|
207
|
+
const segments = r.segments.map((s) => ({ text: s.text.trim(), startSecond: s.startSecond, endSecond: s.endSecond }));
|
|
208
|
+
const cost = num((r.providerMetadata?.gateway as Record<string, unknown> | undefined)?.cost);
|
|
209
|
+
return {
|
|
210
|
+
text: r.text.trim(),
|
|
211
|
+
segments,
|
|
212
|
+
words: spansAreWords(segments) ? segments : [],
|
|
213
|
+
...(r.durationInSeconds !== undefined ? { durationSeconds: r.durationInSeconds } : {}),
|
|
214
|
+
...(cost !== undefined ? { costUsd: cost } : {}),
|
|
215
|
+
};
|
|
216
|
+
};
|
|
217
|
+
|
|
218
|
+
function fileName(mediaType: string): string {
|
|
219
|
+
const sub = mediaType.split(";")[0]!.split("/")[1] ?? "bin";
|
|
220
|
+
return `chunk.${sub.replace(/[^a-z0-9]/gi, "") || "bin"}`;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
async function postForm(f: typeof fetch, url: string, form: FormData, headers: Record<string, string>, signal: AbortSignal | undefined): Promise<TimedTranscript> {
|
|
224
|
+
const res = await f(url, { method: "POST", body: form, headers, ...(signal ? { signal } : {}) });
|
|
225
|
+
if (!res.ok) {
|
|
226
|
+
// The status only: a provider's error body can echo request details.
|
|
227
|
+
throw new Error(`HTTP ${res.status} from ${new URL(url).hostname}`);
|
|
228
|
+
}
|
|
229
|
+
return parseTranscriptionBody(await res.json());
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function deepinfraCall(f: typeof fetch, secret: (name: string) => Promise<string>) {
|
|
233
|
+
return async (r: TranscriptionRouteSpec, req: TranscribeRequest): Promise<TimedTranscript> => {
|
|
234
|
+
const key = await secret(DEEPINFRA_SECRET);
|
|
235
|
+
const form = new FormData();
|
|
236
|
+
form.append("audio", new Blob([req.audio as Uint8Array<ArrayBuffer>], { type: req.mediaType }), fileName(req.mediaType));
|
|
237
|
+
const path = r.id.split("/").map(encodeURIComponent).join("/");
|
|
238
|
+
return await postForm(f, `https://${DEEPINFRA_HOST}/v1/inference/${path}`, form, { authorization: `Bearer ${key}` }, req.abortSignal);
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function selfHostedCall(f: typeof fetch) {
|
|
243
|
+
return async (r: TranscriptionRouteSpec, req: TranscribeRequest): Promise<TimedTranscript> => {
|
|
244
|
+
const form = new FormData();
|
|
245
|
+
form.append("file", new Blob([req.audio as Uint8Array<ArrayBuffer>], { type: req.mediaType }), fileName(req.mediaType));
|
|
246
|
+
form.append("model", r.id);
|
|
247
|
+
form.append("response_format", "verbose_json");
|
|
248
|
+
form.append("timestamp_granularities[]", "word");
|
|
249
|
+
form.append("timestamp_granularities[]", "segment");
|
|
250
|
+
if (req.prompt !== undefined && req.prompt !== "") form.append("prompt", req.prompt);
|
|
251
|
+
const base = (r.baseUrl ?? "").replace(/\/+$/, "");
|
|
252
|
+
return await postForm(f, `${base}/audio/transcriptions`, form, {}, req.abortSignal);
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* The hosts a role's routes reach on the host plane: the gateway host FIRST, then every non-gateway route host.
|
|
258
|
+
* They join the process-wide host set in gateway.ts (`sharedHostPlaneFetch`), so the one shared fetch pushes the
|
|
259
|
+
* union and neither the turn model nor a transcription route drops the other's host.
|
|
260
|
+
*/
|
|
261
|
+
export function roleHostPlaneHosts(role: TranscriptionRoleSpec): string[] {
|
|
262
|
+
const hosts = [GATEWAY_TARGET_HOST];
|
|
263
|
+
for (const r of [role.primary, role.fallback]) {
|
|
264
|
+
if (r === undefined || r.via === "gateway") continue;
|
|
265
|
+
const h = routeHost(r);
|
|
266
|
+
if (!hosts.includes(h)) hosts.push(h);
|
|
267
|
+
}
|
|
268
|
+
return hosts;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** The transport for one route. Exported for the bake-off harness, which calls each route on its own. */
|
|
272
|
+
export function routeTransport(r: TranscriptionRouteSpec, deps: TranscriptionDeps = {}): (req: TranscribeRequest) => Promise<TimedTranscript> {
|
|
273
|
+
if (r.via === "gateway") {
|
|
274
|
+
const g = deps.gateway ?? defaultGateway;
|
|
275
|
+
return (req) => g(r.id, req);
|
|
276
|
+
}
|
|
277
|
+
const f = deps.fetch ?? (deps.hostPlane ?? sharedHostPlaneFetch)(roleHostPlaneHosts({ primary: r })) ?? lateFetch;
|
|
278
|
+
if (r.via === "deepinfra") {
|
|
279
|
+
const call = deepinfraCall(f, deps.secret ?? storeOnlySecret);
|
|
280
|
+
return (req) => call(r, req);
|
|
281
|
+
}
|
|
282
|
+
const call = selfHostedCall(f);
|
|
283
|
+
return (req) => call(r, req);
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/** Which route answered. */
|
|
287
|
+
export interface RoleTranscript extends TimedTranscript {
|
|
288
|
+
readonly answeredBy: "primary" | "fallback";
|
|
289
|
+
readonly route: TranscriptionRouteSpec;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
function describe(r: TranscriptionRouteSpec): string {
|
|
293
|
+
return `${r.via}:${r.id}`;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
function errText(err: unknown): string {
|
|
297
|
+
return err instanceof Error ? err.message : String(err);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
async function attempt(run: (req: TranscribeRequest) => Promise<TimedTranscript>, req: TranscribeRequest, timeoutMs: number): Promise<TimedTranscript> {
|
|
301
|
+
// A plain timer, not `AbortSignal.timeout`: that timer is unref'd, so a process with nothing else pending exits
|
|
302
|
+
// before a hung attempt is cut.
|
|
303
|
+
const limit = new AbortController();
|
|
304
|
+
const timer = setTimeout(() => limit.abort(new Error(`no answer within ${timeoutMs} ms`)), timeoutMs);
|
|
305
|
+
const signal = req.abortSignal ? AbortSignal.any([req.abortSignal, limit.signal]) : limit.signal;
|
|
306
|
+
// The race cuts a transport that ignores the signal, too.
|
|
307
|
+
const aborted = new Promise<never>((_, reject) => {
|
|
308
|
+
if (signal.aborted) reject(signal.reason);
|
|
309
|
+
signal.addEventListener("abort", () => reject(signal.reason), { once: true });
|
|
310
|
+
});
|
|
311
|
+
try {
|
|
312
|
+
return await Promise.race([run({ ...req, abortSignal: signal }), aborted]);
|
|
313
|
+
} finally {
|
|
314
|
+
clearTimeout(timer);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Build the role's transcriber: primary, then fallback on any failure. The routes are validated here, at build time.
|
|
320
|
+
*/
|
|
321
|
+
export function transcribeWithFallback(role: TranscriptionRoleSpec, deps: TranscriptionDeps = {}): (req: TranscribeRequest) => Promise<RoleTranscript> {
|
|
322
|
+
assertTranscriptionRole(role);
|
|
323
|
+
const timeoutMs = deps.attemptTimeoutMs ?? ATTEMPT_TIMEOUT_MS;
|
|
324
|
+
const warn = deps.warn ?? ((m: string) => console.warn(m));
|
|
325
|
+
// ONE host-plane fetch for the whole role, so the two routes never push host lists that drop each other.
|
|
326
|
+
const shared: TranscriptionDeps = { ...deps, fetch: deps.fetch ?? (deps.hostPlane ?? sharedHostPlaneFetch)(roleHostPlaneHosts(role)) ?? lateFetch };
|
|
327
|
+
const primary = routeTransport(role.primary, shared);
|
|
328
|
+
const fallback = role.fallback === undefined ? undefined : routeTransport(role.fallback, shared);
|
|
329
|
+
return async (req) => {
|
|
330
|
+
let first: unknown;
|
|
331
|
+
try {
|
|
332
|
+
return { ...(await attempt(primary, req, timeoutMs)), answeredBy: "primary", route: role.primary };
|
|
333
|
+
} catch (err) {
|
|
334
|
+
if (req.abortSignal?.aborted) throw err;
|
|
335
|
+
first = err;
|
|
336
|
+
}
|
|
337
|
+
if (fallback === undefined || role.fallback === undefined) {
|
|
338
|
+
throw new Error(`transcription failed on ${describe(role.primary)}: ${errText(first)}`);
|
|
339
|
+
}
|
|
340
|
+
warn(`[open-everywhere:transcription] ${describe(role.primary)} failed (${errText(first)}); trying the fallback ${describe(role.fallback)}`);
|
|
341
|
+
try {
|
|
342
|
+
return { ...(await attempt(fallback, req, timeoutMs)), answeredBy: "fallback", route: role.fallback };
|
|
343
|
+
} catch (second) {
|
|
344
|
+
throw new Error(`transcription failed on both routes — ${describe(role.primary)}: ${errText(first)}; ${describe(role.fallback)}: ${errText(second)}`);
|
|
345
|
+
}
|
|
346
|
+
};
|
|
347
|
+
}
|
package/agent/oe-oauth/config.ts
CHANGED
|
@@ -14,6 +14,9 @@ export const OE_OAUTH_MODE_DISCOVERY = "discovery-dcr-pkce";
|
|
|
14
14
|
/** The static confidential-client dialect (OPE-1086) — no discovery, no registration, HTTP Basic. */
|
|
15
15
|
export const OE_OAUTH_MODE_STATIC = "static-confidential";
|
|
16
16
|
|
|
17
|
+
/** The Cross App Access dialect (OPE-2453) — an IdP sign-in once, then an ID-JAG mint per app token. */
|
|
18
|
+
export const OE_OAUTH_MODE_XAA = "xaa";
|
|
19
|
+
|
|
17
20
|
/** How a static provider encodes its token-request body. Declared, never inferred: Notion's REST
|
|
18
21
|
* token endpoint takes JSON, the RFC 6749 default is form-encoded, and sending the wrong one reads
|
|
19
22
|
* as an authentication failure rather than as a shape error. */
|
|
@@ -66,8 +69,36 @@ export interface OeOauthStaticProviderConfig extends OeOauthProviderConfigBase {
|
|
|
66
69
|
readonly bodyEncoding: OeOauthBodyEncoding;
|
|
67
70
|
}
|
|
68
71
|
|
|
69
|
-
/**
|
|
70
|
-
|
|
72
|
+
/**
|
|
73
|
+
* A Cross App Access provider (OPE-2453). The person signs in to the IdP (`idpIssuer`) once; each mint then trades
|
|
74
|
+
* the person's IdP refresh token for an ID-JAG (RFC 8693, at the IdP) and the ID-JAG for the app's access token
|
|
75
|
+
* (RFC 7523, at `resourceTokenEndpoint`). Two confidential clients, two secret REFS, no literal secret.
|
|
76
|
+
*/
|
|
77
|
+
export interface OeOauthXaaProviderConfig extends OeOauthProviderConfigBase {
|
|
78
|
+
readonly mode: typeof OE_OAUTH_MODE_XAA;
|
|
79
|
+
/** The IdP's OIDC issuer. Its `/.well-known/openid-configuration` names the authorize, token, JWKS and
|
|
80
|
+
* revocation endpoints. HTTPS only. */
|
|
81
|
+
readonly idpIssuer: string;
|
|
82
|
+
readonly idpClientId: string;
|
|
83
|
+
readonly idpClientSecretRef: string;
|
|
84
|
+
readonly idpClientSecretBroker: string;
|
|
85
|
+
/** The app's authorization server issuer — the ID-JAG `audience`. */
|
|
86
|
+
readonly audience: string;
|
|
87
|
+
/** The RFC 8707 resource both legs name, when the app asks for one. */
|
|
88
|
+
readonly resource?: string;
|
|
89
|
+
readonly resourceTokenEndpoint: string;
|
|
90
|
+
readonly resourceClientId: string;
|
|
91
|
+
readonly resourceClientSecretRef: string;
|
|
92
|
+
readonly resourceClientSecretBroker: string;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** One provider's OAuth target, in one of the three declared modes. */
|
|
96
|
+
export type OeOauthProviderConfig = OeOauthDiscoveryProviderConfig | OeOauthStaticProviderConfig | OeOauthXaaProviderConfig;
|
|
97
|
+
|
|
98
|
+
/** Narrow a provider config to the Cross App Access mode. Reads the DECLARED `mode` only. */
|
|
99
|
+
export function isXaa(cfg: OeOauthProviderConfig): cfg is OeOauthXaaProviderConfig {
|
|
100
|
+
return cfg.mode === OE_OAUTH_MODE_XAA;
|
|
101
|
+
}
|
|
71
102
|
|
|
72
103
|
/** Narrow a provider config to the static confidential-client mode. Reads the DECLARED `mode` and
|
|
73
104
|
* nothing else — never the presence of a field. */
|
|
@@ -162,6 +193,39 @@ function parseStaticProvider(ref: string, rec: Record<string, unknown>): OeOauth
|
|
|
162
193
|
};
|
|
163
194
|
}
|
|
164
195
|
|
|
196
|
+
/** Parse ONE `xaa` entry, with every refusal stated. */
|
|
197
|
+
function parseXaaProvider(ref: string, rec: Record<string, unknown>): OeOauthXaaProviderConfig {
|
|
198
|
+
const where = `sign-in "${ref}"`;
|
|
199
|
+
// 🔴 A LITERAL SECRET IS REFUSED, as in the static dialect: a declaration is committed source.
|
|
200
|
+
for (const literal of ["clientSecret", "idpClientSecret", "resourceClientSecret"]) {
|
|
201
|
+
if (literal in rec) {
|
|
202
|
+
throw new Error(
|
|
203
|
+
`${where} carries a literal "${literal}". A client secret must NEVER live in a sign-in declaration — store it in the ` +
|
|
204
|
+
`credential store and name it with "${literal === "clientSecret" ? "idpClientSecretRef" : `${literal}Ref`}".`,
|
|
205
|
+
);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
const resource = rec.resource === undefined ? undefined : requireString(rec.resource, `${where}.resource`);
|
|
209
|
+
const scopes = parseScopes(rec.scopes);
|
|
210
|
+
return {
|
|
211
|
+
mode: OE_OAUTH_MODE_XAA,
|
|
212
|
+
ref,
|
|
213
|
+
// The issuer is compared byte-for-byte with the id token's `iss`, so it is kept exactly as declared.
|
|
214
|
+
idpIssuer: requireHttpsUrl(rec.idpIssuer, `${where}.idpIssuer`),
|
|
215
|
+
idpClientId: requireString(rec.idpClientId, `${where}.idpClientId`),
|
|
216
|
+
idpClientSecretRef: requireString(rec.idpClientSecretRef, `${where}.idpClientSecretRef`),
|
|
217
|
+
idpClientSecretBroker: rec.idpClientSecretBroker === undefined ? "db" : requireString(rec.idpClientSecretBroker, `${where}.idpClientSecretBroker`),
|
|
218
|
+
audience: requireString(rec.audience, `${where}.audience`),
|
|
219
|
+
...(resource ? { resource } : {}),
|
|
220
|
+
resourceTokenEndpoint: requireHttpsUrl(rec.resourceTokenEndpoint, `${where}.resourceTokenEndpoint`),
|
|
221
|
+
resourceClientId: requireString(rec.resourceClientId, `${where}.resourceClientId`),
|
|
222
|
+
resourceClientSecretRef: requireString(rec.resourceClientSecretRef, `${where}.resourceClientSecretRef`),
|
|
223
|
+
resourceClientSecretBroker:
|
|
224
|
+
rec.resourceClientSecretBroker === undefined ? "db" : requireString(rec.resourceClientSecretBroker, `${where}.resourceClientSecretBroker`),
|
|
225
|
+
...(scopes ? { scopes } : {}),
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
|
|
165
229
|
/** Parse ONE `discovery-dcr-pkce` entry. */
|
|
166
230
|
function parseDiscoveryProvider(ref: string, rec: Record<string, unknown>): OeOauthDiscoveryProviderConfig {
|
|
167
231
|
const where = `sign-in "${ref}"`;
|
|
@@ -192,9 +256,13 @@ function parseDeclarations(declared: ReadonlyMap<string, unknown>): Map<string,
|
|
|
192
256
|
out.set(ref, parseStaticProvider(ref, rec));
|
|
193
257
|
continue;
|
|
194
258
|
}
|
|
259
|
+
if (mode === OE_OAUTH_MODE_XAA) {
|
|
260
|
+
out.set(ref, parseXaaProvider(ref, rec));
|
|
261
|
+
continue;
|
|
262
|
+
}
|
|
195
263
|
if (mode !== OE_OAUTH_MODE_DISCOVERY) {
|
|
196
264
|
throw new Error(
|
|
197
|
-
`sign-in "${ref}".mode ${JSON.stringify(mode)} is not a known oe-oauth mode — use "${OE_OAUTH_MODE_DISCOVERY}" or "${
|
|
265
|
+
`sign-in "${ref}".mode ${JSON.stringify(mode)} is not a known oe-oauth mode — use "${OE_OAUTH_MODE_DISCOVERY}", "${OE_OAUTH_MODE_STATIC}" or "${OE_OAUTH_MODE_XAA}".`,
|
|
198
266
|
);
|
|
199
267
|
}
|
|
200
268
|
const parsed = parseDiscoveryProvider(ref, rec);
|
package/agent/oe-oauth/errors.ts
CHANGED
|
@@ -40,6 +40,23 @@ export class OeOauthRefreshContendedError extends Error {
|
|
|
40
40
|
}
|
|
41
41
|
}
|
|
42
42
|
|
|
43
|
+
/** Cross App Access (OPE-2453): the IdP or the app REFUSED the mint for a person who is signed in — the IdP admin
|
|
44
|
+
* removed the pairing, unassigned the person, or never granted this app. A new sign-in link cannot fix that, so
|
|
45
|
+
* this error is DELIBERATELY not classified as user-auth-required: the person gets no "Connect my account" link,
|
|
46
|
+
* only the reason. Carries the RFC 6749 error code, never token material. */
|
|
47
|
+
export class OeOauthAccessRemovedError extends Error {
|
|
48
|
+
readonly code = "oe_oauth_access_removed";
|
|
49
|
+
readonly oauthError?: string;
|
|
50
|
+
constructor(providerRef: string, subject: { readonly type: "user"; readonly id: string } | { readonly type: "app" }, where: "idp" | "app", oauthError?: string) {
|
|
51
|
+
super(
|
|
52
|
+
`connection "${providerRef}": ${subjectLabel(subject)} — the ${where === "idp" ? "identity provider" : "app"} refused access` +
|
|
53
|
+
`${oauthError ? ` (${oauthError})` : ""}. Your identity-provider admin manages this access; a new sign-in will not restore it.`,
|
|
54
|
+
);
|
|
55
|
+
this.name = "OeOauthAccessRemovedError";
|
|
56
|
+
if (oauthError) this.oauthError = oauthError;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
43
60
|
/** A generic upstream OAuth failure (token endpoint / discovery / DCR returned an error). `oauthError`
|
|
44
61
|
* is the RFC 6749 `error` code when the body carried one (e.g. "invalid_grant", "invalid_client") —
|
|
45
62
|
* a fixed-vocabulary string, never a secret. */
|