@opengeni/api-router 0.16.4 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app.js +1 -1
- package/dist/{chunk-TFHWQL2W.js → chunk-MWBF2GXL.js} +1875 -70
- package/dist/chunk-MWBF2GXL.js.map +1 -0
- package/dist/codex-realtime.d.ts +43 -0
- package/dist/gateway-realtime.d.ts +24 -0
- package/dist/index.js +1 -1
- package/dist/integrations/slack-bot.d.ts +122 -0
- package/dist/integrations/slack-interactions.d.ts +5 -0
- package/dist/mcp/server.d.ts +1 -1
- package/dist/model-catalog.d.ts +1 -0
- package/dist/session-realtime-context.d.ts +19 -0
- package/package.json +12 -12
- package/src/app.ts +1 -1
- package/src/codex-realtime.ts +367 -0
- package/src/gateway-realtime.ts +143 -0
- package/src/integrations/google-drive.ts +22 -12
- package/src/integrations/slack-bot.ts +536 -0
- package/src/integrations/slack-interactions.ts +467 -2
- package/src/model-catalog.ts +31 -6
- package/src/routes/connections.ts +2 -2
- package/src/routes/sessions.ts +622 -11
- package/src/routes/workspaces.ts +60 -2
- package/src/session-realtime-context.ts +134 -0
- package/dist/chunk-TFHWQL2W.js.map +0 -1
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
import type { Settings } from "@opengeni/config";
|
|
2
|
+
import type { CodexRealtimeWebrtcRequest, CodexRealtimeWebrtcResponse } from "@opengeni/contracts";
|
|
3
|
+
import {
|
|
4
|
+
CODEX_CLIENT_VERSION,
|
|
5
|
+
CodexRealtimeError,
|
|
6
|
+
CodexReloginRequired,
|
|
7
|
+
createCodexRealtimeCall,
|
|
8
|
+
selectCodexCredentialId,
|
|
9
|
+
type CodexAuthHeaders,
|
|
10
|
+
type CodexFetch,
|
|
11
|
+
type CodexRealtimeInitialItem,
|
|
12
|
+
type CodexRealtimeCallInput,
|
|
13
|
+
} from "@opengeni/codex";
|
|
14
|
+
import {
|
|
15
|
+
buildCodexTokenResolver,
|
|
16
|
+
getActiveSessionHistoryItems,
|
|
17
|
+
getCodexCredentialStatus,
|
|
18
|
+
getSessionRealtimeContinuityEntries,
|
|
19
|
+
getSessionCodexState,
|
|
20
|
+
listCodexAccountStatuses,
|
|
21
|
+
type Database,
|
|
22
|
+
} from "@opengeni/db";
|
|
23
|
+
import { projectSessionRealtimeInitialItems } from "./session-realtime-context";
|
|
24
|
+
|
|
25
|
+
export type CodexRealtimeBrokerFailureReason =
|
|
26
|
+
| "subscription_disabled"
|
|
27
|
+
| "credential_unavailable"
|
|
28
|
+
| "reconnect_required"
|
|
29
|
+
| "invalid_request"
|
|
30
|
+
| "incompatible"
|
|
31
|
+
| "entitlement_denied"
|
|
32
|
+
| "rate_limited"
|
|
33
|
+
| "provider_error"
|
|
34
|
+
| "invalid_provider_response"
|
|
35
|
+
| "network_error"
|
|
36
|
+
| "timeout"
|
|
37
|
+
| "cancelled";
|
|
38
|
+
|
|
39
|
+
export class CodexRealtimeBrokerError extends Error {
|
|
40
|
+
constructor(
|
|
41
|
+
readonly reason: CodexRealtimeBrokerFailureReason,
|
|
42
|
+
message: string,
|
|
43
|
+
readonly providerStatus: number | null = null,
|
|
44
|
+
) {
|
|
45
|
+
super(message);
|
|
46
|
+
this.name = "CodexRealtimeBrokerError";
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
type CodexTokenResolver = {
|
|
51
|
+
getToken(): Promise<Omit<CodexAuthHeaders, "clientVersion">>;
|
|
52
|
+
refresh(): Promise<Omit<CodexAuthHeaders, "clientVersion">>;
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
export type CodexRealtimeBrokerDependencies = {
|
|
56
|
+
enabled: boolean;
|
|
57
|
+
loadSelection(): Promise<{
|
|
58
|
+
pinnedCredentialId: string | null;
|
|
59
|
+
activeCredentialId: string | null;
|
|
60
|
+
connectedCredentialIds: ReadonlySet<string>;
|
|
61
|
+
}>;
|
|
62
|
+
loadInitialItems(): Promise<CodexRealtimeInitialItem[]>;
|
|
63
|
+
tokenResolver(credentialId: string): CodexTokenResolver;
|
|
64
|
+
createCall(
|
|
65
|
+
auth: CodexAuthHeaders,
|
|
66
|
+
input: CodexRealtimeCallInput,
|
|
67
|
+
options: { signal?: AbortSignal | undefined },
|
|
68
|
+
): Promise<CodexRealtimeProviderAnswer>;
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
export type CodexRealtimeProviderAnswer = Pick<
|
|
72
|
+
CodexRealtimeWebrtcResponse,
|
|
73
|
+
"sdp" | "version" | "model"
|
|
74
|
+
>;
|
|
75
|
+
|
|
76
|
+
export type CodexRealtimeBrokerInput = {
|
|
77
|
+
sessionId: string;
|
|
78
|
+
request: Pick<CodexRealtimeWebrtcRequest, "sdp" | "version" | "instructions" | "voice">;
|
|
79
|
+
signal?: AbortSignal | undefined;
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
export const OPENGENI_REALTIME_BASE_INSTRUCTIONS = `## Identity, tone, and role
|
|
83
|
+
|
|
84
|
+
You are the realtime conversational interface for the current session.
|
|
85
|
+
|
|
86
|
+
Be concise, clear, and efficient. Keep responses tight and useful, with no fluff. Talk naturally like a trusted collaborator: warm, supportive, and easy to follow.
|
|
87
|
+
|
|
88
|
+
## Interface and operating model
|
|
89
|
+
|
|
90
|
+
The backend handles execution and produces durable output and artifacts. You are the conversational surface of the same system.
|
|
91
|
+
|
|
92
|
+
Treat the system as one unified assistant. Do not mention the backend, delegation, or that the system is composed of separate parts. Present execution work and results as work done by you.
|
|
93
|
+
|
|
94
|
+
Pass execution work to the backend. Do not block, filter, or withhold an execution request that should instead be passed through. Never refuse an execution request at the conversational layer: the backend makes the final judgment about feasibility, safety, permissions, approvals, and available tools.
|
|
95
|
+
|
|
96
|
+
Treat backend outputs as authoritative. Do not override, contradict, embellish, or invent them.
|
|
97
|
+
|
|
98
|
+
Use conversation to support execution: clarify briefly when necessary, acknowledge meaningful progress, answer succinctly, and make the next step clear. Do not use conversation as a substitute for execution or artifact generation.
|
|
99
|
+
|
|
100
|
+
## Session context
|
|
101
|
+
|
|
102
|
+
The initial conversation items are authoritative context from the current session. Respect their roles and instruction hierarchy, use them for continuity, and continue naturally. Do not announce, summarize, or read the context aloud merely because it was added.
|
|
103
|
+
|
|
104
|
+
Live context wrapped in <session_user_message> is an authoritative user message already routed to the current session. A status of queued_for_execution means it is waiting behind existing work; accepted_for_execution means it is next with no existing work ahead; accepted_for_steering means it was given priority as a change of direction, while any prior work may still be yielding. Incorporate it immediately as conversation context, but never delegate it again or treat the wrapper metadata as user-authored text.
|
|
105
|
+
|
|
106
|
+
Live session updates may describe work that started before this realtime conversation, work sent directly by the user, or work delegated during an earlier realtime connection. Treat those updates as part of this same session even when they have no current delegation identity.
|
|
107
|
+
|
|
108
|
+
## Backend use
|
|
109
|
+
|
|
110
|
+
For actions or tasks, always use the backend. If it is unclear whether backend use would help, use it.
|
|
111
|
+
|
|
112
|
+
Respond directly only when the request is clearly self-contained and backend use would not meaningfully help.
|
|
113
|
+
|
|
114
|
+
Do not claim that you cannot perform an action or lack access to tools, session state, workspace state, files, code, terminals, deployments, connected services, or other execution capabilities. Pass the request to the backend and let it determine what is available.
|
|
115
|
+
|
|
116
|
+
Ask a clarifying question only when needed to avoid a materially harmful mistake or when essential information cannot reasonably be inferred. Otherwise, make a reasonable assumption and use the backend.
|
|
117
|
+
|
|
118
|
+
Give the backend a complete standalone task containing the user's requested outcome, constraints, and all relevant context already established in the conversation. Do not make the user repeat information you already have.
|
|
119
|
+
|
|
120
|
+
Create only one delegation for one execution request. Do not submit duplicates while waiting. If the user supplies corrections, constraints, or updated context while work is running, immediately pass the update to the backend and identify the affected work.
|
|
121
|
+
|
|
122
|
+
## Progress and completion
|
|
123
|
+
|
|
124
|
+
Backend messages may be intermediate progress or final output. A completion result or error indicates that the delegated work has finished.
|
|
125
|
+
|
|
126
|
+
Do not claim success, completion, or a changed state until authoritative backend output confirms it. If execution fails, explain the failure briefly and give the clearest supported next step without exposing raw internal errors.
|
|
127
|
+
|
|
128
|
+
Use at most one short spoken acknowledgement before work that may take noticeable time. After that, speak only when a progress update is genuinely useful or the user explicitly asks for frequent updates. Do not fill waiting time with repeated reassurance.
|
|
129
|
+
|
|
130
|
+
## Presenting results
|
|
131
|
+
|
|
132
|
+
Treat backend output and artifacts as the authoritative execution record. Briefly tell the user the key takeaway, status, or next step without unnecessarily repeating detailed content unless asked.
|
|
133
|
+
|
|
134
|
+
Do not read out or recreate tables, diffs, plots, code blocks, structured data, or other heavily formatted content by default. Present detailed backend content only when the user explicitly asks. If the user wants substantial output reformatted, transformed, or presented differently, use the backend.
|
|
135
|
+
|
|
136
|
+
## Task-level user preferences
|
|
137
|
+
|
|
138
|
+
Treat instructions about update frequency, verbosity, pacing, detail level, and presentation style as active task-level preferences. Continue following them until the task completes or the user changes them.
|
|
139
|
+
|
|
140
|
+
## Voice behavior
|
|
141
|
+
|
|
142
|
+
Keep direct answers to one or two short sentences by default. Ask one clarification question at a time. Give tool or execution results as the outcome first, followed only by the next useful action.
|
|
143
|
+
|
|
144
|
+
Only act on audio you understand with sufficient confidence. If speech is unclear, incomplete, ambiguous, or likely background conversation, ask for a brief clarification instead of guessing, reasoning from missing words, or using the backend.
|
|
145
|
+
|
|
146
|
+
## Communication style
|
|
147
|
+
|
|
148
|
+
When the user makes a clear request, proceed directly. Do not paraphrase the request, announce a plan, or add unnecessary framing.
|
|
149
|
+
|
|
150
|
+
Avoid repetitive confirmation, filler, re-acknowledgement, and obvious play-by-play. By default, share progress only when it is brief, grounded, and genuinely useful.`;
|
|
151
|
+
|
|
152
|
+
const REALTIME_INSTRUCTIONS_MAX_BYTES = 32_768;
|
|
153
|
+
|
|
154
|
+
export function openGeniRealtimeInstructions(additional?: string): string {
|
|
155
|
+
const trimmed = additional?.trim();
|
|
156
|
+
if (!trimmed) return OPENGENI_REALTIME_BASE_INSTRUCTIONS;
|
|
157
|
+
const heading =
|
|
158
|
+
"\n\n## Additional realtime guidance\nFollow the guidance below for this conversation unless it conflicts with the operating, delegation, safety, permission, or context-handling rules above.\n";
|
|
159
|
+
const prefix = `${OPENGENI_REALTIME_BASE_INSTRUCTIONS}${heading}`;
|
|
160
|
+
const remaining = REALTIME_INSTRUCTIONS_MAX_BYTES - Buffer.byteLength(prefix, "utf8");
|
|
161
|
+
return `${prefix}${takeUtf8Head(trimmed, Math.max(0, remaining))}`;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function takeUtf8Head(value: string, maximumBytes: number): string {
|
|
165
|
+
if (Buffer.byteLength(value, "utf8") <= maximumBytes) return value;
|
|
166
|
+
const bytes = Buffer.from(value, "utf8");
|
|
167
|
+
let end = maximumBytes;
|
|
168
|
+
while (end > 0 && (bytes[end]! & 0xc0) === 0x80) end -= 1;
|
|
169
|
+
return bytes.subarray(0, end).toString("utf8");
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Credential-bound server broker. Selection is identical to a turn (pin then
|
|
174
|
+
* workspace active), and only a provider 401 permits one forced refresh/retry.
|
|
175
|
+
*/
|
|
176
|
+
export async function brokerSessionCodexRealtime(
|
|
177
|
+
deps: CodexRealtimeBrokerDependencies,
|
|
178
|
+
input: CodexRealtimeBrokerInput,
|
|
179
|
+
): Promise<CodexRealtimeProviderAnswer> {
|
|
180
|
+
if (!deps.enabled) {
|
|
181
|
+
throw new CodexRealtimeBrokerError(
|
|
182
|
+
"subscription_disabled",
|
|
183
|
+
"Connected Codex subscription realtime is disabled",
|
|
184
|
+
);
|
|
185
|
+
}
|
|
186
|
+
const selection = await deps.loadSelection();
|
|
187
|
+
const credentialId = selectCodexCredentialId({
|
|
188
|
+
sessionPinnedCredentialId: selection.pinnedCredentialId,
|
|
189
|
+
activeCredentialId: selection.activeCredentialId,
|
|
190
|
+
connectedIds: selection.connectedCredentialIds,
|
|
191
|
+
});
|
|
192
|
+
if (!credentialId) {
|
|
193
|
+
throw new CodexRealtimeBrokerError(
|
|
194
|
+
"credential_unavailable",
|
|
195
|
+
"No connected Codex subscription is available for this session",
|
|
196
|
+
);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// This comes from active session_history_items after lifecycle owner proof;
|
|
200
|
+
// it is not accepted from the browser request and is replayed identically on
|
|
201
|
+
// the one authentication-only retry below.
|
|
202
|
+
const initialItems = await deps.loadInitialItems();
|
|
203
|
+
|
|
204
|
+
const resolver = deps.tokenResolver(credentialId);
|
|
205
|
+
let token: Omit<CodexAuthHeaders, "clientVersion">;
|
|
206
|
+
try {
|
|
207
|
+
token = await resolver.getToken();
|
|
208
|
+
} catch (error) {
|
|
209
|
+
throw credentialError(error);
|
|
210
|
+
}
|
|
211
|
+
const callInput: CodexRealtimeCallInput = {
|
|
212
|
+
...input.request,
|
|
213
|
+
sessionId: input.sessionId,
|
|
214
|
+
initialItems,
|
|
215
|
+
instructions: openGeniRealtimeInstructions(input.request.instructions),
|
|
216
|
+
};
|
|
217
|
+
try {
|
|
218
|
+
return await deps.createCall({ ...token, clientVersion: CODEX_CLIENT_VERSION }, callInput, {
|
|
219
|
+
signal: input.signal,
|
|
220
|
+
});
|
|
221
|
+
} catch (error) {
|
|
222
|
+
if (!(error instanceof CodexRealtimeError) || error.code !== "authentication") {
|
|
223
|
+
throw brokerProviderError(error);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// A provider 401 is the only replay-safe credential lifecycle exception: the
|
|
228
|
+
// call was rejected before authentication, so force exactly one refresh and
|
|
229
|
+
// repeat the same SDP request once. No other provider outcome is retried.
|
|
230
|
+
try {
|
|
231
|
+
token = await resolver.refresh();
|
|
232
|
+
} catch (error) {
|
|
233
|
+
throw credentialError(error);
|
|
234
|
+
}
|
|
235
|
+
try {
|
|
236
|
+
return await deps.createCall({ ...token, clientVersion: CODEX_CLIENT_VERSION }, callInput, {
|
|
237
|
+
signal: input.signal,
|
|
238
|
+
});
|
|
239
|
+
} catch (error) {
|
|
240
|
+
if (error instanceof CodexRealtimeError && error.code === "authentication") {
|
|
241
|
+
throw new CodexRealtimeBrokerError(
|
|
242
|
+
"reconnect_required",
|
|
243
|
+
"Codex subscription must be reconnected for realtime",
|
|
244
|
+
error.providerStatus,
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
throw brokerProviderError(error);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/** Bind the pure broker to OpenGeni's encrypted DB credential lifecycle. */
|
|
252
|
+
export function buildSessionCodexRealtimeBroker(
|
|
253
|
+
db: Database,
|
|
254
|
+
settings: Settings,
|
|
255
|
+
workspaceId: string,
|
|
256
|
+
sessionId: string,
|
|
257
|
+
fetchImpl: CodexFetch = fetch,
|
|
258
|
+
): (input: Omit<CodexRealtimeBrokerInput, "sessionId">) => Promise<CodexRealtimeProviderAnswer> {
|
|
259
|
+
return async (input) =>
|
|
260
|
+
await brokerSessionCodexRealtime(
|
|
261
|
+
{
|
|
262
|
+
enabled: settings.codexSubscriptionEnabled,
|
|
263
|
+
loadSelection: async () => {
|
|
264
|
+
const [sessionState, status, accounts] = await Promise.all([
|
|
265
|
+
getSessionCodexState(db, workspaceId, sessionId),
|
|
266
|
+
getCodexCredentialStatus(db, workspaceId),
|
|
267
|
+
listCodexAccountStatuses(db, workspaceId),
|
|
268
|
+
]);
|
|
269
|
+
if (!sessionState) {
|
|
270
|
+
throw new CodexRealtimeBrokerError(
|
|
271
|
+
"credential_unavailable",
|
|
272
|
+
"Session is unavailable for Codex realtime",
|
|
273
|
+
);
|
|
274
|
+
}
|
|
275
|
+
return {
|
|
276
|
+
pinnedCredentialId: sessionState.pinnedCredentialId,
|
|
277
|
+
activeCredentialId: status?.credentialId ?? null,
|
|
278
|
+
connectedCredentialIds: new Set(
|
|
279
|
+
accounts
|
|
280
|
+
.filter((account) => account.status === "active")
|
|
281
|
+
.map((account) => account.id),
|
|
282
|
+
),
|
|
283
|
+
};
|
|
284
|
+
},
|
|
285
|
+
loadInitialItems: async () => {
|
|
286
|
+
const [history, continuity] = await Promise.all([
|
|
287
|
+
getActiveSessionHistoryItems(db, workspaceId, sessionId),
|
|
288
|
+
getSessionRealtimeContinuityEntries(db, workspaceId, sessionId),
|
|
289
|
+
]);
|
|
290
|
+
return projectSessionRealtimeInitialItems(history, continuity);
|
|
291
|
+
},
|
|
292
|
+
tokenResolver: (credentialId) =>
|
|
293
|
+
buildCodexTokenResolver(db, settings, workspaceId, credentialId),
|
|
294
|
+
createCall: async (auth, callInput, options) =>
|
|
295
|
+
await createCodexRealtimeCall(auth, callInput, fetchImpl, options),
|
|
296
|
+
},
|
|
297
|
+
{ ...input, sessionId },
|
|
298
|
+
);
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
function credentialError(error: unknown): CodexRealtimeBrokerError {
|
|
302
|
+
if (error instanceof CodexReloginRequired) {
|
|
303
|
+
return new CodexRealtimeBrokerError(
|
|
304
|
+
"reconnect_required",
|
|
305
|
+
"Codex subscription must be reconnected for realtime",
|
|
306
|
+
);
|
|
307
|
+
}
|
|
308
|
+
return new CodexRealtimeBrokerError(
|
|
309
|
+
"credential_unavailable",
|
|
310
|
+
"Codex subscription credential is unavailable",
|
|
311
|
+
);
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
function brokerProviderError(error: unknown): CodexRealtimeBrokerError {
|
|
315
|
+
if (!(error instanceof CodexRealtimeError)) {
|
|
316
|
+
return new CodexRealtimeBrokerError("network_error", "Codex realtime provider request failed");
|
|
317
|
+
}
|
|
318
|
+
const reason: CodexRealtimeBrokerFailureReason =
|
|
319
|
+
error.code === "invalid_request"
|
|
320
|
+
? "invalid_request"
|
|
321
|
+
: error.code === "incompatible"
|
|
322
|
+
? "incompatible"
|
|
323
|
+
: error.code === "authentication"
|
|
324
|
+
? "reconnect_required"
|
|
325
|
+
: error.code === "entitlement"
|
|
326
|
+
? "entitlement_denied"
|
|
327
|
+
: error.code === "rate_limited"
|
|
328
|
+
? "rate_limited"
|
|
329
|
+
: error.code === "invalid_response"
|
|
330
|
+
? "invalid_provider_response"
|
|
331
|
+
: error.code === "timeout"
|
|
332
|
+
? "timeout"
|
|
333
|
+
: error.code === "cancelled"
|
|
334
|
+
? "cancelled"
|
|
335
|
+
: error.code === "network"
|
|
336
|
+
? "network_error"
|
|
337
|
+
: "provider_error";
|
|
338
|
+
return new CodexRealtimeBrokerError(reason, safeBrokerMessage(reason), error.providerStatus);
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
function safeBrokerMessage(reason: CodexRealtimeBrokerFailureReason): string {
|
|
342
|
+
switch (reason) {
|
|
343
|
+
case "invalid_request":
|
|
344
|
+
return "Codex realtime request is invalid";
|
|
345
|
+
case "incompatible":
|
|
346
|
+
return "Connected Codex subscription is not compatible with realtime V3";
|
|
347
|
+
case "reconnect_required":
|
|
348
|
+
return "Codex subscription must be reconnected for realtime";
|
|
349
|
+
case "entitlement_denied":
|
|
350
|
+
return "Connected Codex subscription does not include realtime access";
|
|
351
|
+
case "rate_limited":
|
|
352
|
+
return "Codex realtime is rate limited";
|
|
353
|
+
case "invalid_provider_response":
|
|
354
|
+
return "Codex realtime returned an incompatible response";
|
|
355
|
+
case "timeout":
|
|
356
|
+
return "Codex realtime negotiation timed out";
|
|
357
|
+
case "cancelled":
|
|
358
|
+
return "Codex realtime negotiation was cancelled";
|
|
359
|
+
case "network_error":
|
|
360
|
+
case "provider_error":
|
|
361
|
+
return "Codex realtime provider request failed";
|
|
362
|
+
case "subscription_disabled":
|
|
363
|
+
return "Connected Codex subscription realtime is disabled";
|
|
364
|
+
case "credential_unavailable":
|
|
365
|
+
return "No connected Codex subscription is available for this session";
|
|
366
|
+
}
|
|
367
|
+
}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import {
|
|
2
|
+
VERCEL_AI_GATEWAY_AI_SDK_BASE_URL,
|
|
3
|
+
VERCEL_AI_GATEWAY_BASE_URL,
|
|
4
|
+
resolveAiGatewayRealtimeModel,
|
|
5
|
+
type Settings,
|
|
6
|
+
} from "@opengeni/config";
|
|
7
|
+
import type { GatewayRealtimeInitialItem, SessionRealtimeModel } from "@opengeni/contracts";
|
|
8
|
+
import {
|
|
9
|
+
getActiveSessionHistoryItems,
|
|
10
|
+
getSessionRealtimeContinuityEntries,
|
|
11
|
+
loadWorkspaceVercelAiGatewayApiKey,
|
|
12
|
+
type Database,
|
|
13
|
+
} from "@opengeni/db";
|
|
14
|
+
|
|
15
|
+
import { openGeniRealtimeInstructions } from "./codex-realtime";
|
|
16
|
+
import { projectSessionRealtimeInitialItems } from "./session-realtime-context";
|
|
17
|
+
|
|
18
|
+
export class GatewayRealtimeBrokerError extends Error {
|
|
19
|
+
constructor(
|
|
20
|
+
readonly code:
|
|
21
|
+
| "model_unavailable"
|
|
22
|
+
| "credential_unavailable"
|
|
23
|
+
| "provider_error"
|
|
24
|
+
| "invalid_provider_response",
|
|
25
|
+
message: string,
|
|
26
|
+
readonly providerStatus: number | null = null,
|
|
27
|
+
) {
|
|
28
|
+
super(message);
|
|
29
|
+
this.name = "GatewayRealtimeBrokerError";
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export type GatewayRealtimeConnectionSecret = {
|
|
34
|
+
token: string;
|
|
35
|
+
url: string;
|
|
36
|
+
upstreamModelId: string;
|
|
37
|
+
expiresAt: number | null;
|
|
38
|
+
initialItems: GatewayRealtimeInitialItem[];
|
|
39
|
+
instructions: string;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
export async function createGatewayRealtimeConnectionSecret(input: {
|
|
43
|
+
db: Database;
|
|
44
|
+
settings: Settings;
|
|
45
|
+
workspaceId: string;
|
|
46
|
+
sessionId: string;
|
|
47
|
+
model: SessionRealtimeModel;
|
|
48
|
+
fetchImpl?: typeof fetch;
|
|
49
|
+
}): Promise<GatewayRealtimeConnectionSecret> {
|
|
50
|
+
const resolved = resolveAiGatewayRealtimeModel(input.model);
|
|
51
|
+
if (!resolved) {
|
|
52
|
+
throw new GatewayRealtimeBrokerError(
|
|
53
|
+
"model_unavailable",
|
|
54
|
+
"The selected model is not an AI Gateway realtime model",
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
const apiKey =
|
|
58
|
+
resolved.source === "managed"
|
|
59
|
+
? input.settings.vercelAiGatewayApiKey
|
|
60
|
+
: await loadWorkspaceVercelAiGatewayApiKey(input.db, input.settings, input.workspaceId);
|
|
61
|
+
if (!apiKey) {
|
|
62
|
+
throw new GatewayRealtimeBrokerError(
|
|
63
|
+
"credential_unavailable",
|
|
64
|
+
resolved.source === "managed"
|
|
65
|
+
? "OpenGeni Gateway voice is not configured"
|
|
66
|
+
: "The workspace AI Gateway connection is unavailable",
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const [history, continuity, minted] = await Promise.all([
|
|
71
|
+
getActiveSessionHistoryItems(input.db, input.workspaceId, input.sessionId),
|
|
72
|
+
getSessionRealtimeContinuityEntries(input.db, input.workspaceId, input.sessionId),
|
|
73
|
+
mintGatewayClientSecret({
|
|
74
|
+
apiKey,
|
|
75
|
+
upstreamModelId: resolved.upstreamModelId,
|
|
76
|
+
fetchImpl: input.fetchImpl ?? fetch,
|
|
77
|
+
}),
|
|
78
|
+
]);
|
|
79
|
+
return {
|
|
80
|
+
...minted,
|
|
81
|
+
upstreamModelId: resolved.upstreamModelId,
|
|
82
|
+
initialItems: projectSessionRealtimeInitialItems(history, continuity),
|
|
83
|
+
instructions: openGeniRealtimeInstructions(),
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
async function mintGatewayClientSecret(input: {
|
|
88
|
+
apiKey: string;
|
|
89
|
+
upstreamModelId: string;
|
|
90
|
+
fetchImpl: typeof fetch;
|
|
91
|
+
}): Promise<{ token: string; url: string; expiresAt: number | null }> {
|
|
92
|
+
const mintUrl = new URL("/v1/realtime/client-secrets", VERCEL_AI_GATEWAY_BASE_URL);
|
|
93
|
+
let response: Response;
|
|
94
|
+
try {
|
|
95
|
+
response = await input.fetchImpl(mintUrl, {
|
|
96
|
+
method: "POST",
|
|
97
|
+
headers: {
|
|
98
|
+
authorization: `Bearer ${input.apiKey}`,
|
|
99
|
+
"content-type": "application/json",
|
|
100
|
+
"ai-gateway-auth-method": "api-key",
|
|
101
|
+
"ai-gateway-protocol-version": "0.0.1",
|
|
102
|
+
},
|
|
103
|
+
body: JSON.stringify({ model: input.upstreamModelId, expiresIn: 120 }),
|
|
104
|
+
});
|
|
105
|
+
} catch {
|
|
106
|
+
throw new GatewayRealtimeBrokerError(
|
|
107
|
+
"provider_error",
|
|
108
|
+
"AI Gateway realtime token request failed",
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
if (!response.ok) {
|
|
112
|
+
throw new GatewayRealtimeBrokerError(
|
|
113
|
+
response.status === 401 || response.status === 403
|
|
114
|
+
? "credential_unavailable"
|
|
115
|
+
: "provider_error",
|
|
116
|
+
response.status === 401 || response.status === 403
|
|
117
|
+
? "AI Gateway credentials were rejected"
|
|
118
|
+
: "AI Gateway realtime token request failed",
|
|
119
|
+
response.status,
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
const body = (await response.json().catch(() => null)) as Record<string, unknown> | null;
|
|
123
|
+
const token = body?.token;
|
|
124
|
+
const expiresAt = body?.expiresAt;
|
|
125
|
+
if (
|
|
126
|
+
typeof token !== "string" ||
|
|
127
|
+
token.length === 0 ||
|
|
128
|
+
(expiresAt !== undefined && expiresAt !== null && typeof expiresAt !== "number")
|
|
129
|
+
) {
|
|
130
|
+
throw new GatewayRealtimeBrokerError(
|
|
131
|
+
"invalid_provider_response",
|
|
132
|
+
"AI Gateway returned an invalid realtime token",
|
|
133
|
+
response.status,
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
const url = new URL(`${VERCEL_AI_GATEWAY_AI_SDK_BASE_URL.replace(/^http/, "ws")}/realtime-model`);
|
|
137
|
+
url.searchParams.set("ai-model-id", input.upstreamModelId);
|
|
138
|
+
return {
|
|
139
|
+
token,
|
|
140
|
+
url: url.toString(),
|
|
141
|
+
expiresAt: typeof expiresAt === "number" ? expiresAt : null,
|
|
142
|
+
};
|
|
143
|
+
}
|
|
@@ -3,7 +3,6 @@ import type { Settings } from "@opengeni/config";
|
|
|
3
3
|
import {
|
|
4
4
|
GOOGLE_DRIVE_CREDENTIAL_LABEL,
|
|
5
5
|
GOOGLE_DRIVE_CREDENTIAL_ROLE,
|
|
6
|
-
GOOGLE_DRIVE_METADATA_READONLY_SCOPE,
|
|
7
6
|
GOOGLE_DRIVE_PROVIDER_DOMAIN,
|
|
8
7
|
GOOGLE_DRIVE_READONLY_SCOPE,
|
|
9
8
|
GoogleDriveBrowseItem,
|
|
@@ -11,6 +10,8 @@ import {
|
|
|
11
10
|
GoogleDriveConnectionMetadata,
|
|
12
11
|
GoogleDriveOAuthStartResponse,
|
|
13
12
|
SaveGoogleDriveSourceRequest,
|
|
13
|
+
googleDriveOAuthScopeDecision,
|
|
14
|
+
googleDriveScopesAllowCapability,
|
|
14
15
|
type GoogleDriveOAuthStartRequest,
|
|
15
16
|
} from "@opengeni/contracts/google-drive";
|
|
16
17
|
import { hasPermission, requireEnvironmentEncryption } from "@opengeni/core";
|
|
@@ -170,7 +171,11 @@ export async function completeGoogleDriveOAuthCallback(
|
|
|
170
171
|
},
|
|
171
172
|
fetchImpl,
|
|
172
173
|
);
|
|
173
|
-
|
|
174
|
+
const scopeDecision = googleDriveOAuthScopeDecision(token.scopes);
|
|
175
|
+
if (
|
|
176
|
+
scopeDecision.accessMode !== "readonly" ||
|
|
177
|
+
!scopeDecision.capabilities.includes("recursive_source_sync")
|
|
178
|
+
) {
|
|
174
179
|
throw new GoogleDriveCallbackError("scope_not_granted");
|
|
175
180
|
}
|
|
176
181
|
const identity = await verifyGoogleDriveIdentity(token.accessToken, fetchImpl);
|
|
@@ -230,7 +235,7 @@ export async function completeGoogleDriveOAuthCallback(
|
|
|
230
235
|
googleEmail: identity.emailAddress,
|
|
231
236
|
googleDisplayName: identity.displayName,
|
|
232
237
|
verifiedAt: new Date().toISOString(),
|
|
233
|
-
accessMode:
|
|
238
|
+
accessMode: scopeDecision.accessMode,
|
|
234
239
|
...(previousMetadata?.selectedSources
|
|
235
240
|
? { selectedSources: previousMetadata.selectedSources }
|
|
236
241
|
: previousMetadata?.selectedSource
|
|
@@ -302,7 +307,7 @@ export async function browseGoogleDrive(
|
|
|
302
307
|
if (!connection) {
|
|
303
308
|
throw new HTTPException(404, { message: "Google Drive connection not found" });
|
|
304
309
|
}
|
|
305
|
-
|
|
310
|
+
requireGoogleDriveSourceConnection(connection, input.subjectId);
|
|
306
311
|
const parentId = validDriveId(input.parentId, "parentId");
|
|
307
312
|
const currentItem = await resolveGoogleDriveBoundaryItem(deps, {
|
|
308
313
|
workspaceId: input.workspaceId,
|
|
@@ -377,7 +382,7 @@ export async function saveGoogleDriveSource(
|
|
|
377
382
|
if (!existing) {
|
|
378
383
|
throw new HTTPException(404, { message: "Google Drive connection not found" });
|
|
379
384
|
}
|
|
380
|
-
|
|
385
|
+
requireGoogleDriveSourceConnection(existing, input.subjectId);
|
|
381
386
|
const verifiedSources = [];
|
|
382
387
|
for (const source of payload.sources) {
|
|
383
388
|
const sourceId = validDriveId(source.id, "source.id");
|
|
@@ -405,7 +410,7 @@ export async function saveGoogleDriveSource(
|
|
|
405
410
|
input.connectionId,
|
|
406
411
|
input.subjectId,
|
|
407
412
|
)) ?? existing;
|
|
408
|
-
const latestMetadata =
|
|
413
|
+
const latestMetadata = requireGoogleDriveSourceConnection(latest, input.subjectId);
|
|
409
414
|
const updated = await updateConnection(deps.db, {
|
|
410
415
|
workspaceId: input.workspaceId,
|
|
411
416
|
connectionId: latest.id,
|
|
@@ -546,15 +551,20 @@ function requireGoogleDriveConnection(
|
|
|
546
551
|
) {
|
|
547
552
|
throw new HTTPException(422, { message: "connection is not this user's Google Drive" });
|
|
548
553
|
}
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
554
|
+
return parsed.data;
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
function requireGoogleDriveSourceConnection(
|
|
558
|
+
connection: Parameters<typeof requireGoogleDriveConnection>[0],
|
|
559
|
+
subjectId: string,
|
|
560
|
+
) {
|
|
561
|
+
const metadata = requireGoogleDriveConnection(connection, subjectId);
|
|
562
|
+
if (!googleDriveScopesAllowCapability(connection.grantedScopes, "recursive_source_sync")) {
|
|
553
563
|
throw new HTTPException(401, {
|
|
554
|
-
message: "Google Drive needs to be reconnected with
|
|
564
|
+
message: "Google Drive needs to be reconnected with selected-source read access",
|
|
555
565
|
});
|
|
556
566
|
}
|
|
557
|
-
return
|
|
567
|
+
return metadata;
|
|
558
568
|
}
|
|
559
569
|
|
|
560
570
|
function readGoogleDriveOAuthState(
|