@stratusagent/gateway 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/index.d.ts +90 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +580 -37
- package/dist/index.js.map +1 -1
- package/package.json +6 -5
- package/dist/tsconfig.test.tsbuildinfo +0 -1
package/dist/index.js
CHANGED
|
@@ -1,13 +1,25 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto';
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
2
2
|
import { chmodSync, mkdirSync } from 'node:fs';
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
import { DatabaseSync } from 'node:sqlite';
|
|
5
|
-
import { AgentRegistry, AgentRunner, EventBus, RunAbortedError, ToolRegistry, } from '@stratusagent/core';
|
|
5
|
+
import { AgentRegistry, AgentRunner, EventBus, RunAbortedError, ToolRegistry, readPendingApproval, } from '@stratusagent/core';
|
|
6
6
|
import { createDelegateTool, createRememberTool, } from '@stratusagent/agents';
|
|
7
7
|
import { createLocalCommandExecutor } from '@stratusagent/executor-local';
|
|
8
8
|
import { createDemoTool, createFileMemoryStore, createRuntimeProvider, DEFAULT_STRATUS_AGENT, loadRosterSouls, FALLBACK_ACTIVE_METADATA_KEY, loadSoulFile, memoryFilePath, migrateLegacyMemory, ConfigFileError, loadConfigFile, readNonEmptyString, resolveConfigLocation, resolveConfiguredSoul, resolveRuntimeConfig, stratusHomePath, withLegacyDefaultMemories, } from '@stratusagent/state';
|
|
9
9
|
const SESSIONS_DB_FILENAME = 'sessions.db';
|
|
10
10
|
const DEFAULT_IDLE_TIMEOUT_MS = 120_000;
|
|
11
|
+
/**
|
|
12
|
+
* How long a parked call waits for a person. Long enough to survive a
|
|
13
|
+
* meeting, short enough that a request nobody saw does not hold a turn (and
|
|
14
|
+
* the Slack thread it is answering) open indefinitely.
|
|
15
|
+
*/
|
|
16
|
+
const DEFAULT_APPROVAL_TIMEOUT_MS = 900_000;
|
|
17
|
+
/**
|
|
18
|
+
* Node's largest `setTimeout` delay. Anything above it does not become a
|
|
19
|
+
* long wait — it silently becomes a 1ms one, so a config asking for a
|
|
20
|
+
* 30-day approval window would expire every request almost immediately.
|
|
21
|
+
*/
|
|
22
|
+
const MAX_TIMER_DELAY_MS = 2_147_483_647;
|
|
11
23
|
export class SqliteSessionStore {
|
|
12
24
|
db;
|
|
13
25
|
constructor(filePath, options = {}) {
|
|
@@ -84,6 +96,17 @@ export class SqliteSessionStore {
|
|
|
84
96
|
.prepare('INSERT OR REPLACE INTO sessions (id, agent_id, status, body, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)')
|
|
85
97
|
.run(updated.id, updated.agent.id, updated.status, JSON.stringify(updated), updated.createdAt, updated.updatedAt);
|
|
86
98
|
}
|
|
99
|
+
/**
|
|
100
|
+
* Session ids in a state, oldest first — the index a restarting daemon
|
|
101
|
+
* sweeps for turns parked on a human. The `status` column carries it, so
|
|
102
|
+
* no conversation body is deserialized to answer the question.
|
|
103
|
+
*/
|
|
104
|
+
async listIdsByStatus(status) {
|
|
105
|
+
const rows = this.db
|
|
106
|
+
.prepare('SELECT id FROM sessions WHERE status = ? ORDER BY updated_at ASC')
|
|
107
|
+
.all(status);
|
|
108
|
+
return rows.map((row) => row.id);
|
|
109
|
+
}
|
|
87
110
|
/** Newest-first session listing for one agent (or all agents). */
|
|
88
111
|
list(agentId) {
|
|
89
112
|
const rows = (agentId
|
|
@@ -160,6 +183,16 @@ export const applySoulPins = (pins, selection, env, context) => {
|
|
|
160
183
|
}
|
|
161
184
|
return { selection, env: { ...env, processEnv } };
|
|
162
185
|
};
|
|
186
|
+
/**
|
|
187
|
+
* Recorded on a turn the previous process was still running when it died.
|
|
188
|
+
*
|
|
189
|
+
* Exported because "distinguishable" is the whole point: a surface, a
|
|
190
|
+
* test, or an operator reading `lastError` has to be able to tell a turn
|
|
191
|
+
* nobody finished apart from one a provider or a tool actually failed, and
|
|
192
|
+
* matching on prose that lives in one place beats each caller inventing
|
|
193
|
+
* its own guess at the wording.
|
|
194
|
+
*/
|
|
195
|
+
export const ABANDONED_TURN_ERROR = 'stratusd stopped while this turn was still running; it was not resumed. Send the message again.';
|
|
163
196
|
/**
|
|
164
197
|
* The always-on Stratus process. One gateway holds the roster, the durable
|
|
165
198
|
* session store, the shared event bus, and a pool of runners keyed by
|
|
@@ -181,6 +214,253 @@ export const createGateway = (options = {}) => {
|
|
|
181
214
|
? new SqliteSessionStore(options.sessionDbPath)
|
|
182
215
|
: new SqliteSessionStore(path.join(stratusHomePath(env), SESSIONS_DB_FILENAME), { ownedDirectory: true });
|
|
183
216
|
const memory = withLegacyDefaultMemories(createFileMemoryStore(memoryFilePath(env)));
|
|
217
|
+
// ---- approval brokering -------------------------------------------------
|
|
218
|
+
const approvalTimeoutMs = options.approvalTimeoutMs ?? DEFAULT_APPROVAL_TIMEOUT_MS;
|
|
219
|
+
// Refused at construction, not absorbed: a negative or NaN timeout fails
|
|
220
|
+
// the `> 0` test below and quietly means "never expire" — the one
|
|
221
|
+
// behavior documented for an explicit 0, and the last one a caller who
|
|
222
|
+
// typed a bad number wants. Config-file values are already rejected with
|
|
223
|
+
// a better message; this catches the programmatic path.
|
|
224
|
+
if (!Number.isFinite(approvalTimeoutMs) || approvalTimeoutMs < 0) {
|
|
225
|
+
throw new Error(`approvalTimeoutMs must be a non-negative number of milliseconds (0 to wait indefinitely); received ${String(options.approvalTimeoutMs)}.`);
|
|
226
|
+
}
|
|
227
|
+
// Clamped, not trusted: a delay past Node's timer range is not a long
|
|
228
|
+
// wait, it is a 1ms one, so an over-large value would turn every approval
|
|
229
|
+
// into an instant expiry — the opposite of what it asked for. The CLI
|
|
230
|
+
// rejects these at config load with a better message; this is the
|
|
231
|
+
// backstop for a programmatic caller.
|
|
232
|
+
const effectiveApprovalTimeoutMs = Math.min(approvalTimeoutMs, MAX_TIMER_DELAY_MS);
|
|
233
|
+
if (approvalTimeoutMs > MAX_TIMER_DELAY_MS) {
|
|
234
|
+
warn(`approval timeout ${approvalTimeoutMs}ms is above Node's maximum timer delay; using ${MAX_TIMER_DELAY_MS}ms (~24.8 days)`);
|
|
235
|
+
}
|
|
236
|
+
const pendingApprovals = new Map();
|
|
237
|
+
/**
|
|
238
|
+
* In-flight `tool.approval-resolved` emissions. `EventBus.emit` awaits its
|
|
239
|
+
* subscribers in order, so one async handler ahead of a channel adapter
|
|
240
|
+
* suspends the emission before the adapter ever sees it. Nothing today is
|
|
241
|
+
* async in front of one — which is exactly the problem: the shutdown
|
|
242
|
+
* guarantee below would rest on that staying true, and the first async
|
|
243
|
+
* subscriber anyone adds would silently leave live-looking buttons in a
|
|
244
|
+
* workspace the daemon has left.
|
|
245
|
+
*/
|
|
246
|
+
const approvalEmissions = new Set();
|
|
247
|
+
/**
|
|
248
|
+
* Publishes one parked call and settles when somebody answers it, the
|
|
249
|
+
* request expires, or the turn is cancelled.
|
|
250
|
+
*
|
|
251
|
+
* All three endings run through the same `settle`, which removes the
|
|
252
|
+
* request from the registry BEFORE resolving. That ordering is the whole
|
|
253
|
+
* point: a request that is no longer pending cannot be resolved twice,
|
|
254
|
+
* so a click racing the timeout — or arriving after an abort — is refused
|
|
255
|
+
* rather than executing a tool for a turn that has already moved on.
|
|
256
|
+
*/
|
|
257
|
+
const requestApproval = (request) => new Promise((resolve) => {
|
|
258
|
+
// A shutdown denies what is parked once, at the top of stop(). Turns
|
|
259
|
+
// already running keep going through the drain, though, and one of them
|
|
260
|
+
// can reach a gated tool AFTER that snapshot — finishing a provider
|
|
261
|
+
// call, or moving to the next call in the same response. Parking it
|
|
262
|
+
// would deadlock the drain against a question nobody is left to answer:
|
|
263
|
+
// stop() waits for the turn, the turn waits for the approval, and the
|
|
264
|
+
// approval waits out its timeout — forever when there is none.
|
|
265
|
+
if (stopping) {
|
|
266
|
+
resolve('deny');
|
|
267
|
+
void bus.emit({
|
|
268
|
+
type: 'tool.approval-resolved',
|
|
269
|
+
sessionId: request.session.id,
|
|
270
|
+
requestId: randomUUID(),
|
|
271
|
+
answer: 'deny',
|
|
272
|
+
reason: 'cancelled',
|
|
273
|
+
});
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
276
|
+
const requestId = randomUUID();
|
|
277
|
+
let timer;
|
|
278
|
+
const onAbort = () => settle('deny', 'cancelled');
|
|
279
|
+
/**
|
|
280
|
+
* Settles once the request has finished being announced.
|
|
281
|
+
*
|
|
282
|
+
* Both emissions go through the same subscriber list, in order, and
|
|
283
|
+
* `EventBus.emit` awaits each one — so an async subscriber ahead of a
|
|
284
|
+
* channel can hold the announcement while a timeout or abort fires
|
|
285
|
+
* behind it, and the channel would hear that a request it has never
|
|
286
|
+
* heard of was resolved. It drops that, then renders the announcement
|
|
287
|
+
* when it finally arrives, leaving buttons for a settled request that
|
|
288
|
+
* nothing will ever retract.
|
|
289
|
+
*
|
|
290
|
+
* Chaining is the fix rather than remembering orphaned resolutions in
|
|
291
|
+
* the channel: a resolution that arrives before its request is not
|
|
292
|
+
* something a consumer can act on, so it should not be able to happen.
|
|
293
|
+
* Starts resolved, because a turn already aborted never announces at
|
|
294
|
+
* all and its denial should not wait on an emission that never runs.
|
|
295
|
+
*/
|
|
296
|
+
let announced = Promise.resolve();
|
|
297
|
+
const settle = (answer, reason, actor) => {
|
|
298
|
+
// Identity, not presence: a request that already settled is gone from
|
|
299
|
+
// the registry, and comparing against this exact function is what
|
|
300
|
+
// makes every ending — click, timeout, abort, shutdown — idempotent.
|
|
301
|
+
if (pendingApprovals.get(requestId) !== settle) {
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
304
|
+
pendingApprovals.delete(requestId);
|
|
305
|
+
if (timer) {
|
|
306
|
+
clearTimeout(timer);
|
|
307
|
+
}
|
|
308
|
+
request.signal?.removeEventListener('abort', onAbort);
|
|
309
|
+
resolve(answer);
|
|
310
|
+
const emitted = announced.then(() => bus.emit({
|
|
311
|
+
type: 'tool.approval-resolved',
|
|
312
|
+
sessionId: request.session.id,
|
|
313
|
+
requestId,
|
|
314
|
+
answer,
|
|
315
|
+
reason,
|
|
316
|
+
...(actor ? { actor } : {}),
|
|
317
|
+
}));
|
|
318
|
+
approvalEmissions.add(emitted);
|
|
319
|
+
void emitted.finally(() => approvalEmissions.delete(emitted));
|
|
320
|
+
};
|
|
321
|
+
pendingApprovals.set(requestId, settle);
|
|
322
|
+
// What is left of the window, not a fresh one: a re-asked request
|
|
323
|
+
// carries when it originally parked, and the elapsed time counts.
|
|
324
|
+
const alreadyWaitedMs = request.parkedAt ? Date.now() - Date.parse(request.parkedAt) : 0;
|
|
325
|
+
const remainingTimeoutMs = effectiveApprovalTimeoutMs > 0
|
|
326
|
+
? effectiveApprovalTimeoutMs - (Number.isFinite(alreadyWaitedMs) ? Math.max(0, alreadyWaitedMs) : 0)
|
|
327
|
+
: 0;
|
|
328
|
+
if (effectiveApprovalTimeoutMs > 0 && remainingTimeoutMs <= 0) {
|
|
329
|
+
// Nothing left to wait with. Settled here rather than armed with a
|
|
330
|
+
// zero timer, which `setTimeout` would still run a tick later — long
|
|
331
|
+
// enough to announce a request that is already over.
|
|
332
|
+
settle('deny', 'timeout');
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
335
|
+
if (remainingTimeoutMs > 0) {
|
|
336
|
+
// Deliberately NOT unref'd. A parked turn is real outstanding work,
|
|
337
|
+
// and its timer is the only thing that will ever finish it — letting
|
|
338
|
+
// the process exit out from under one would abandon the turn instead
|
|
339
|
+
// of denying it. Shutdown is handled where it belongs, in stop(),
|
|
340
|
+
// which settles every outstanding request before draining.
|
|
341
|
+
timer = setTimeout(() => settle('deny', 'timeout'), remainingTimeoutMs);
|
|
342
|
+
}
|
|
343
|
+
// The abort listener is attached before the request is announced, so a
|
|
344
|
+
// turn cancelled while the event is still being delivered still ends
|
|
345
|
+
// the request rather than leaving it pending forever.
|
|
346
|
+
if (request.signal) {
|
|
347
|
+
if (request.signal.aborted) {
|
|
348
|
+
settle('deny', 'cancelled');
|
|
349
|
+
return;
|
|
350
|
+
}
|
|
351
|
+
request.signal.addEventListener('abort', onAbort, { once: true });
|
|
352
|
+
}
|
|
353
|
+
// `announced` is assigned BEFORE the emission starts, because emit runs
|
|
354
|
+
// its first subscribers synchronously — one of them settling the
|
|
355
|
+
// request inline would otherwise chain onto the already-resolved
|
|
356
|
+
// placeholder and race the very announcement it is answering.
|
|
357
|
+
let markAnnounced = () => { };
|
|
358
|
+
announced = new Promise((resolveAnnounced) => {
|
|
359
|
+
markAnnounced = resolveAnnounced;
|
|
360
|
+
});
|
|
361
|
+
const announcing = bus.emit({
|
|
362
|
+
type: 'tool.approval-requested',
|
|
363
|
+
sessionId: request.session.id,
|
|
364
|
+
agentId: request.session.agent.id,
|
|
365
|
+
requestId,
|
|
366
|
+
call: request.call,
|
|
367
|
+
risk: request.risk,
|
|
368
|
+
...(request.session.metadata ? { metadata: request.session.metadata } : {}),
|
|
369
|
+
...(remainingTimeoutMs > 0
|
|
370
|
+
? { expiresAt: new Date(Date.now() + remainingTimeoutMs).toISOString() }
|
|
371
|
+
: {}),
|
|
372
|
+
});
|
|
373
|
+
// Drained at shutdown alongside the resolutions: a channel that has not
|
|
374
|
+
// yet been told a request exists cannot be told it was denied either.
|
|
375
|
+
approvalEmissions.add(announcing);
|
|
376
|
+
void announcing.finally(() => {
|
|
377
|
+
approvalEmissions.delete(announcing);
|
|
378
|
+
markAnnounced();
|
|
379
|
+
});
|
|
380
|
+
});
|
|
381
|
+
const resolveApproval = (input) => {
|
|
382
|
+
const settle = pendingApprovals.get(input.requestId);
|
|
383
|
+
if (!settle) {
|
|
384
|
+
return false;
|
|
385
|
+
}
|
|
386
|
+
settle(input.answer, input.reason ?? 'decided', input.actor);
|
|
387
|
+
return true;
|
|
388
|
+
};
|
|
389
|
+
/**
|
|
390
|
+
* How a turn's watchdog stops its own clock, registered while it runs.
|
|
391
|
+
*
|
|
392
|
+
* Recording that a tool phase is open is not enough on its own: the
|
|
393
|
+
* timer is only re-evaluated when an event arrives, so a phase opened by
|
|
394
|
+
* something that emits nothing — an approval policy handed straight to
|
|
395
|
+
* the gateway — would leave an already-armed timer running to fire
|
|
396
|
+
* mid-wait.
|
|
397
|
+
*/
|
|
398
|
+
const suspendWatchdog = new Map();
|
|
399
|
+
/**
|
|
400
|
+
* Approval waits in progress, per session.
|
|
401
|
+
*
|
|
402
|
+
* Deliberately a balanced counter and not a set of call ids. An earlier
|
|
403
|
+
* shape had the approval wrapper *open* a phase that the watchdog's
|
|
404
|
+
* event observer was expected to close, and that handoff was wrong in
|
|
405
|
+
* every way it could be: the observer does not exist when no watchdog is
|
|
406
|
+
* armed, so nothing closed it; a policy that threw closed nothing; and a
|
|
407
|
+
* stale entry did not merely leak, it suppressed the watchdog for the
|
|
408
|
+
* next turn on that session. Four separate exits, each patched in turn.
|
|
409
|
+
*
|
|
410
|
+
* A counter incremented and decremented in one `finally` cannot have any
|
|
411
|
+
* of those bugs, because the thing that opens the phase is the thing
|
|
412
|
+
* that closes it, in the same scope, on every path out.
|
|
413
|
+
*/
|
|
414
|
+
const approvalWaits = new Map();
|
|
415
|
+
const holdApprovalWait = (sessionId) => {
|
|
416
|
+
approvalWaits.set(sessionId, (approvalWaits.get(sessionId) ?? 0) + 1);
|
|
417
|
+
// Stop any clock already running: this wait produces no events, so
|
|
418
|
+
// nothing else would.
|
|
419
|
+
suspendWatchdog.get(sessionId)?.();
|
|
420
|
+
let released = false;
|
|
421
|
+
return () => {
|
|
422
|
+
if (released) {
|
|
423
|
+
return;
|
|
424
|
+
}
|
|
425
|
+
released = true;
|
|
426
|
+
const remaining = (approvalWaits.get(sessionId) ?? 1) - 1;
|
|
427
|
+
if (remaining > 0) {
|
|
428
|
+
approvalWaits.set(sessionId, remaining);
|
|
429
|
+
}
|
|
430
|
+
else {
|
|
431
|
+
approvalWaits.delete(sessionId);
|
|
432
|
+
}
|
|
433
|
+
};
|
|
434
|
+
};
|
|
435
|
+
const configuredApprovals = typeof options.approvals === 'function'
|
|
436
|
+
? options.approvals({ request: requestApproval })
|
|
437
|
+
: options.approvals;
|
|
438
|
+
/**
|
|
439
|
+
* Every approval opens a tool phase, whatever policy answers it.
|
|
440
|
+
*
|
|
441
|
+
* `tool.approval-requested` is the broker's event, so it exists only for
|
|
442
|
+
* the remote path; a policy handed straight to the gateway emits nothing
|
|
443
|
+
* at all, and `tool.called` comes only *after* the answer. Without this,
|
|
444
|
+
* a slow custom policy on a streaming provider that hosts its own loop
|
|
445
|
+
* would be indistinguishable from a stalled turn — the case this
|
|
446
|
+
* watchdog work exists to stop, arriving through a different door.
|
|
447
|
+
*/
|
|
448
|
+
const approvals = configuredApprovals
|
|
449
|
+
? {
|
|
450
|
+
async approve(context) {
|
|
451
|
+
const release = holdApprovalWait(context.session.id);
|
|
452
|
+
try {
|
|
453
|
+
return await configuredApprovals.approve(context);
|
|
454
|
+
}
|
|
455
|
+
finally {
|
|
456
|
+
// Every path out, including a policy that throws. What happens
|
|
457
|
+
// after the answer is the tool events' business, and they are
|
|
458
|
+
// observed by the watchdog itself.
|
|
459
|
+
release();
|
|
460
|
+
}
|
|
461
|
+
},
|
|
462
|
+
}
|
|
463
|
+
: undefined;
|
|
184
464
|
const registry = new AgentRegistry();
|
|
185
465
|
const sources = new Map();
|
|
186
466
|
// ---- roster -------------------------------------------------------------
|
|
@@ -249,15 +529,21 @@ export const createGateway = (options = {}) => {
|
|
|
249
529
|
// The built-in id is reserved BEFORE ordinary roster entries load: a
|
|
250
530
|
// roster file that happens to declare id "stratus" must not hijack
|
|
251
531
|
// the documented built-in fallback for agentId-less dispatches — the
|
|
252
|
-
//
|
|
532
|
+
// guard below skips it with a warning. Only the explicitly
|
|
253
533
|
// configured default soul may take the id over (defaultAgentId
|
|
254
534
|
// replaces a pathless source).
|
|
255
535
|
registerSource({ definition: { ...DEFAULT_STRATUS_AGENT } });
|
|
536
|
+
// Throws on two roster files claiming one id: that is a collision with
|
|
537
|
+
// no correct winner, and refusing to serve beats letting sort order
|
|
538
|
+
// decide whose sessions, memory, and credentials an agent inherits.
|
|
256
539
|
const entries = await loadRosterSouls(env, warn);
|
|
257
540
|
for (const entry of entries) {
|
|
258
541
|
if (sources.has(entry.soul.agent.id)) {
|
|
259
|
-
|
|
260
|
-
|
|
542
|
+
// Only reachable for the reserved built-in id now — roster-vs-
|
|
543
|
+
// roster duplicates never get this far. Reserved is not the same
|
|
544
|
+
// failure as duplicated: a soul may not take the fallback over,
|
|
545
|
+
// and must not be able to take the daemon down by trying.
|
|
546
|
+
warn(`agent id ${entry.soul.agent.id} is reserved for the built-in agent; ignoring ${entry.path}`);
|
|
261
547
|
continue;
|
|
262
548
|
}
|
|
263
549
|
registerSource({ definition: entry.soul.agent, soulPath: entry.path, soul: entry.soul });
|
|
@@ -357,7 +643,7 @@ export const createGateway = (options = {}) => {
|
|
|
357
643
|
provider,
|
|
358
644
|
tools,
|
|
359
645
|
executor: createLocalCommandExecutor(),
|
|
360
|
-
...(
|
|
646
|
+
...(approvals ? { approvals } : {}),
|
|
361
647
|
store,
|
|
362
648
|
bus,
|
|
363
649
|
agents: registry,
|
|
@@ -379,8 +665,16 @@ export const createGateway = (options = {}) => {
|
|
|
379
665
|
* that kills slow-but-healthy turns, so it stays off for them (their
|
|
380
666
|
* tool phases are covered by executor timeouts regardless).
|
|
381
667
|
*/
|
|
382
|
-
const fallbackStreamsDeltas = (fallback) =>
|
|
383
|
-
|
|
668
|
+
const fallbackStreamsDeltas = (fallback) =>
|
|
669
|
+
// The same two modes as a primary. A subscription fallback that is not
|
|
670
|
+
// recognised as streaming gets no watchdog at all once a session
|
|
671
|
+
// switches to it, so a stall there would run to the provider's own
|
|
672
|
+
// ten-minute timer instead of the idle timeout the operator set.
|
|
673
|
+
fallback.provider === 'anthropic' && Boolean(fallback.apiKey || fallback.authToken);
|
|
674
|
+
const streamsDeltas = (config) =>
|
|
675
|
+
// Both Anthropic modes stream now: an API key through the Messages
|
|
676
|
+
// API, a subscription token through the Agent SDK's partial messages.
|
|
677
|
+
config.provider === 'anthropic' && Boolean(config.apiKey || config.authToken);
|
|
384
678
|
/**
|
|
385
679
|
* Progress-based abort, armed only while the turn is actually awaiting a
|
|
386
680
|
* streaming provider — the one phase where silence means a stall. Tool
|
|
@@ -413,6 +707,18 @@ export const createGateway = (options = {}) => {
|
|
|
413
707
|
// the primary until a reset delta announces the sticky fallback switch.
|
|
414
708
|
let streamingActive = idleEnabled;
|
|
415
709
|
let pendingTools = 0;
|
|
710
|
+
/**
|
|
711
|
+
* Tool calls this turn has started and not finished.
|
|
712
|
+
*
|
|
713
|
+
* `pendingTools` above reads a count off `provider.response`, which
|
|
714
|
+
* only a provider that hands its calls back to the kernel loop emits.
|
|
715
|
+
* A provider hosting its own loop dispatches inside `generate`, so that
|
|
716
|
+
* count is zero for the whole of a hosted tool. These are the kernel's
|
|
717
|
+
* own tool events, which every path emits because every path runs its
|
|
718
|
+
* tools through the same executor — and they are turn-local, so an
|
|
719
|
+
* aborted turn takes them with it rather than leaving anything behind.
|
|
720
|
+
*/
|
|
721
|
+
const openToolCalls = new Set();
|
|
416
722
|
// Set at a text-only provider.response: the turn is wrapping up
|
|
417
723
|
// (saves, completion events); nothing left for the timer to guard.
|
|
418
724
|
let turnSettling = false;
|
|
@@ -450,9 +756,13 @@ export const createGateway = (options = {}) => {
|
|
|
450
756
|
}
|
|
451
757
|
switch (event.type) {
|
|
452
758
|
case 'provider.delta':
|
|
453
|
-
//
|
|
454
|
-
// the timer follows
|
|
455
|
-
|
|
759
|
+
// Only a *fallback* reset marks the mid-turn, session-sticky
|
|
760
|
+
// switch that the timer follows. A provider retrying its own
|
|
761
|
+
// attempt abandons its partial output too, but nothing about
|
|
762
|
+
// who is serving the turn changed — reading that as a switch
|
|
763
|
+
// would disarm the watchdog for the rest of a turn that is
|
|
764
|
+
// still running on the provider it started on.
|
|
765
|
+
if (event.delta.type === 'reset' && event.delta.reason === 'fallback') {
|
|
456
766
|
streamingActive = fallbackStreams;
|
|
457
767
|
}
|
|
458
768
|
break;
|
|
@@ -466,8 +776,23 @@ export const createGateway = (options = {}) => {
|
|
|
466
776
|
turnSettling = true;
|
|
467
777
|
}
|
|
468
778
|
break;
|
|
779
|
+
case 'tool.approval-requested':
|
|
780
|
+
// Redundant with the approval wrapper for a gateway-owned
|
|
781
|
+
// transport, and not redundant for a recovered turn, whose
|
|
782
|
+
// request is re-announced without going through a policy again.
|
|
783
|
+
openToolCalls.add(event.call.id);
|
|
784
|
+
break;
|
|
785
|
+
case 'tool.called':
|
|
786
|
+
openToolCalls.add(event.call.id);
|
|
787
|
+
break;
|
|
469
788
|
case 'tool.completed':
|
|
789
|
+
// Completion names the call through its result; denial carries
|
|
790
|
+
// the call itself. Both settle it.
|
|
791
|
+
openToolCalls.delete(event.result.callId);
|
|
792
|
+
pendingTools = Math.max(0, pendingTools - 1);
|
|
793
|
+
break;
|
|
470
794
|
case 'tool.denied':
|
|
795
|
+
openToolCalls.delete(event.call.id);
|
|
471
796
|
pendingTools = Math.max(0, pendingTools - 1);
|
|
472
797
|
break;
|
|
473
798
|
default:
|
|
@@ -479,10 +804,15 @@ export const createGateway = (options = {}) => {
|
|
|
479
804
|
if (!('sessionId' in event) || event.sessionId !== sessionId) {
|
|
480
805
|
return;
|
|
481
806
|
}
|
|
482
|
-
if (streamingActive
|
|
807
|
+
if (streamingActive
|
|
808
|
+
&& pendingTools === 0
|
|
809
|
+
&& openToolCalls.size === 0
|
|
810
|
+
&& (approvalWaits.get(sessionId) ?? 0) === 0
|
|
811
|
+
&& !turnSettling) {
|
|
483
812
|
armTimer();
|
|
484
813
|
}
|
|
485
814
|
});
|
|
815
|
+
suspendWatchdog.set(sessionId, suspendTimer);
|
|
486
816
|
if (streamingActive) {
|
|
487
817
|
armTimer();
|
|
488
818
|
}
|
|
@@ -499,6 +829,7 @@ export const createGateway = (options = {}) => {
|
|
|
499
829
|
suspendTimer();
|
|
500
830
|
unsubscribeObserver();
|
|
501
831
|
unsubscribeDrain();
|
|
832
|
+
suspendWatchdog.delete(sessionId);
|
|
502
833
|
external?.removeEventListener('abort', onExternalAbort);
|
|
503
834
|
}
|
|
504
835
|
};
|
|
@@ -509,23 +840,16 @@ export const createGateway = (options = {}) => {
|
|
|
509
840
|
const sessionChains = new Map();
|
|
510
841
|
const inflight = new Set();
|
|
511
842
|
let stopping = false;
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
// configured default soul has changed since — only a brand-new session
|
|
523
|
-
// takes the current default. (Loaded here, before the watchdog, also
|
|
524
|
-
// because whether this turn streams depends on durable session state.)
|
|
525
|
-
const existing = await store.get(input.sessionId);
|
|
526
|
-
const agentId = input.agentId ?? existing?.agent.id ?? await defaultAgentId();
|
|
527
|
-
const source = await refreshAgent(agentId);
|
|
528
|
-
const agent = source.definition;
|
|
843
|
+
/**
|
|
844
|
+
* The runtime one agent would run on right now — config snapshot, soul
|
|
845
|
+
* pins, degradation and all.
|
|
846
|
+
*
|
|
847
|
+
* Extracted so recovery resolves it exactly as a dispatch does. A second
|
|
848
|
+
* copy of this chain is the difference between a recovered turn finishing
|
|
849
|
+
* on its agent's provider and finishing on the daemon's default, and it
|
|
850
|
+
* would drift the first time either side gained a rule.
|
|
851
|
+
*/
|
|
852
|
+
const runtimeForAgent = async (source) => {
|
|
529
853
|
// Config re-resolves per dispatch, so a changed model, credential, or
|
|
530
854
|
// soul pin applies without a restart. The pool dedupes runners by the
|
|
531
855
|
// resolved configuration. The gateway-wide selection and the daemon's
|
|
@@ -584,7 +908,26 @@ export const createGateway = (options = {}) => {
|
|
|
584
908
|
}
|
|
585
909
|
// The preset snapshot means resolution reads no config file — every
|
|
586
910
|
// failure from here is a credential or provider problem and surfaces.
|
|
587
|
-
|
|
911
|
+
return resolveRuntimeConfig(selection, resolveEnv);
|
|
912
|
+
};
|
|
913
|
+
const dispatchInternal = async (input) => {
|
|
914
|
+
// A dispatch whose signal fired while it queued behind another turn
|
|
915
|
+
// must not touch durable state: without this check, the runner would
|
|
916
|
+
// load the session, append the cancelled user message, and save it as
|
|
917
|
+
// failed — polluting future model history with input never processed.
|
|
918
|
+
if (input.signal?.aborted) {
|
|
919
|
+
throw new RunAbortedError();
|
|
920
|
+
}
|
|
921
|
+
// A session pins its agent: when the caller names none, an existing
|
|
922
|
+
// conversation keeps the agent it was created with even if the
|
|
923
|
+
// configured default soul has changed since — only a brand-new session
|
|
924
|
+
// takes the current default. (Loaded here, before the watchdog, also
|
|
925
|
+
// because whether this turn streams depends on durable session state.)
|
|
926
|
+
const existing = await store.get(input.sessionId);
|
|
927
|
+
const agentId = input.agentId ?? existing?.agent.id ?? await defaultAgentId();
|
|
928
|
+
const source = await refreshAgent(agentId);
|
|
929
|
+
const agent = source.definition;
|
|
930
|
+
const config = await runtimeForAgent(source);
|
|
588
931
|
const runner = runnerFor(config);
|
|
589
932
|
const metadata = {
|
|
590
933
|
...(config.provider === 'demo'
|
|
@@ -626,23 +969,193 @@ export const createGateway = (options = {}) => {
|
|
|
626
969
|
});
|
|
627
970
|
});
|
|
628
971
|
};
|
|
629
|
-
|
|
972
|
+
/**
|
|
973
|
+
* Finishes the turns that were parked on a human when this daemon last
|
|
974
|
+
* went down.
|
|
975
|
+
*
|
|
976
|
+
* Run after the channels are up, deliberately: recovery re-asks, and a
|
|
977
|
+
* request emitted into a gateway with no live adapter would be declined
|
|
978
|
+
* for want of anywhere to put it — turning "your approval survived the
|
|
979
|
+
* restart" into "your approval was denied by the restart".
|
|
980
|
+
*
|
|
981
|
+
* Each session is recovered independently and failures are per-session:
|
|
982
|
+
* one unrecoverable turn must not stop the daemon from serving, or from
|
|
983
|
+
* recovering the others.
|
|
984
|
+
*/
|
|
985
|
+
const recoverParkedTurns = async () => {
|
|
986
|
+
if (!store.listIdsByStatus) {
|
|
987
|
+
return;
|
|
988
|
+
}
|
|
989
|
+
const parked = await store.listIdsByStatus('pending_approval');
|
|
990
|
+
if (parked.length === 0) {
|
|
991
|
+
return;
|
|
992
|
+
}
|
|
993
|
+
log(`recovering ${parked.length} turn(s) parked on approval`);
|
|
994
|
+
// Started together, not one after another. Each recovery re-asks and
|
|
995
|
+
// then waits on a human, so awaiting them in sequence would mean the
|
|
996
|
+
// second parked turn is not even *announced* until the first is
|
|
997
|
+
// answered — fifteen minutes later by default, and never at all with a
|
|
998
|
+
// zero timeout. They are independent turns; only same-session work is
|
|
999
|
+
// ordered, and onSessionChain already guarantees that.
|
|
1000
|
+
await Promise.allSettled(parked.map((sessionId) => recoverOne(sessionId)));
|
|
1001
|
+
};
|
|
1002
|
+
const recoverOne = (sessionId) => onSessionChain(sessionId, async () => {
|
|
630
1003
|
if (stopping) {
|
|
631
|
-
|
|
1004
|
+
return;
|
|
1005
|
+
}
|
|
1006
|
+
try {
|
|
1007
|
+
const session = await store.get(sessionId);
|
|
1008
|
+
if (!session) {
|
|
1009
|
+
return;
|
|
1010
|
+
}
|
|
1011
|
+
const record = readPendingApproval(session);
|
|
1012
|
+
// A wait that outlived its window while the daemon was down is
|
|
1013
|
+
// honoured, not restarted: the request really did go unanswered for
|
|
1014
|
+
// the whole time it was configured to wait, and downtime is not a
|
|
1015
|
+
// reason to extend a security decision. Measured from when the turn
|
|
1016
|
+
// parked against this daemon's timeout — the transport's original
|
|
1017
|
+
// deadline was chosen after the checkpoint was written and is gone
|
|
1018
|
+
// with the process that chose it. Denying goes through the same
|
|
1019
|
+
// recovery path, so the queue behind it still drains.
|
|
1020
|
+
const parkedAt = record ? Date.parse(record.parkedAt) : Number.NaN;
|
|
1021
|
+
const expired = effectiveApprovalTimeoutMs > 0
|
|
1022
|
+
&& Number.isFinite(parkedAt)
|
|
1023
|
+
&& Date.now() - parkedAt >= effectiveApprovalTimeoutMs;
|
|
1024
|
+
if (expired) {
|
|
1025
|
+
log(`${sessionId}: the approval for ${record?.call.toolName} outlived its window while the daemon was down; denying it`);
|
|
1026
|
+
}
|
|
1027
|
+
// Resolved the way a dispatch for this agent would resolve it, so a
|
|
1028
|
+
// recovered turn finishes on the same provider, model, and
|
|
1029
|
+
// credentials it was parked on.
|
|
1030
|
+
const source = await refreshAgent(session.agent.id);
|
|
1031
|
+
// And on the CURRENT definition, saved before recovery reads the
|
|
1032
|
+
// session back. An allowlist is a permission boundary: a soul that
|
|
1033
|
+
// dropped a tool while the daemon was down must not have that tool
|
|
1034
|
+
// executed by the turn that outlived the change, and
|
|
1035
|
+
// `allowedToolsFor` reads the definition frozen into the session
|
|
1036
|
+
// ahead of the registry. Dispatch refreshes it the same way before
|
|
1037
|
+
// resuming.
|
|
1038
|
+
session.agent = source.definition;
|
|
1039
|
+
await store.save(session);
|
|
1040
|
+
const runner = runnerFor(await runtimeForAgent(source));
|
|
1041
|
+
await runner.recoverPendingApproval(sessionId, { denyPending: expired });
|
|
1042
|
+
}
|
|
1043
|
+
catch (error) {
|
|
1044
|
+
warn(`could not recover parked session ${sessionId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1045
|
+
}
|
|
1046
|
+
});
|
|
1047
|
+
/**
|
|
1048
|
+
* Turns the last process was still running when it stopped being able to
|
|
1049
|
+
* finish them.
|
|
1050
|
+
*
|
|
1051
|
+
* `pending_approval` is the one window a turn can be picked up from, and
|
|
1052
|
+
* it has its own sweep above. Everything else a turn does is saved as
|
|
1053
|
+
* `running`: the provider call, the executor, and — on a provider that
|
|
1054
|
+
* hosts its own loop — the approval wait too, since a hosted call is
|
|
1055
|
+
* deliberately not checkpointed (`executeHostedToolCall` passes
|
|
1056
|
+
* `recoverable: false`, because the SDK's inner loop is not something a
|
|
1057
|
+
* restart can rebuild). None of those can be resumed, and nothing else
|
|
1058
|
+
* sweeps them, so the record goes on claiming the turn is running for as
|
|
1059
|
+
* long as the session exists.
|
|
1060
|
+
*
|
|
1061
|
+
* Note what this is *not* for. A graceful stop denies everything parked
|
|
1062
|
+
* and drains the turns those denials release, so an operator restarting
|
|
1063
|
+
* the daemon normally has nothing here. These are the turns a crash, a
|
|
1064
|
+
* SIGKILL, or a lost machine left behind.
|
|
1065
|
+
*/
|
|
1066
|
+
const listAbandonedTurns = async () => {
|
|
1067
|
+
if (!store.listIdsByStatus) {
|
|
1068
|
+
return [];
|
|
1069
|
+
}
|
|
1070
|
+
return store.listIdsByStatus('running');
|
|
1071
|
+
};
|
|
1072
|
+
/**
|
|
1073
|
+
* Marks each of them failed, with a reason that says what happened.
|
|
1074
|
+
*
|
|
1075
|
+
* Read BEFORE the channels start and applied after, which is the only
|
|
1076
|
+
* ordering that is both honest and safe. Reading first means the list is
|
|
1077
|
+
* exactly what the last process left, with no chance of a turn this
|
|
1078
|
+
* process started being in it. Applying after means the failure is
|
|
1079
|
+
* emitted into a gateway whose surfaces are listening, rather than into
|
|
1080
|
+
* one where nothing is.
|
|
1081
|
+
*
|
|
1082
|
+
* The gap between the two is real, though: an inbound message for one of
|
|
1083
|
+
* these sessions can arrive as soon as the channels are up. What makes
|
|
1084
|
+
* that safe is single-flight — every writer of a session goes through
|
|
1085
|
+
* `onSessionChain`, so this runs either wholly before that turn (it
|
|
1086
|
+
* fails, and `resume()` then clears `lastError` and starts fresh) or
|
|
1087
|
+
* wholly after it (the status has moved off `running`, and the re-read
|
|
1088
|
+
* below leaves it alone). There is no interleaving in which this
|
|
1089
|
+
* overwrites a live turn.
|
|
1090
|
+
*
|
|
1091
|
+
* What that invariant does NOT cover is a second daemon on the same
|
|
1092
|
+
* database, whose turns are running with no chain of ours in front of
|
|
1093
|
+
* them. Two processes sharing a session store is already unsupported —
|
|
1094
|
+
* the parked sweep above would re-ask that daemon's approvals for the
|
|
1095
|
+
* same reason — and no check here would make it supported.
|
|
1096
|
+
*/
|
|
1097
|
+
const failAbandonedTurns = async (abandoned) => {
|
|
1098
|
+
if (abandoned.length === 0) {
|
|
1099
|
+
return;
|
|
1100
|
+
}
|
|
1101
|
+
log(`failing ${abandoned.length} turn(s) left running by the last stratusd`);
|
|
1102
|
+
await Promise.allSettled(abandoned.map((id) => failAbandonedTurn(id)));
|
|
1103
|
+
};
|
|
1104
|
+
const failAbandonedTurn = (id) => onSessionChain(id, async () => {
|
|
1105
|
+
if (stopping) {
|
|
1106
|
+
return;
|
|
632
1107
|
}
|
|
633
|
-
|
|
634
|
-
|
|
1108
|
+
try {
|
|
1109
|
+
const session = await store.get(id);
|
|
1110
|
+
// Re-read on the chain, because the snapshot is from before the
|
|
1111
|
+
// channels came up: a message that arrived since has already taken
|
|
1112
|
+
// this session somewhere else, and that turn owns it now.
|
|
1113
|
+
if (!session || session.status !== 'running') {
|
|
1114
|
+
return;
|
|
1115
|
+
}
|
|
1116
|
+
session.status = 'failed';
|
|
1117
|
+
session.lastError = ABANDONED_TURN_ERROR;
|
|
1118
|
+
await store.save(session);
|
|
1119
|
+
await bus.emit({ type: 'session.updated', sessionId: id, status: 'failed' });
|
|
1120
|
+
await bus.emit({ type: 'session.failed', sessionId: id, error: ABANDONED_TURN_ERROR });
|
|
1121
|
+
}
|
|
1122
|
+
catch (error) {
|
|
1123
|
+
warn(`could not fail abandoned turn ${id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1124
|
+
}
|
|
1125
|
+
});
|
|
1126
|
+
/**
|
|
1127
|
+
* Runs `work` as the session's next turn: queued behind whatever that
|
|
1128
|
+
* session is already doing, and tracked so shutdown drains it.
|
|
1129
|
+
*
|
|
1130
|
+
* Recovery goes through here as well as dispatch, and must. Channels are
|
|
1131
|
+
* live before a sweep finishes, so an inbound message for a parked
|
|
1132
|
+
* session could otherwise `resume()` alongside `recoverPendingApproval()`
|
|
1133
|
+
* — one closing the parked call as interrupted while the other re-enters
|
|
1134
|
+
* it, both saving divergent copies of the same transcript, and the tool
|
|
1135
|
+
* possibly running twice. Single-flight per session is the invariant that
|
|
1136
|
+
* stops that, and it only holds if everything that writes a session goes
|
|
1137
|
+
* through it.
|
|
1138
|
+
*/
|
|
1139
|
+
const onSessionChain = (sessionId, work) => {
|
|
1140
|
+
const previous = sessionChains.get(sessionId) ?? Promise.resolve();
|
|
1141
|
+
const turn = previous.then(work, work);
|
|
635
1142
|
const settled = turn.catch(() => { });
|
|
636
|
-
sessionChains.set(
|
|
1143
|
+
sessionChains.set(sessionId, settled);
|
|
637
1144
|
inflight.add(settled);
|
|
638
1145
|
void settled.finally(() => {
|
|
639
1146
|
inflight.delete(settled);
|
|
640
|
-
if (sessionChains.get(
|
|
641
|
-
sessionChains.delete(
|
|
1147
|
+
if (sessionChains.get(sessionId) === settled) {
|
|
1148
|
+
sessionChains.delete(sessionId);
|
|
642
1149
|
}
|
|
643
1150
|
});
|
|
644
1151
|
return turn;
|
|
645
1152
|
};
|
|
1153
|
+
const dispatch = async (input) => {
|
|
1154
|
+
if (stopping) {
|
|
1155
|
+
throw new Error('The gateway is stopping and no longer accepts new work.');
|
|
1156
|
+
}
|
|
1157
|
+
return onSessionChain(input.sessionId, () => dispatchInternal(input));
|
|
1158
|
+
};
|
|
646
1159
|
const startedChannels = [];
|
|
647
1160
|
const gateway = {
|
|
648
1161
|
bus,
|
|
@@ -652,6 +1165,11 @@ export const createGateway = (options = {}) => {
|
|
|
652
1165
|
await loadRoster();
|
|
653
1166
|
const named = registry.list().map((agent) => agent.name).join(', ');
|
|
654
1167
|
log(`stratusd ready — ${registry.list().length} agent(s): ${named}`);
|
|
1168
|
+
// Read before anything can dispatch, so this is exactly what the
|
|
1169
|
+
// last process left running and not a turn of ours caught in flight.
|
|
1170
|
+
// Applied further down, once there is somewhere for the failure to
|
|
1171
|
+
// be heard.
|
|
1172
|
+
const abandoned = await listAbandonedTurns();
|
|
655
1173
|
// Channels come up after the roster so their first inbound message
|
|
656
1174
|
// already has agents to dispatch to. One failing adapter must not
|
|
657
1175
|
// keep the rest (or the gateway) down.
|
|
@@ -673,6 +1191,19 @@ export const createGateway = (options = {}) => {
|
|
|
673
1191
|
}
|
|
674
1192
|
}
|
|
675
1193
|
}
|
|
1194
|
+
// Last, and not awaited: recovery re-asks, so it needs the channels
|
|
1195
|
+
// above already listening — but a turn parked behind a slow approver
|
|
1196
|
+
// must not hold up start(), or a daemon with one outstanding request
|
|
1197
|
+
// would refuse to finish booting.
|
|
1198
|
+
void recoverParkedTurns().catch((error) => {
|
|
1199
|
+
warn(`recovering parked turns failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1200
|
+
});
|
|
1201
|
+
// Not awaited for the same reason, and independent of the sweep
|
|
1202
|
+
// above: these sessions are not parked, so the two never touch the
|
|
1203
|
+
// same one.
|
|
1204
|
+
void failAbandonedTurns(abandoned).catch((error) => {
|
|
1205
|
+
warn(`failing abandoned turns failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1206
|
+
});
|
|
676
1207
|
},
|
|
677
1208
|
// Drain: channels stop taking messages, in-flight turns finish, new
|
|
678
1209
|
// dispatches are refused, then the database closes. SIGTERM handling
|
|
@@ -682,6 +1213,17 @@ export const createGateway = (options = {}) => {
|
|
|
682
1213
|
// in-flight turns (already-started dispatches keep running), drain,
|
|
683
1214
|
// and close.
|
|
684
1215
|
stopping = true;
|
|
1216
|
+
// Deny what is parked before draining, or the drain waits out every
|
|
1217
|
+
// outstanding approval timeout — a shutdown would hang for as long as
|
|
1218
|
+
// the longest request had left. Each denial is a real decision the
|
|
1219
|
+
// turn continues from, so the drain below still finishes those turns.
|
|
1220
|
+
for (const settle of [...pendingApprovals.values()]) {
|
|
1221
|
+
settle('deny', 'cancelled');
|
|
1222
|
+
}
|
|
1223
|
+
// Let those resolutions reach their subscribers before the channels
|
|
1224
|
+
// go down, or a channel that renders approvals never learns to
|
|
1225
|
+
// retract the buttons it is still showing.
|
|
1226
|
+
await Promise.allSettled([...approvalEmissions]);
|
|
685
1227
|
await Promise.allSettled(startedChannels.map((adapter) => adapter.stop()));
|
|
686
1228
|
startedChannels.length = 0;
|
|
687
1229
|
await Promise.allSettled([...inflight]);
|
|
@@ -692,6 +1234,7 @@ export const createGateway = (options = {}) => {
|
|
|
692
1234
|
agents() {
|
|
693
1235
|
return registry.list();
|
|
694
1236
|
},
|
|
1237
|
+
resolveApproval,
|
|
695
1238
|
};
|
|
696
1239
|
return gateway;
|
|
697
1240
|
};
|