@stratusagent/gateway 0.12.0 → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/index.d.ts +42 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +322 -45
- package/dist/index.js.map +1 -1
- package/dist/sessions.js +1 -1
- package/package.json +7 -7
package/dist/index.js
CHANGED
|
@@ -9,7 +9,7 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
|
|
|
9
9
|
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
|
10
10
|
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
|
11
11
|
import path from 'node:path';
|
|
12
|
-
import { filePathsOf, isUnaddressedTurn, workItemState, lastResponseIsAnswer, MAX_IDEMPOTENCY_KEY_LENGTH, recordTurnFailure, completeAnsweredTurn, abortErrorFor, AgentRegistry, AgentRunner, ChannelRegistry, ContributionRegistry, createRoutedMemoryStore, EventBus, RunAbortedError, SkillRegistry, ToolRegistry, createSkillReadTool, missingSkillRequirements, describeToolAllowlistFinding, unmatchedToolAllowlist, CONTEXT_FLOOR_METADATA_KEY, PENDING_APPROVAL_METADATA_KEY, SESSION_TAINTED_BY_METADATA_KEY, SESSION_TRUST_METADATA_KEY, latestTurnReply, readPendingApproval, conversationContextFrom, CONVERSATION_METADATA_KEY, STRATUS_SKILL_ID, matchesToolAllowlist, } from '@stratusagent/core';
|
|
12
|
+
import { filePathsOf, isUnaddressedTurn, carriesKey, leastTrusted, senderTrustOf, SENDER_TRUST_METADATA_KEY, workItemState, lastResponseIsAnswer, MAX_IDEMPOTENCY_KEY_LENGTH, recordTurnFailure, completeAnsweredTurn, abortErrorFor, AgentRegistry, AgentRunner, ChannelRegistry, ContributionRegistry, createRoutedMemoryStore, EventBus, RunAbortedError, SkillRegistry, ToolRegistry, createSkillReadTool, missingSkillRequirements, describeToolAllowlistFinding, unmatchedToolAllowlist, CONTEXT_FLOOR_METADATA_KEY, PENDING_APPROVAL_METADATA_KEY, SESSION_TAINTED_BY_METADATA_KEY, SESSION_TRUST_METADATA_KEY, latestTurnReply, readPendingApproval, conversationContextFrom, CONVERSATION_METADATA_KEY, STRATUS_SKILL_ID, imagesWithinReplayBudget, matchesToolAllowlist, } from '@stratusagent/core';
|
|
13
13
|
import { CREDENTIAL_REQUEST_TOOL_NAME, createCredentialRequestTool, createDelegateTool, createForgetTool, createPinTool, createMessageSendTool, createMessageReadTool, createRecallTool, createRememberTool, createScheduleTools, DELEGATED_BY_METADATA_KEY, DELEGATION_DEPTH_METADATA_KEY, delegatingSessionIdOf, isDelegatedSession, outstandingDelegationFor, ROOT_SESSION_ID_METADATA_KEY, SCHEDULE_ID_METADATA_KEY, SCHEDULED_TURN_METADATA_KEY, withoutDelegation, loadStratusSkill, STRATUS_SKILL_PATH, } from '@stratusagent/agents';
|
|
14
14
|
import { createSchedulerRuntime, isScheduleSessionId, SqliteScheduleStore, SCHEDULE_SESSION_ID_PREFIX, } from "./schedules.js";
|
|
15
15
|
import { ShardedSessionStore } from "./sessions.js";
|
|
@@ -44,12 +44,23 @@ const MAX_TIMER_DELAY_MS = 2_147_483_647;
|
|
|
44
44
|
export { applySoulPins } from '@stratusagent/state';
|
|
45
45
|
/** What a turn cut short by an announced restart records as its failure. */
|
|
46
46
|
export const RESTARTING_TURN_ERROR = 'Run aborted: stratusd is restarting';
|
|
47
|
+
/** The error a turn still running when a stop's drain window closes is aborted with. */
|
|
48
|
+
export const STOPPING_TURN_ERROR = 'Run aborted: stratusd is stopping';
|
|
47
49
|
/**
|
|
48
50
|
* How long an announced restart lets in-flight turns finish before aborting
|
|
49
|
-
* them.
|
|
50
|
-
* operator can hit agree.
|
|
51
|
+
* them. The same as a stop's window (`DEFAULT_STOP_DRAIN_TIMEOUT_MS`), so the
|
|
52
|
+
* two windows an operator can hit agree.
|
|
51
53
|
*/
|
|
52
54
|
export const DEFAULT_RESTART_DRAIN_TIMEOUT_MS = 30_000;
|
|
55
|
+
/**
|
|
56
|
+
* How long a stop lets in-flight turns finish before aborting them. The
|
|
57
|
+
* daemon's documented drain ("up to 30 seconds"), and shorter than the
|
|
58
|
+
* service units' stop timeouts, so the abort happens in this process
|
|
59
|
+
* rather than as the manager's SIGKILL. That matters beyond the turn's
|
|
60
|
+
* record: a shell command runs in its own process group, which launchd's
|
|
61
|
+
* kill of the job does not reach, so only the turn's abort stops it.
|
|
62
|
+
*/
|
|
63
|
+
export const DEFAULT_STOP_DRAIN_TIMEOUT_MS = 30_000;
|
|
53
64
|
/** What a paused daemon says when nobody gave it a sentence of its own. */
|
|
54
65
|
export const DEFAULT_PAUSED_MESSAGE = "I'm paused right now and not taking new work. Please try again later.";
|
|
55
66
|
/** What a held gateway answers a dispatch with, before it is promoted. */
|
|
@@ -148,6 +159,38 @@ export class RestartUnsupportedError extends Error {
|
|
|
148
159
|
this.name = 'RestartUnsupportedError';
|
|
149
160
|
}
|
|
150
161
|
}
|
|
162
|
+
/**
|
|
163
|
+
* The most text one merged user message may hold (card 423). The runner's
|
|
164
|
+
* overflow recovery can drop earlier transcript but never part of the
|
|
165
|
+
* newest message, so a batch must not grow that message past what one
|
|
166
|
+
* message could be on its own: this is Slack's own message-length limit.
|
|
167
|
+
* A single message longer than this still runs, alone, as it would have
|
|
168
|
+
* without merging.
|
|
169
|
+
*/
|
|
170
|
+
const MERGED_TEXT_BUDGET_CHARS = 40_000;
|
|
171
|
+
/** Separator between merged messages; counted against the text budget. */
|
|
172
|
+
const MERGED_SEPARATOR = '\n\n';
|
|
173
|
+
/**
|
|
174
|
+
* Whether `images` all fit the replay budget together. A merged turn's
|
|
175
|
+
* images are one message's, and the runner empties whatever falls outside
|
|
176
|
+
* the budget before the provider sees it, so a batch that would overflow
|
|
177
|
+
* must not take the message that overflows it (card 423).
|
|
178
|
+
*/
|
|
179
|
+
const fitsReplayBudget = (images) => images.length === 0 || imagesWithinReplayBudget([{ id: '', createdAt: '', role: 'user', content: '', images: [...images] }]).size === images.length;
|
|
180
|
+
/** A signal that aborts once every one of `signals` has. */
|
|
181
|
+
const allAborted = (signals) => {
|
|
182
|
+
const controller = new AbortController();
|
|
183
|
+
const check = () => {
|
|
184
|
+
if (!controller.signal.aborted && signals.every((signal) => signal.aborted)) {
|
|
185
|
+
controller.abort(signals[signals.length - 1]?.reason);
|
|
186
|
+
}
|
|
187
|
+
};
|
|
188
|
+
for (const signal of signals) {
|
|
189
|
+
signal.addEventListener('abort', check, { once: true });
|
|
190
|
+
}
|
|
191
|
+
check();
|
|
192
|
+
return controller.signal;
|
|
193
|
+
};
|
|
151
194
|
/**
|
|
152
195
|
* Recorded on a turn the previous process was still running when it died.
|
|
153
196
|
*
|
|
@@ -1562,8 +1605,8 @@ export const createGateway = (options = {}) => {
|
|
|
1562
1605
|
// announced restart must be able to cut short at the window.
|
|
1563
1606
|
const idle = new AbortController();
|
|
1564
1607
|
turnControllers.add(idle);
|
|
1565
|
-
if (
|
|
1566
|
-
idle.abort(new RunAbortedError(
|
|
1608
|
+
if (abortingWith !== undefined) {
|
|
1609
|
+
idle.abort(new RunAbortedError(abortingWith));
|
|
1567
1610
|
}
|
|
1568
1611
|
try {
|
|
1569
1612
|
return await run(idle.signal);
|
|
@@ -1574,8 +1617,8 @@ export const createGateway = (options = {}) => {
|
|
|
1574
1617
|
}
|
|
1575
1618
|
const controller = new AbortController();
|
|
1576
1619
|
turnControllers.add(controller);
|
|
1577
|
-
if (
|
|
1578
|
-
controller.abort(new RunAbortedError(
|
|
1620
|
+
if (abortingWith !== undefined) {
|
|
1621
|
+
controller.abort(new RunAbortedError(abortingWith));
|
|
1579
1622
|
}
|
|
1580
1623
|
// The external reason travels: a caller that aborted with its own
|
|
1581
1624
|
// `RunAbortedError` gets it recorded on the session, like the watchdog's.
|
|
@@ -1773,14 +1816,15 @@ export const createGateway = (options = {}) => {
|
|
|
1773
1816
|
*/
|
|
1774
1817
|
const turnControllers = new Set();
|
|
1775
1818
|
/**
|
|
1776
|
-
* Set once a restart's window has closed and the turns in it
|
|
1777
|
-
* aborted
|
|
1778
|
-
* aborted turn on the same session,
|
|
1779
|
-
* is aborted the moment it gets its
|
|
1780
|
-
* rather than running unseen until the
|
|
1781
|
-
* it and leaves it recorded as nothing
|
|
1819
|
+
* Set once a restart's or a stop's window has closed and the turns in it
|
|
1820
|
+
* were aborted, to the reason they were aborted with. A turn that starts
|
|
1821
|
+
* after that — one queued behind the aborted turn on the same session,
|
|
1822
|
+
* accepted before the announcement — is aborted the moment it gets its
|
|
1823
|
+
* controller, with the same reason, rather than running unseen until the
|
|
1824
|
+
* shutdown closes the store under it and leaves it recorded as nothing
|
|
1825
|
+
* at all.
|
|
1782
1826
|
*/
|
|
1783
|
-
let
|
|
1827
|
+
let abortingWith;
|
|
1784
1828
|
/**
|
|
1785
1829
|
* Set once `start()` has finished. A restart before that would stop the
|
|
1786
1830
|
* channels started so far and close the stores while the rest of
|
|
@@ -1819,6 +1863,13 @@ export const createGateway = (options = {}) => {
|
|
|
1819
1863
|
// then the entry is a claim that may yet be refused. `addressed` is false
|
|
1820
1864
|
// for a turn nobody asked for, which `holdsMessage` does not count.
|
|
1821
1865
|
const liveWorkItems = new Map();
|
|
1866
|
+
/**
|
|
1867
|
+
* The merged turn still taking messages, per session (card 423): led by
|
|
1868
|
+
* the first opted-in dispatch that queued behind a running turn, closed
|
|
1869
|
+
* the moment that dispatch reaches the head of the chain. See
|
|
1870
|
+
* `DispatchInput.onAbsorbed`.
|
|
1871
|
+
*/
|
|
1872
|
+
const openBatches = new Map();
|
|
1822
1873
|
/** The start-up snapshot `recoverParkedTurns` judges orphans from, for a repeat that gets there first. */
|
|
1823
1874
|
let orphanedAtStart = new Set();
|
|
1824
1875
|
/**
|
|
@@ -1918,7 +1969,7 @@ export const createGateway = (options = {}) => {
|
|
|
1918
1969
|
? runsOnHarness(fallback)
|
|
1919
1970
|
: runsOnHarness(config) || (fallback !== undefined && runsOnHarness(fallback));
|
|
1920
1971
|
};
|
|
1921
|
-
const dispatchInternal = async (input) => {
|
|
1972
|
+
const dispatchInternal = async (input, absorbedKeys = []) => {
|
|
1922
1973
|
// Re-read on every turn, not once at start: a newer build that stamps
|
|
1923
1974
|
// the home while this daemon is serving has formats this build does not
|
|
1924
1975
|
// understand, and a daemon that kept writing into them would be the
|
|
@@ -2028,6 +2079,7 @@ export const createGateway = (options = {}) => {
|
|
|
2028
2079
|
...(input.images !== undefined ? { images: input.images } : {}),
|
|
2029
2080
|
...(input.addressed !== undefined ? { addressed: input.addressed } : {}),
|
|
2030
2081
|
...(input.idempotencyKey !== undefined ? { idempotencyKey: input.idempotencyKey } : {}),
|
|
2082
|
+
...(absorbedKeys.length > 0 ? { absorbedKeys: [...absorbedKeys] } : {}),
|
|
2031
2083
|
...(hostedLoop ? { hostedLoop } : {}),
|
|
2032
2084
|
...(input.metadata ? { metadata: input.metadata } : {}),
|
|
2033
2085
|
runtime: runtimeContextFor(source, config, switchedToFallback, input.metadata),
|
|
@@ -2041,6 +2093,7 @@ export const createGateway = (options = {}) => {
|
|
|
2041
2093
|
...(input.images !== undefined ? { images: input.images } : {}),
|
|
2042
2094
|
...(input.addressed !== undefined ? { addressed: input.addressed } : {}),
|
|
2043
2095
|
...(input.idempotencyKey !== undefined ? { idempotencyKey: input.idempotencyKey } : {}),
|
|
2096
|
+
...(absorbedKeys.length > 0 ? { absorbedKeys: [...absorbedKeys] } : {}),
|
|
2044
2097
|
...(hostedLoop ? { hostedLoop } : {}),
|
|
2045
2098
|
...(input.earlier !== undefined && input.earlier.length > 0 ? { earlier: input.earlier } : {}),
|
|
2046
2099
|
metadata,
|
|
@@ -2200,8 +2253,8 @@ export const createGateway = (options = {}) => {
|
|
|
2200
2253
|
// reports undrained for a turn it could have finished cleanly.
|
|
2201
2254
|
const controller = new AbortController();
|
|
2202
2255
|
turnControllers.add(controller);
|
|
2203
|
-
if (
|
|
2204
|
-
controller.abort(new RunAbortedError(
|
|
2256
|
+
if (abortingWith !== undefined) {
|
|
2257
|
+
controller.abort(new RunAbortedError(abortingWith));
|
|
2205
2258
|
}
|
|
2206
2259
|
try {
|
|
2207
2260
|
await runner.recoverPendingApproval(sessionId, {
|
|
@@ -2684,6 +2737,55 @@ export const createGateway = (options = {}) => {
|
|
|
2684
2737
|
};
|
|
2685
2738
|
return attach();
|
|
2686
2739
|
}
|
|
2740
|
+
// Merged turns (card 423): an opted-in message behind a running turn
|
|
2741
|
+
// joins the batch already waiting, if there is one for the same agent.
|
|
2742
|
+
const mergeable = input.onAbsorbed !== undefined && input.addressed !== false;
|
|
2743
|
+
const open = openBatches.get(input.sessionId);
|
|
2744
|
+
if (mergeable &&
|
|
2745
|
+
open !== undefined &&
|
|
2746
|
+
open.agentId === input.agentId &&
|
|
2747
|
+
fitsReplayBudget([...open.images, ...(input.images ?? [])]) &&
|
|
2748
|
+
open.chars + MERGED_SEPARATOR.length + input.userMessage.length <= MERGED_TEXT_BUDGET_CHARS) {
|
|
2749
|
+
open.images.push(...(input.images ?? []));
|
|
2750
|
+
open.chars += MERGED_SEPARATOR.length + input.userMessage.length;
|
|
2751
|
+
waitingDispatches.set(input.sessionId, (waitingDispatches.get(input.sessionId) ?? 0) + 1);
|
|
2752
|
+
const joined = new Promise((resolve, reject) => {
|
|
2753
|
+
open.absorbed.push({ input, resolve, reject });
|
|
2754
|
+
});
|
|
2755
|
+
if (idempotencyKey !== undefined) {
|
|
2756
|
+
// Provisional, like a queued dispatch's: a redelivery while it waits
|
|
2757
|
+
// attaches here; the merged turn confirms it once it owns the key.
|
|
2758
|
+
const items = liveWorkItems.get(input.sessionId) ?? new Map();
|
|
2759
|
+
if (!items.has(idempotencyKey)) {
|
|
2760
|
+
items.set(idempotencyKey, { turn: joined, agentId: input.agentId, confirmed: false, addressed: true });
|
|
2761
|
+
liveWorkItems.set(input.sessionId, items);
|
|
2762
|
+
}
|
|
2763
|
+
void joined.catch(() => { }).finally(() => {
|
|
2764
|
+
const current = liveWorkItems.get(input.sessionId);
|
|
2765
|
+
if (current?.get(idempotencyKey)?.turn === joined) {
|
|
2766
|
+
current.delete(idempotencyKey);
|
|
2767
|
+
if (current.size === 0) {
|
|
2768
|
+
liveWorkItems.delete(input.sessionId);
|
|
2769
|
+
}
|
|
2770
|
+
}
|
|
2771
|
+
});
|
|
2772
|
+
}
|
|
2773
|
+
return joined;
|
|
2774
|
+
}
|
|
2775
|
+
// It cannot join (or its images or text would overflow the batch's
|
|
2776
|
+
// budget), so the open batch takes no more: a message arriving
|
|
2777
|
+
// after this one must not be answered ahead of it.
|
|
2778
|
+
if (open !== undefined) {
|
|
2779
|
+
openBatches.delete(input.sessionId);
|
|
2780
|
+
}
|
|
2781
|
+
// Otherwise it leads a new batch, if it is waiting behind a turn at all:
|
|
2782
|
+
// a message to an idle session runs at once, alone.
|
|
2783
|
+
const batch = mergeable && sessionChains.has(input.sessionId)
|
|
2784
|
+
? { agentId: input.agentId, absorbed: [], images: [...(input.images ?? [])], chars: input.userMessage.length }
|
|
2785
|
+
: undefined;
|
|
2786
|
+
if (batch !== undefined) {
|
|
2787
|
+
openBatches.set(input.sessionId, batch);
|
|
2788
|
+
}
|
|
2687
2789
|
// Reads `turn` only once it is assigned: the registration below runs
|
|
2688
2790
|
// after it, and the chain's work runs in a later tick (`.then`).
|
|
2689
2791
|
const claimLiveItem = (key, confirmed) => {
|
|
@@ -2691,16 +2793,149 @@ export const createGateway = (options = {}) => {
|
|
|
2691
2793
|
items.set(key, { turn, agentId: input.agentId, confirmed, addressed: input.addressed !== false });
|
|
2692
2794
|
liveWorkItems.set(input.sessionId, items);
|
|
2693
2795
|
};
|
|
2796
|
+
/**
|
|
2797
|
+
* This dispatch's turn with the messages it absorbed (card 423). Each
|
|
2798
|
+
* part is judged on its own first, as it would have been alone: one
|
|
2799
|
+
* cancelled while it waited is refused, one that repeats a turn the
|
|
2800
|
+
* session already holds is that turn. What is left runs as one turn.
|
|
2801
|
+
*/
|
|
2802
|
+
const runMerged = async (absorbed) => {
|
|
2803
|
+
const parts = [{ input }, ...absorbed.map((joiner) => ({ input: joiner.input, joiner }))];
|
|
2804
|
+
let leader;
|
|
2805
|
+
const settle = (part, outcome) => {
|
|
2806
|
+
if (part.joiner === undefined) {
|
|
2807
|
+
// Settled the moment it is known, not when the chain moves on: a
|
|
2808
|
+
// leader cancelled or repeated while it waited must leave its
|
|
2809
|
+
// caller's queue before the turn it handed over starts rendering.
|
|
2810
|
+
leader = outcome;
|
|
2811
|
+
settleLeader(outcome);
|
|
2812
|
+
}
|
|
2813
|
+
else if ('session' in outcome) {
|
|
2814
|
+
part.joiner.resolve(outcome.session);
|
|
2815
|
+
}
|
|
2816
|
+
else {
|
|
2817
|
+
part.joiner.reject(outcome.error);
|
|
2818
|
+
}
|
|
2819
|
+
};
|
|
2820
|
+
const runnable = [];
|
|
2821
|
+
for (const part of parts) {
|
|
2822
|
+
const own = part.input;
|
|
2823
|
+
if (own.signal?.aborted) {
|
|
2824
|
+
settle(part, { error: abortErrorFor(own.signal) });
|
|
2825
|
+
continue;
|
|
2826
|
+
}
|
|
2827
|
+
if (own.idempotencyKey !== undefined) {
|
|
2828
|
+
try {
|
|
2829
|
+
const repeated = await settleRepeat(input.sessionId, own.idempotencyKey, own.agentId, own.turnId, own.onRepeat);
|
|
2830
|
+
if (repeated !== undefined) {
|
|
2831
|
+
settle(part, { session: repeated });
|
|
2832
|
+
continue;
|
|
2833
|
+
}
|
|
2834
|
+
}
|
|
2835
|
+
catch (error) {
|
|
2836
|
+
settle(part, { error });
|
|
2837
|
+
continue;
|
|
2838
|
+
}
|
|
2839
|
+
}
|
|
2840
|
+
runnable.push(part);
|
|
2841
|
+
}
|
|
2842
|
+
const head = runnable[0];
|
|
2843
|
+
if (head === undefined) {
|
|
2844
|
+
return finish(leader);
|
|
2845
|
+
}
|
|
2846
|
+
// Every runnable key now belongs to this turn.
|
|
2847
|
+
const keys = runnable.flatMap((part) => (part.input.idempotencyKey !== undefined ? [part.input.idempotencyKey] : []));
|
|
2848
|
+
const items = liveWorkItems.get(input.sessionId);
|
|
2849
|
+
for (const key of keys) {
|
|
2850
|
+
const item = items?.get(key);
|
|
2851
|
+
if (item !== undefined) {
|
|
2852
|
+
item.confirmed = true;
|
|
2853
|
+
}
|
|
2854
|
+
}
|
|
2855
|
+
// The turn runs at the least trusted sender in it: everyone's words
|
|
2856
|
+
// are in its one message.
|
|
2857
|
+
const labelled = runnable.some((part) => part.input.metadata !== undefined && SENDER_TRUST_METADATA_KEY in part.input.metadata);
|
|
2858
|
+
const lowest = leastTrusted(...runnable.map((part) => senderTrustOf(part.input.metadata)));
|
|
2859
|
+
const newest = runnable[runnable.length - 1].input;
|
|
2860
|
+
const metadata = labelled ? { ...newest.metadata, [SENDER_TRUST_METADATA_KEY]: lowest } : newest.metadata;
|
|
2861
|
+
const images = runnable.flatMap((part) => part.input.images ?? []);
|
|
2862
|
+
const signals = runnable.map((part) => part.input.signal);
|
|
2863
|
+
// Cancelled only when every caller in it is: one caller giving up
|
|
2864
|
+
// must not take the others' answer with it.
|
|
2865
|
+
const signal = signals.every((candidate) => candidate !== undefined)
|
|
2866
|
+
? allAborted(signals)
|
|
2867
|
+
: undefined;
|
|
2868
|
+
const [primaryKey, ...absorbedKeys] = keys;
|
|
2869
|
+
const merged = {
|
|
2870
|
+
sessionId: input.sessionId,
|
|
2871
|
+
...(head.input.agentId !== undefined ? { agentId: head.input.agentId } : {}),
|
|
2872
|
+
userMessage: runnable.map((part) => part.input.userMessage).join(MERGED_SEPARATOR),
|
|
2873
|
+
...(images.length > 0 ? { images } : {}),
|
|
2874
|
+
...(head.input.earlier !== undefined ? { earlier: head.input.earlier } : {}),
|
|
2875
|
+
...(metadata !== undefined ? { metadata } : {}),
|
|
2876
|
+
...(signal !== undefined ? { signal } : {}),
|
|
2877
|
+
...(head.input.turnId !== undefined ? { turnId: head.input.turnId } : {}),
|
|
2878
|
+
...(primaryKey !== undefined ? { idempotencyKey: primaryKey } : {}),
|
|
2879
|
+
};
|
|
2880
|
+
await settleOrphanedItem(input.sessionId);
|
|
2881
|
+
let outcome;
|
|
2882
|
+
if (merged.turnId !== undefined) {
|
|
2883
|
+
activeTurns.set(input.sessionId, merged.turnId);
|
|
2884
|
+
}
|
|
2885
|
+
try {
|
|
2886
|
+
outcome = { session: await dispatchInternal(merged, absorbedKeys) };
|
|
2887
|
+
}
|
|
2888
|
+
catch (error) {
|
|
2889
|
+
outcome = { error };
|
|
2890
|
+
}
|
|
2891
|
+
finally {
|
|
2892
|
+
if (merged.turnId !== undefined) {
|
|
2893
|
+
activeTurns.delete(input.sessionId);
|
|
2894
|
+
}
|
|
2895
|
+
}
|
|
2896
|
+
for (const part of runnable) {
|
|
2897
|
+
if (part !== head) {
|
|
2898
|
+
part.input.onAbsorbed?.();
|
|
2899
|
+
}
|
|
2900
|
+
settle(part, outcome);
|
|
2901
|
+
}
|
|
2902
|
+
return finish(leader);
|
|
2903
|
+
};
|
|
2904
|
+
const finish = (outcome) => {
|
|
2905
|
+
if (outcome === undefined) {
|
|
2906
|
+
throw new Error('A merged turn settled without an outcome for its leader.');
|
|
2907
|
+
}
|
|
2908
|
+
if ('error' in outcome) {
|
|
2909
|
+
throw outcome.error;
|
|
2910
|
+
}
|
|
2911
|
+
return outcome.session;
|
|
2912
|
+
};
|
|
2913
|
+
// What this caller is handed. It follows the chain's work, except that a
|
|
2914
|
+
// batch leader settles it early (`settle` in `runMerged`).
|
|
2915
|
+
let settleLeader = () => { };
|
|
2916
|
+
const turn = new Promise((resolve, reject) => {
|
|
2917
|
+
settleLeader = (outcome) => ('error' in outcome ? reject(outcome.error) : resolve(outcome.session));
|
|
2918
|
+
});
|
|
2694
2919
|
waitingDispatches.set(input.sessionId, (waitingDispatches.get(input.sessionId) ?? 0) + 1);
|
|
2695
|
-
const
|
|
2696
|
-
//
|
|
2697
|
-
|
|
2920
|
+
const chained = onSessionChain(input.sessionId, async () => {
|
|
2921
|
+
// The batch closes here: anything arriving from now on waits behind
|
|
2922
|
+
// this turn and leads the next one.
|
|
2923
|
+
if (batch !== undefined && openBatches.get(input.sessionId) === batch) {
|
|
2924
|
+
openBatches.delete(input.sessionId);
|
|
2925
|
+
}
|
|
2926
|
+
const absorbed = batch?.absorbed ?? [];
|
|
2927
|
+
// No longer waiting once it reaches the head of the chain — nor is
|
|
2928
|
+
// anything it absorbed.
|
|
2929
|
+
const left = (waitingDispatches.get(input.sessionId) ?? 1) - 1 - absorbed.length;
|
|
2698
2930
|
if (left > 0) {
|
|
2699
2931
|
waitingDispatches.set(input.sessionId, left);
|
|
2700
2932
|
}
|
|
2701
2933
|
else {
|
|
2702
2934
|
waitingDispatches.delete(input.sessionId);
|
|
2703
2935
|
}
|
|
2936
|
+
if (absorbed.length > 0) {
|
|
2937
|
+
return runMerged(absorbed);
|
|
2938
|
+
}
|
|
2704
2939
|
// Before `activeTurns` is touched: a repeat that runs nothing must
|
|
2705
2940
|
// not claim the session's turn id, even for the instant it takes.
|
|
2706
2941
|
const repeated = idempotencyKey !== undefined ? await settleRepeat(input.sessionId, idempotencyKey, input.agentId, turnId, input.onRepeat) : undefined;
|
|
@@ -2730,6 +2965,8 @@ export const createGateway = (options = {}) => {
|
|
|
2730
2965
|
activeTurns.delete(input.sessionId);
|
|
2731
2966
|
}
|
|
2732
2967
|
});
|
|
2968
|
+
// A second settle (a leader already settled early) is a no-op.
|
|
2969
|
+
chained.then((session) => settleLeader({ session }), (error) => settleLeader({ error }));
|
|
2733
2970
|
if (idempotencyKey !== undefined) {
|
|
2734
2971
|
// A repeat that queued instead of attaching — it named another agent,
|
|
2735
2972
|
// and is about to be refused — must not displace the turn it repeats.
|
|
@@ -2765,6 +3002,10 @@ export const createGateway = (options = {}) => {
|
|
|
2765
3002
|
if (reserved !== undefined) {
|
|
2766
3003
|
throw new Error(`Session metadata key "${reserved}" is reserved for the daemon's own records and cannot be supplied by an observe. Reserved keys: ${RESERVED_SESSION_METADATA_KEYS.join(', ')}.`);
|
|
2767
3004
|
}
|
|
3005
|
+
// Heard after the batch waiting on this session opened, so anything
|
|
3006
|
+
// said after it must not be answered ahead of it: the batch takes no
|
|
3007
|
+
// more, and a later message leads the next one (card 423).
|
|
3008
|
+
openBatches.delete(input.sessionId);
|
|
2768
3009
|
return onSessionChain(input.sessionId, async () => {
|
|
2769
3010
|
assertStateCompatible(env);
|
|
2770
3011
|
const existing = await store.get(input.sessionId);
|
|
@@ -3108,6 +3349,15 @@ export const createGateway = (options = {}) => {
|
|
|
3108
3349
|
deferredStart = undefined;
|
|
3109
3350
|
heldState = false;
|
|
3110
3351
|
log('stratusd promoted: channels connected, schedules running');
|
|
3352
|
+
if (options.onPromoted) {
|
|
3353
|
+
// Synchronous throws too: the promotion has happened either way.
|
|
3354
|
+
try {
|
|
3355
|
+
await options.onPromoted();
|
|
3356
|
+
}
|
|
3357
|
+
catch (error) {
|
|
3358
|
+
warn(`recording the promotion failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
3359
|
+
}
|
|
3360
|
+
}
|
|
3111
3361
|
})().finally(() => {
|
|
3112
3362
|
promoting = undefined;
|
|
3113
3363
|
});
|
|
@@ -3158,11 +3408,14 @@ export const createGateway = (options = {}) => {
|
|
|
3158
3408
|
* The one shutdown sequence — for a stop, for a restart, and for a
|
|
3159
3409
|
* start() that failed, whose caller never reaches stop().
|
|
3160
3410
|
*
|
|
3161
|
-
* `abortAfterMs` is
|
|
3162
|
-
*
|
|
3163
|
-
*
|
|
3164
|
-
*
|
|
3165
|
-
*
|
|
3411
|
+
* `abortAfterMs` is how long in-flight turns get before what is still
|
|
3412
|
+
* running is aborted, with `kind` as the reason: a restart so the daemon
|
|
3413
|
+
* can come back instead of waiting on one stuck turn, a stop so the
|
|
3414
|
+
* abort happens here and not as the service manager's SIGKILL, which
|
|
3415
|
+
* would leave the turn unrecorded and its shell commands' process groups
|
|
3416
|
+
* running after the daemon. Undefined waits however long it takes: only
|
|
3417
|
+
* a start() that failed, whose turns are the few a channel accepted in
|
|
3418
|
+
* the moment before. Resolves with whether everything finished — after an abort,
|
|
3166
3419
|
* a turn that still ignores its signal for a second window is shut down
|
|
3167
3420
|
* around, and the host is told so it can exit rather than start a new
|
|
3168
3421
|
* daemon beside it. Every wait after the window opens is bounded by it,
|
|
@@ -3170,7 +3423,7 @@ export const createGateway = (options = {}) => {
|
|
|
3170
3423
|
* the turn drain gave up on, and an unbounded wait there would hold the
|
|
3171
3424
|
* restart forever on exactly the turn the window exists for.
|
|
3172
3425
|
*/
|
|
3173
|
-
const shutdown = async (abortAfterMs) => {
|
|
3426
|
+
const shutdown = async (abortAfterMs, kind = 'restart') => {
|
|
3174
3427
|
// Refuse new work first; then let channels finish rendering their
|
|
3175
3428
|
// in-flight turns (already-started dispatches keep running), drain,
|
|
3176
3429
|
// and close.
|
|
@@ -3216,6 +3469,14 @@ export const createGateway = (options = {}) => {
|
|
|
3216
3469
|
startedChannels.length = 0;
|
|
3217
3470
|
await Promise.allSettled([...inflight]);
|
|
3218
3471
|
})();
|
|
3472
|
+
// One deadline for the whole bounded shutdown: the drain window, then
|
|
3473
|
+
// one more window that everything after it shares (the aborted turns'
|
|
3474
|
+
// record, the scheduler's drain, the plugins' dispose). A service
|
|
3475
|
+
// manager's stop timeout is sized for exactly that (see the CLI's
|
|
3476
|
+
// SERVICE_STOP_TIMEOUT_SECONDS); a window per phase would add up past
|
|
3477
|
+
// it and land its SIGKILL mid-cleanup.
|
|
3478
|
+
const deadline = abortAfterMs === undefined ? undefined : Date.now() + 2 * abortAfterMs;
|
|
3479
|
+
const remaining = () => deadline === undefined || abortAfterMs === undefined ? 0 : Math.min(abortAfterMs, Math.max(0, deadline - Date.now()));
|
|
3219
3480
|
let drained = true;
|
|
3220
3481
|
if (abortAfterMs === undefined) {
|
|
3221
3482
|
await drain;
|
|
@@ -3224,19 +3485,19 @@ export const createGateway = (options = {}) => {
|
|
|
3224
3485
|
drained = channelsReleased;
|
|
3225
3486
|
}
|
|
3226
3487
|
else {
|
|
3227
|
-
warn(
|
|
3488
|
+
warn(`${kind}: ${inflight.size} turn(s) still running after ${abortAfterMs}ms; aborting them`);
|
|
3228
3489
|
// The watchdog's own mechanism: the runner fails each session with
|
|
3229
3490
|
// the reason on the signal, saves it, and emits session.failed —
|
|
3230
3491
|
// so the next daemon finds a finished turn, not an abandoned one.
|
|
3231
3492
|
// Before the loop, so a turn queued behind one aborted here, which
|
|
3232
3493
|
// starts as soon as that one settles, is aborted the same way.
|
|
3233
|
-
|
|
3494
|
+
abortingWith = kind === 'stop' ? STOPPING_TURN_ERROR : RESTARTING_TURN_ERROR;
|
|
3234
3495
|
for (const controller of turnControllers) {
|
|
3235
|
-
controller.abort(new RunAbortedError(
|
|
3496
|
+
controller.abort(new RunAbortedError(abortingWith));
|
|
3236
3497
|
}
|
|
3237
|
-
drained = await settlesWithin(drain,
|
|
3498
|
+
drained = await settlesWithin(drain, remaining());
|
|
3238
3499
|
if (!drained) {
|
|
3239
|
-
warn(
|
|
3500
|
+
warn(`${kind}: ${inflight.size} turn(s) did not stop after being aborted; shutting down around them`);
|
|
3240
3501
|
}
|
|
3241
3502
|
drained = drained && channelsReleased;
|
|
3242
3503
|
}
|
|
@@ -3249,8 +3510,8 @@ export const createGateway = (options = {}) => {
|
|
|
3249
3510
|
if (drained || abortAfterMs === undefined) {
|
|
3250
3511
|
await schedulerDrain;
|
|
3251
3512
|
}
|
|
3252
|
-
else if (!(await settlesWithin(schedulerDrain,
|
|
3253
|
-
warn(
|
|
3513
|
+
else if (!(await settlesWithin(schedulerDrain, remaining()))) {
|
|
3514
|
+
warn(`${kind}: a scheduled firing did not stop after being aborted; shutting down around it`);
|
|
3254
3515
|
}
|
|
3255
3516
|
// After the turns that might still be using them. A plugin holding a
|
|
3256
3517
|
// browser is the reason this exists, and closing it out from under a
|
|
@@ -3260,16 +3521,17 @@ export const createGateway = (options = {}) => {
|
|
|
3260
3521
|
// coming back, and a plugin still holding on is one more thing the
|
|
3261
3522
|
// host should not start a new daemon beside.
|
|
3262
3523
|
const disposal = disposePlugins();
|
|
3524
|
+
const disposalWindow = remaining();
|
|
3263
3525
|
if (abortAfterMs === undefined) {
|
|
3264
3526
|
await disposal;
|
|
3265
3527
|
}
|
|
3266
|
-
else if (await settlesWithin(disposal,
|
|
3528
|
+
else if (await settlesWithin(disposal, disposalWindow)) {
|
|
3267
3529
|
// A dispose that rejected has already been named; what it may still
|
|
3268
3530
|
// hold is the same problem as one that never answered.
|
|
3269
3531
|
drained = drained && await disposal;
|
|
3270
3532
|
}
|
|
3271
3533
|
else {
|
|
3272
|
-
warn(
|
|
3534
|
+
warn(`${kind}: a plugin did not release what it holds within ${disposalWindow}ms; shutting down around it`);
|
|
3273
3535
|
drained = false;
|
|
3274
3536
|
}
|
|
3275
3537
|
closeStores();
|
|
@@ -3316,14 +3578,25 @@ export const createGateway = (options = {}) => {
|
|
|
3316
3578
|
}
|
|
3317
3579
|
serving = true;
|
|
3318
3580
|
},
|
|
3319
|
-
// Drain: channels stop taking messages, in-flight turns
|
|
3320
|
-
//
|
|
3321
|
-
//
|
|
3322
|
-
//
|
|
3323
|
-
//
|
|
3324
|
-
// and
|
|
3581
|
+
// Drain: channels stop taking messages, in-flight turns get the stop
|
|
3582
|
+
// window to finish and are aborted after it, new dispatches are
|
|
3583
|
+
// refused, then the database closes. SIGTERM handling in `stratus
|
|
3584
|
+
// serve` calls this. Idempotent: a restart stops the gateway itself
|
|
3585
|
+
// before handing the process to the host, whose own shutdown path then
|
|
3586
|
+
// calls this again and must find nothing to do — and a start() that
|
|
3587
|
+
// failed has already run the same sequence.
|
|
3325
3588
|
stop() {
|
|
3326
|
-
stopped
|
|
3589
|
+
if (stopped === undefined) {
|
|
3590
|
+
const requested = options.stopDrainTimeoutMs ?? DEFAULT_STOP_DRAIN_TIMEOUT_MS;
|
|
3591
|
+
// A stop cannot refuse to happen over its own setting, so a window
|
|
3592
|
+
// that would misfire (negative, NaN, or past Node's timer limit,
|
|
3593
|
+
// where setTimeout fires at once) falls back rather than throwing.
|
|
3594
|
+
const usable = Number.isFinite(requested) && requested >= 0 && requested <= MAX_TIMER_DELAY_MS;
|
|
3595
|
+
if (!usable) {
|
|
3596
|
+
warn(`stop drain window ${String(requested)}ms is not usable; using ${DEFAULT_STOP_DRAIN_TIMEOUT_MS}ms`);
|
|
3597
|
+
}
|
|
3598
|
+
stopped = shutdown(usable ? requested : DEFAULT_STOP_DRAIN_TIMEOUT_MS, 'stop').then(() => undefined);
|
|
3599
|
+
}
|
|
3327
3600
|
return stopped;
|
|
3328
3601
|
},
|
|
3329
3602
|
dispatch,
|
|
@@ -3343,7 +3616,7 @@ export const createGateway = (options = {}) => {
|
|
|
3343
3616
|
const transcript = workItemState(session, idempotencyKey) !== undefined ? session : await archivedRepeat(session, idempotencyKey);
|
|
3344
3617
|
// Accepted as addressed: a turn nobody asked for (`addressed: false`,
|
|
3345
3618
|
// stored overheard) was this agent hearing the message, not taking it.
|
|
3346
|
-
const keyed = transcript?.messages.findLast((message) => message
|
|
3619
|
+
const keyed = transcript?.messages.findLast((message) => carriesKey(message, idempotencyKey));
|
|
3347
3620
|
return keyed !== undefined && keyed.overheard !== true;
|
|
3348
3621
|
},
|
|
3349
3622
|
async sessionRouting(sessionId) {
|
|
@@ -3370,8 +3643,12 @@ export const createGateway = (options = {}) => {
|
|
|
3370
3643
|
// said, for text and file alike, not what the channel delivered: a
|
|
3371
3644
|
// post Slack refused or an upload that failed reads as spoken here.
|
|
3372
3645
|
// The channel's own record has the delivery, and this is the
|
|
3373
|
-
// reconstruction for a process that lost it
|
|
3374
|
-
//
|
|
3646
|
+
// reconstruction for a process that lost it. Still open: a delivery
|
|
3647
|
+
// record the channel writes back into the session would make this
|
|
3648
|
+
// exact (a refused post, a failed upload, a line streamed into a
|
|
3649
|
+
// placeholder by a process that died before saving its reply). It is
|
|
3650
|
+
// a new gateway seam, and it has to be ordered against the overhears
|
|
3651
|
+
// queued on the same session chain, so it is not built yet.
|
|
3375
3652
|
const spoken = (message) => (message.role === 'assistant' && message.content.trim().length > 0)
|
|
3376
3653
|
|| (message.role === 'tool' && message.toolResult !== undefined && filePathsOf(message.toolResult).length > 0);
|
|
3377
3654
|
const spokeIndex = session.messages.findLastIndex(spoken);
|