@nylorun/runtime 0.10.0-beta → 0.12.0-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +86 -0
- package/README.md +53 -51
- package/dist/adapters/execution/restate.d.ts +130 -0
- package/dist/adapters/execution/restate.js +442 -0
- package/dist/adapters/sandbox/virtual.js +1 -1
- package/dist/adapters/streams/s2.d.ts +18 -0
- package/dist/adapters/streams/s2.js +361 -0
- package/dist/core/executors.d.ts +4 -1
- package/dist/core/executors.js +20 -1
- package/dist/core/flow-host.d.ts +93 -132
- package/dist/core/flow-host.js +282 -547
- package/dist/core/sandbox-routes.d.ts +13 -4
- package/dist/core/sandbox-routes.js +14 -12
- package/dist/execution/memory.d.ts +54 -0
- package/dist/execution/memory.js +217 -0
- package/dist/execution/types.d.ts +125 -0
- package/dist/execution/types.js +52 -0
- package/dist/host/config-for.d.ts +2 -1
- package/dist/host/config-for.js +3 -21
- package/dist/host/config.d.ts +0 -8
- package/dist/host/create-host.d.ts +36 -1
- package/dist/host/create-host.js +51 -24
- package/dist/host/execution.d.ts +61 -0
- package/dist/host/execution.js +69 -0
- package/dist/host/http.d.ts +4 -0
- package/dist/host/http.js +6 -0
- package/dist/host/index.d.ts +3 -1
- package/dist/host/index.js +2 -0
- package/dist/host/main.js +106 -47
- package/dist/host/stack-config.d.ts +72 -0
- package/dist/host/stack-config.js +193 -0
- package/dist/index.d.ts +1 -1
- package/dist/infra/database.d.ts +16 -0
- package/dist/infra/database.js +31 -0
- package/dist/infra/execution.d.ts +39 -0
- package/dist/infra/execution.js +68 -0
- package/dist/infra/index.d.ts +42 -0
- package/dist/infra/index.js +56 -0
- package/dist/infra/readiness.d.ts +33 -0
- package/dist/infra/readiness.js +53 -0
- package/dist/infra/streams.d.ts +17 -0
- package/dist/infra/streams.js +33 -0
- package/dist/infra/workers.d.ts +39 -0
- package/dist/infra/workers.js +31 -0
- package/dist/model/pi-model.d.ts +2 -2
- package/dist/model/pi-model.js +5 -5
- package/dist/sandbox/manager.d.ts +26 -8
- package/dist/sandbox/manager.js +95 -47
- package/dist/sandbox/select.d.ts +1 -1
- package/dist/sandbox/select.js +4 -5
- package/dist/sandbox/types.d.ts +2 -2
- package/dist/store/canonical.d.ts +5 -0
- package/dist/store/canonical.js +17 -0
- package/dist/store/cursor.d.ts +9 -0
- package/dist/store/cursor.js +18 -0
- package/dist/store/memory.d.ts +57 -0
- package/dist/store/memory.js +645 -0
- package/dist/store/ownership.d.ts +19 -0
- package/dist/store/ownership.js +33 -0
- package/dist/store/postgres/connect.d.ts +25 -0
- package/dist/store/postgres/connect.js +25 -0
- package/dist/store/postgres/locking.d.ts +16 -0
- package/dist/store/postgres/locking.js +22 -0
- package/dist/store/postgres/migrations/001_initial.d.ts +27 -0
- package/dist/store/postgres/migrations/001_initial.js +196 -0
- package/dist/store/postgres/migrations/002_session_owner.d.ts +6 -0
- package/dist/store/postgres/migrations/002_session_owner.js +13 -0
- package/dist/store/postgres/migrations/index.d.ts +49 -0
- package/dist/store/postgres/migrations/index.js +76 -0
- package/dist/store/postgres/names.d.ts +11 -0
- package/dist/store/postgres/names.js +42 -0
- package/dist/store/postgres/store.d.ts +11 -0
- package/dist/store/postgres/store.js +797 -0
- package/dist/store/postgres/tenants.d.ts +90 -0
- package/dist/store/postgres/tenants.js +214 -0
- package/dist/store/types.d.ts +442 -0
- package/dist/store/types.js +10 -0
- package/dist/streams/basin.d.ts +23 -0
- package/dist/streams/basin.js +51 -0
- package/dist/streams/memory.d.ts +25 -0
- package/dist/streams/memory.js +124 -0
- package/dist/streams/relay.d.ts +37 -0
- package/dist/streams/relay.js +254 -0
- package/dist/streams/types.d.ts +115 -0
- package/dist/streams/types.js +86 -0
- package/dist/tenant/actions.d.ts +153 -0
- package/dist/tenant/actions.js +191 -0
- package/dist/tenant/advance.d.ts +15 -0
- package/dist/tenant/advance.js +439 -0
- package/dist/tenant/auth.d.ts +32 -0
- package/dist/tenant/auth.js +169 -0
- package/dist/tenant/commands.d.ts +3 -0
- package/dist/tenant/commands.js +393 -0
- package/dist/tenant/context.d.ts +147 -0
- package/dist/tenant/context.js +48 -0
- package/dist/tenant/effects.d.ts +20 -0
- package/dist/tenant/effects.js +564 -0
- package/dist/tenant/envelope.d.ts +0 -3
- package/dist/tenant/envelope.js +2 -34
- package/dist/tenant/ephemeral.d.ts +15 -4
- package/dist/tenant/ephemeral.js +106 -38
- package/dist/tenant/http.d.ts +38 -0
- package/dist/tenant/http.js +53 -0
- package/dist/tenant/live.d.ts +92 -0
- package/dist/tenant/live.js +290 -0
- package/dist/tenant/model-setting.d.ts +16 -0
- package/dist/tenant/model-setting.js +12 -0
- package/dist/tenant/module.d.ts +29 -8
- package/dist/tenant/module.js +165 -158
- package/dist/tenant/paths.d.ts +12 -8
- package/dist/tenant/paths.js +38 -12
- package/dist/tenant/principals.d.ts +12 -13
- package/dist/tenant/principals.js +50 -38
- package/dist/tenant/quarantine-error.d.ts +1 -1
- package/dist/tenant/quarantine-error.js +2 -8
- package/dist/tenant/quarantine.d.ts +0 -8
- package/dist/tenant/quarantine.js +8 -30
- package/dist/tenant/reset.d.ts +7 -6
- package/dist/tenant/reset.js +7 -51
- package/dist/tenant/routes-tenant.d.ts +12 -0
- package/dist/tenant/routes-tenant.js +137 -0
- package/dist/tenant/routes.d.ts +9 -0
- package/dist/tenant/routes.js +151 -0
- package/dist/tenant/runtime.d.ts +59 -58
- package/dist/tenant/runtime.js +243 -2201
- package/dist/tenant/scheduler.d.ts +25 -0
- package/dist/tenant/scheduler.js +84 -0
- package/dist/tenant/session.d.ts +60 -0
- package/dist/tenant/session.js +94 -0
- package/dist/tenant/sessions.d.ts +40 -0
- package/dist/tenant/sessions.js +163 -0
- package/dist/tenant/status.d.ts +13 -4
- package/dist/tenant/status.js +76 -41
- package/dist/tenant/store-memory.d.ts +9 -12
- package/dist/tenant/store-memory.js +79 -68
- package/dist/tenant/store-pg.d.ts +15 -0
- package/dist/tenant/store-pg.js +140 -0
- package/dist/tenant/streams.d.ts +126 -0
- package/dist/tenant/streams.js +419 -0
- package/dist/tenant/sweep.d.ts +15 -0
- package/dist/tenant/sweep.js +122 -0
- package/dist/tenant/transcript.d.ts +104 -0
- package/dist/tenant/transcript.js +58 -0
- package/dist/tenant/types.d.ts +101 -15
- package/dist/tenant/types.js +23 -1
- package/dist/tenant/worker.d.ts +84 -0
- package/dist/tenant/worker.js +189 -0
- package/dist/vault/service.d.ts +44 -31
- package/dist/vault/service.js +454 -360
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +15 -14
- package/dist/adapters/sandbox/microsandbox.d.ts +0 -2
- package/dist/adapters/sandbox/microsandbox.js +0 -238
- package/dist/core/store.d.ts +0 -32
- package/dist/core/store.js +0 -143
- package/dist/launcher/commands.d.ts +0 -24
- package/dist/launcher/commands.js +0 -260
- package/dist/launcher/ensure-config.d.ts +0 -16
- package/dist/launcher/ensure-config.js +0 -79
- package/dist/launcher/errors.d.ts +0 -12
- package/dist/launcher/errors.js +0 -22
- package/dist/launcher/host-config.d.ts +0 -17
- package/dist/launcher/host-config.js +0 -64
- package/dist/launcher/host-state.d.ts +0 -17
- package/dist/launcher/host-state.js +0 -61
- package/dist/launcher/lifecycle.d.ts +0 -66
- package/dist/launcher/lifecycle.js +0 -407
- package/dist/launcher/locks.d.ts +0 -26
- package/dist/launcher/locks.js +0 -117
- package/dist/launcher/logs.d.ts +0 -18
- package/dist/launcher/logs.js +0 -117
- package/dist/launcher/main.d.ts +0 -2
- package/dist/launcher/main.js +0 -66
- package/dist/launcher/output.d.ts +0 -11
- package/dist/launcher/output.js +0 -68
- package/dist/launcher/paths.d.ts +0 -20
- package/dist/launcher/paths.js +0 -36
- package/dist/launcher/probe.d.ts +0 -23
- package/dist/launcher/probe.js +0 -72
- package/dist/launcher/protocol.d.ts +0 -62
- package/dist/launcher/protocol.js +0 -1
- package/dist/launcher/schema-guard.d.ts +0 -6
- package/dist/launcher/schema-guard.js +0 -36
- package/dist/launcher/spawn.d.ts +0 -22
- package/dist/launcher/spawn.js +0 -85
- package/dist/launcher/status.d.ts +0 -6
- package/dist/launcher/status.js +0 -65
- package/dist/tenant/migration.d.ts +0 -16
- package/dist/tenant/migration.js +0 -116
- package/dist/tenant/schema.d.ts +0 -23
- package/dist/tenant/schema.js +0 -151
- package/dist/tenant/store-fs.d.ts +0 -16
- package/dist/tenant/store-fs.js +0 -205
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Durable Streams seam of an open Tenant (architecture §12.4), wired by one
|
|
3
|
+
* `wireStreams(ctx, …)` call in the composition root:
|
|
4
|
+
*
|
|
5
|
+
* - **Basin.** The Tenant's basin is created when the Tenant is created (`createTenantStreams`)
|
|
6
|
+
* and checked when it opens. A missing basin (S2 down at creation, a crash in between) is
|
|
7
|
+
* repaired on first use: a failed `ensureTenant` or relay append retries `ensureTenant` in the
|
|
8
|
+
* background with backoff, and `streamsStatus` reports the basin's state. Only Tenant deletion
|
|
9
|
+
* deletes the basin (`deleteTenantStreams`); a reset never does (s2-lite keeps a deleted basin's
|
|
10
|
+
* name for about a minute).
|
|
11
|
+
* - **Relay.** The outbox relay (`streams/relay.ts`) subscribes to the Session Store's commits,
|
|
12
|
+
* appends committed events to the session's stream, and appends `work_available` to
|
|
13
|
+
* `tenant/work` after a commit that called `t.signalWork()`. It is the only writer of either;
|
|
14
|
+
* business code never publishes or notifies.
|
|
15
|
+
* - **Readers.** History and session SSE read the session's stream (`tenant/live.ts`). This
|
|
16
|
+
* module runs one `tenant/work` reader per Tenant per process, which wakes connected
|
|
17
|
+
* executors, and one `tenant/control` reader, which calls `ctx.abortLocal` for each
|
|
18
|
+
* `session.cancel` and checks the session feeds for each `sessions.reset`.
|
|
19
|
+
* - **Incarnations.** A session's stream is `sessions/<id>/<incarnation>`, with the incarnation
|
|
20
|
+
* stored on the session when it is created (`streams/types.ts`). A reset abandons the
|
|
21
|
+
* streams of the sessions it deletes instead of deleting and re-creating them, so a session
|
|
22
|
+
* created again with the same id (during or after the reset, on any node) starts a new stream
|
|
23
|
+
* at sequence 0. Abandoned streams are deleted best effort by `collectSessionStreams`: right
|
|
24
|
+
* after the reset, and by the Tenant sweep until a collection succeeds. The pending
|
|
25
|
+
* collection is a Tenant setting written before the reset, so it survives a crash.
|
|
26
|
+
* - **Recovery.** `drainOutbox(ctx)` appends leftover outbox rows (after a crash or an S2
|
|
27
|
+
* outage). The Tenant sweep calls it.
|
|
28
|
+
*
|
|
29
|
+
* The caller passes the streams: the Host's S2 streams, or `MemoryStreams` for tests and a
|
|
30
|
+
* local development Host (not durable).
|
|
31
|
+
*/
|
|
32
|
+
import { randomBytes } from "node:crypto";
|
|
33
|
+
import { createRelay, signalCancel, signalSessionsReset, StreamGapError, } from "../streams/relay.js";
|
|
34
|
+
import { CONTROL_STREAM, SESSION_STREAM_PREFIX, WORK_STREAM, parseSessionStream, streamOfSession, } from "../streams/types.js";
|
|
35
|
+
import { announceWork, checkFeeds, sleep } from "./live.js";
|
|
36
|
+
const RETRY_MIN_MS = 100;
|
|
37
|
+
const RETRY_MAX_MS = 2000;
|
|
38
|
+
/** The longest wait between two `ensureTenant` attempts while the basin is being repaired. */
|
|
39
|
+
const BASIN_RETRY_MAX_MS = 5000;
|
|
40
|
+
/** A repair started within this long after the last one succeeded waits for the next failure. */
|
|
41
|
+
const BASIN_REPAIR_COOLDOWN_MS = 1000;
|
|
42
|
+
/** Outbox rows per drain page at open. */
|
|
43
|
+
const OPEN_DRAIN_PAGE = 1000;
|
|
44
|
+
/** How often every session feed is checked against its session (backstop for lost signals). */
|
|
45
|
+
const FEED_CHECK_MS = 30_000;
|
|
46
|
+
/** The Tenant setting that holds a pending collection's token ("" when none is pending). */
|
|
47
|
+
export const COLLECT_SETTING = "streams.collect";
|
|
48
|
+
/** Sessions looked up per transaction while collecting. */
|
|
49
|
+
const COLLECT_LOOKUP_PAGE = 500;
|
|
50
|
+
/** Stream deletions in flight at once while collecting. */
|
|
51
|
+
const COLLECT_PARALLEL = 8;
|
|
52
|
+
const messageOf = (error) => error instanceof Error ? error.message : String(error);
|
|
53
|
+
/**
|
|
54
|
+
* Wires the Tenant to Durable Streams: checks the basin, creates the relay (unless given),
|
|
55
|
+
* starts the work and control readers, and records the handles on `ctx.live.wiring`. Call it
|
|
56
|
+
* once, after `ctx` is built and before the Tenant serves requests.
|
|
57
|
+
*/
|
|
58
|
+
export async function wireStreams(ctx, options) {
|
|
59
|
+
if (ctx.live.wiring)
|
|
60
|
+
throw new Error("Streams are already wired");
|
|
61
|
+
const { store, streams, tenantId } = options;
|
|
62
|
+
const logger = ctx.config.logger;
|
|
63
|
+
const stop = new AbortController();
|
|
64
|
+
// Nothing is reported once the wiring stops: the Tenant (and its log directory) may be gone,
|
|
65
|
+
// and a throwing report in a detached loop would be an unhandled rejection.
|
|
66
|
+
const report = (what) => (error) => {
|
|
67
|
+
if (!stop.signal.aborted)
|
|
68
|
+
logger.warn(what, { message: messageOf(error) });
|
|
69
|
+
};
|
|
70
|
+
const basin = basinKeeper(streams, tenantId, stop.signal, report("stream basin unavailable; retrying"));
|
|
71
|
+
// Wait for the first check so the first commits find the basin; a failure repairs it later.
|
|
72
|
+
await basin.ensure().catch((error) => {
|
|
73
|
+
report("stream basin unavailable; relaying later")(error);
|
|
74
|
+
basin.repair();
|
|
75
|
+
});
|
|
76
|
+
const relay = options.relay ??
|
|
77
|
+
createRelay({
|
|
78
|
+
store,
|
|
79
|
+
streams,
|
|
80
|
+
tenantId,
|
|
81
|
+
onError: (error) => {
|
|
82
|
+
report("event relay failed; events stay in the outbox")(error);
|
|
83
|
+
// The basin may be missing (never created, or deleted underneath); make sure of it.
|
|
84
|
+
if (!(error instanceof StreamGapError))
|
|
85
|
+
basin.repair();
|
|
86
|
+
},
|
|
87
|
+
});
|
|
88
|
+
const work = follow(streams, tenantId, WORK_STREAM, stop.signal, () => announceWork(ctx.live), report("work stream read failed; retrying"));
|
|
89
|
+
const control = follow(streams, tenantId, CONTROL_STREAM, stop.signal, (body) => {
|
|
90
|
+
const signal = body;
|
|
91
|
+
if (signal?.type === "session.cancel" && typeof signal.sessionId === "string")
|
|
92
|
+
ctx.abortLocal(signal.sessionId);
|
|
93
|
+
else if (signal?.type === "sessions.reset")
|
|
94
|
+
void checkFeeds(ctx).catch(report("session feed check failed"));
|
|
95
|
+
}, report("control stream read failed; retrying"));
|
|
96
|
+
// Signals appended from here on reach this process.
|
|
97
|
+
await Promise.all([work, control]);
|
|
98
|
+
// Feeds whose session was reset end on the `sessions.reset` signal; this catches lost ones.
|
|
99
|
+
const feedCheck = setInterval(() => void checkFeeds(ctx).catch(report("session feed check failed")), FEED_CHECK_MS);
|
|
100
|
+
feedCheck.unref();
|
|
101
|
+
let collecting;
|
|
102
|
+
const wiring = {
|
|
103
|
+
streams,
|
|
104
|
+
relay,
|
|
105
|
+
basin: () => basin.status(),
|
|
106
|
+
async drain(limit) {
|
|
107
|
+
await basin.ensure();
|
|
108
|
+
return relay.drain(limit);
|
|
109
|
+
},
|
|
110
|
+
collect() {
|
|
111
|
+
collecting ??= collectSessionStreams(ctx, streams).finally(() => {
|
|
112
|
+
collecting = undefined;
|
|
113
|
+
});
|
|
114
|
+
return collecting;
|
|
115
|
+
},
|
|
116
|
+
async close() {
|
|
117
|
+
stop.abort();
|
|
118
|
+
clearInterval(feedCheck);
|
|
119
|
+
unregisterSweep();
|
|
120
|
+
if (!options.relay)
|
|
121
|
+
await relay.close();
|
|
122
|
+
await collecting?.catch(() => undefined);
|
|
123
|
+
if (options.ownsStreams)
|
|
124
|
+
await streams.close();
|
|
125
|
+
},
|
|
126
|
+
};
|
|
127
|
+
ctx.live.wiring = wiring;
|
|
128
|
+
// The sweep retries a collection a reset left pending (or one a crash interrupted).
|
|
129
|
+
const unregisterSweep = ctx.onSweep(async () => {
|
|
130
|
+
await wiring.collect();
|
|
131
|
+
});
|
|
132
|
+
// Rows a previous process committed but did not relay; the sweep picks up what this misses.
|
|
133
|
+
void (async () => {
|
|
134
|
+
while ((await wiring.drain(OPEN_DRAIN_PAGE)) >= OPEN_DRAIN_PAGE)
|
|
135
|
+
;
|
|
136
|
+
})().catch(report("outbox drain at open failed"));
|
|
137
|
+
return wiring;
|
|
138
|
+
}
|
|
139
|
+
/** The Tenant sweep's outbox recovery: appends leftover rows, at most `limit`. */
|
|
140
|
+
export function drainOutbox(ctx, limit) {
|
|
141
|
+
const wiring = ctx.live.wiring;
|
|
142
|
+
return wiring ? wiring.drain(limit) : Promise.resolve(0);
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Appends `session.cancel` for `sessionId` to `tenant/control`, so the process running its
|
|
146
|
+
* advance aborts it. Call it from `t.afterCommit` after a cancel commits. A lost signal costs
|
|
147
|
+
* latency only (the advance checks the Session Store before every effect), so failures are
|
|
148
|
+
* logged, never thrown.
|
|
149
|
+
*/
|
|
150
|
+
export function signalSessionCancel(ctx, sessionId) {
|
|
151
|
+
const streams = ctx.live.wiring?.streams;
|
|
152
|
+
if (!streams)
|
|
153
|
+
return;
|
|
154
|
+
void signalCancel(streams, ctx.config.tenantId, sessionId).catch((error) => ctx.config.logger.warn("cancel signal failed", {
|
|
155
|
+
sessionId,
|
|
156
|
+
message: messageOf(error),
|
|
157
|
+
}));
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Before a reset that deletes sessions: records a pending collection of their streams in the
|
|
161
|
+
* Tenant settings, so it is retried by the sweep even if this process stops before it runs.
|
|
162
|
+
*/
|
|
163
|
+
export async function requestStreamCollection(ctx) {
|
|
164
|
+
const token = `${Date.now()}-${randomBytes(6).toString("hex")}`;
|
|
165
|
+
await ctx.store.tx((t) => t.putSetting(COLLECT_SETTING, token));
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* After a reset deleted sessions: tells every process with the Tenant open to check its
|
|
169
|
+
* session feeds (`sessions.reset`), and deletes the abandoned streams now, best effort.
|
|
170
|
+
* Failures are logged; the sweep retries the collection.
|
|
171
|
+
*/
|
|
172
|
+
export function sessionStreamsAbandoned(ctx) {
|
|
173
|
+
const wiring = ctx.live.wiring;
|
|
174
|
+
if (!wiring)
|
|
175
|
+
return;
|
|
176
|
+
const warn = (what) => (error) => ctx.config.logger.warn(what, { message: messageOf(error) });
|
|
177
|
+
void signalSessionsReset(wiring.streams, ctx.config.tenantId).catch(warn("sessions.reset signal failed"));
|
|
178
|
+
void wiring.collect().catch(warn("session stream collection failed; the sweep retries"));
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Deletes the Tenant's session streams that no session uses any more: streams of deleted
|
|
182
|
+
* sessions and of earlier incarnations. Runs only while a collection is pending
|
|
183
|
+
* (`requestStreamCollection`), and clears it once every deletion succeeded, unless another
|
|
184
|
+
* reset requested a new one meanwhile.
|
|
185
|
+
*
|
|
186
|
+
* Streams are listed before their sessions are read, so a session created meanwhile keeps
|
|
187
|
+
* its stream: its row committed before the relay created the stream.
|
|
188
|
+
*/
|
|
189
|
+
export async function collectSessionStreams(ctx, streams) {
|
|
190
|
+
const { store } = ctx;
|
|
191
|
+
const tenantId = ctx.config.tenantId;
|
|
192
|
+
const token = await store.tx((t) => t.getSetting(COLLECT_SETTING));
|
|
193
|
+
if (!token)
|
|
194
|
+
return undefined;
|
|
195
|
+
const bySession = new Map();
|
|
196
|
+
for (const name of await streams.listStreams(tenantId, SESSION_STREAM_PREFIX)) {
|
|
197
|
+
const parsed = parseSessionStream(name);
|
|
198
|
+
if (!parsed)
|
|
199
|
+
continue;
|
|
200
|
+
let names = bySession.get(parsed.sessionId);
|
|
201
|
+
if (!names)
|
|
202
|
+
bySession.set(parsed.sessionId, (names = []));
|
|
203
|
+
names.push(name);
|
|
204
|
+
}
|
|
205
|
+
const ids = [...bySession.keys()];
|
|
206
|
+
const current = new Map();
|
|
207
|
+
for (let i = 0; i < ids.length; i += COLLECT_LOOKUP_PAGE) {
|
|
208
|
+
const page = ids.slice(i, i + COLLECT_LOOKUP_PAGE);
|
|
209
|
+
await store.tx(async (t) => {
|
|
210
|
+
for (const id of page) {
|
|
211
|
+
const session = await t.get("sessions", id);
|
|
212
|
+
if (session)
|
|
213
|
+
current.set(id, streamOfSession(session));
|
|
214
|
+
}
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
const abandoned = [...bySession].flatMap(([id, names]) => names.filter((name) => current.get(id) !== name));
|
|
218
|
+
const result = {
|
|
219
|
+
deleted: 0,
|
|
220
|
+
kept: [...bySession.values()].flat().length - abandoned.length,
|
|
221
|
+
failed: 0,
|
|
222
|
+
};
|
|
223
|
+
let next = 0;
|
|
224
|
+
await Promise.all(Array.from({ length: Math.min(COLLECT_PARALLEL, abandoned.length) }, async () => {
|
|
225
|
+
while (next < abandoned.length) {
|
|
226
|
+
const name = abandoned[next++];
|
|
227
|
+
try {
|
|
228
|
+
await streams.deleteStream(tenantId, name);
|
|
229
|
+
result.deleted += 1;
|
|
230
|
+
}
|
|
231
|
+
catch (error) {
|
|
232
|
+
result.failed += 1;
|
|
233
|
+
ctx.config.logger.warn("session stream delete failed; the sweep retries", {
|
|
234
|
+
stream: name,
|
|
235
|
+
message: messageOf(error),
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}));
|
|
240
|
+
if (result.failed === 0)
|
|
241
|
+
await store.tx(async (t) => {
|
|
242
|
+
if ((await t.getSetting(COLLECT_SETTING)) === token)
|
|
243
|
+
await t.putSetting(COLLECT_SETTING, "");
|
|
244
|
+
});
|
|
245
|
+
return result;
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* The streams seam's status for an open Tenant: S2 reachability, the basin, outbox depth and
|
|
249
|
+
* relay lag. Reads the Session Store and probes the streams (at most `probeTimeoutMs`).
|
|
250
|
+
*/
|
|
251
|
+
export async function streamsStatus(ctx, options = {}) {
|
|
252
|
+
const wiring = ctx.live.wiring;
|
|
253
|
+
const now = options.now ?? Date.now;
|
|
254
|
+
const [reachable, stored] = await Promise.all([
|
|
255
|
+
wiring
|
|
256
|
+
? probe(wiring.streams, options.probeTimeoutMs ?? 2000)
|
|
257
|
+
: Promise.resolve(false),
|
|
258
|
+
ctx.store.tx(async (t) => ({
|
|
259
|
+
outbox: await t.outboxStats(),
|
|
260
|
+
collect: await t.getSetting(COLLECT_SETTING),
|
|
261
|
+
})),
|
|
262
|
+
]);
|
|
263
|
+
const oldest = stored.outbox.oldestCreatedAt;
|
|
264
|
+
const oldestAgeMs = oldest === null ? null : Math.max(0, now() - Date.parse(oldest));
|
|
265
|
+
return {
|
|
266
|
+
reachable,
|
|
267
|
+
basin: wiring?.basin() ?? { ready: false, failures: 0, lastError: null },
|
|
268
|
+
outbox: { depth: stored.outbox.depth, oldestAgeMs },
|
|
269
|
+
relayLagMs: oldestAgeMs ?? 0,
|
|
270
|
+
collectionPending: !!stored.collect,
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
/** Resolves true when the streams answer a probe within `timeoutMs` (in-process ones always do). */
|
|
274
|
+
async function probe(streams, timeoutMs) {
|
|
275
|
+
try {
|
|
276
|
+
await streams.probe?.(AbortSignal.timeout(timeoutMs));
|
|
277
|
+
return true;
|
|
278
|
+
}
|
|
279
|
+
catch {
|
|
280
|
+
return false;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* Creates a new Tenant's basin. Call it when the Tenant is created, after its schema commits;
|
|
285
|
+
* retries transient failures, then throws. A failure need not fail the creation: opening the
|
|
286
|
+
* Tenant repairs a missing basin.
|
|
287
|
+
*/
|
|
288
|
+
export async function createTenantStreams(streams, tenantId, options = {}) {
|
|
289
|
+
await withRetries(() => streams.ensureTenant(tenantId), options.attempts ?? 3);
|
|
290
|
+
}
|
|
291
|
+
/**
|
|
292
|
+
* Deletes a Tenant's basin and every stream in it. Call it when the Tenant is deleted (never
|
|
293
|
+
* on reset). Idempotent; retries transient failures, then throws.
|
|
294
|
+
*/
|
|
295
|
+
export async function deleteTenantStreams(streams, tenantId, options = {}) {
|
|
296
|
+
await withRetries(() => streams.deleteTenant(tenantId), options.attempts ?? 5);
|
|
297
|
+
}
|
|
298
|
+
/** Close: stops the readers and relay; the observers were ended by `endAllStreams`. */
|
|
299
|
+
export async function closeStreams(ctx) {
|
|
300
|
+
await ctx.live.wiring?.close();
|
|
301
|
+
}
|
|
302
|
+
// ---------------------------------------------------------------------------
|
|
303
|
+
async function withRetries(task, attempts) {
|
|
304
|
+
let delay = RETRY_MIN_MS;
|
|
305
|
+
for (let attempt = 1;; attempt += 1) {
|
|
306
|
+
try {
|
|
307
|
+
return await task();
|
|
308
|
+
}
|
|
309
|
+
catch (error) {
|
|
310
|
+
if (attempt >= attempts)
|
|
311
|
+
throw error;
|
|
312
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
313
|
+
delay = Math.min(delay * 2, RETRY_MAX_MS);
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
function basinKeeper(streams, tenantId, signal, onError) {
|
|
318
|
+
let ready = false;
|
|
319
|
+
let failures = 0;
|
|
320
|
+
let lastError = null;
|
|
321
|
+
let attempt;
|
|
322
|
+
let repairing = false;
|
|
323
|
+
let repairedAt = 0;
|
|
324
|
+
const ensure = () => {
|
|
325
|
+
if (ready)
|
|
326
|
+
return Promise.resolve();
|
|
327
|
+
attempt ??= streams
|
|
328
|
+
.ensureTenant(tenantId)
|
|
329
|
+
.then(() => {
|
|
330
|
+
ready = true;
|
|
331
|
+
failures = 0;
|
|
332
|
+
lastError = null;
|
|
333
|
+
}, (error) => {
|
|
334
|
+
failures += 1;
|
|
335
|
+
lastError = messageOf(error);
|
|
336
|
+
throw error;
|
|
337
|
+
})
|
|
338
|
+
.finally(() => {
|
|
339
|
+
attempt = undefined;
|
|
340
|
+
});
|
|
341
|
+
return attempt;
|
|
342
|
+
};
|
|
343
|
+
return {
|
|
344
|
+
ensure,
|
|
345
|
+
repair() {
|
|
346
|
+
if (repairing || signal.aborted)
|
|
347
|
+
return;
|
|
348
|
+
if (ready && Date.now() - repairedAt < BASIN_REPAIR_COOLDOWN_MS)
|
|
349
|
+
return;
|
|
350
|
+
repairing = true;
|
|
351
|
+
ready = false;
|
|
352
|
+
void (async () => {
|
|
353
|
+
let delay = RETRY_MIN_MS;
|
|
354
|
+
while (!signal.aborted) {
|
|
355
|
+
try {
|
|
356
|
+
await ensure();
|
|
357
|
+
repairedAt = Date.now();
|
|
358
|
+
break;
|
|
359
|
+
}
|
|
360
|
+
catch (error) {
|
|
361
|
+
if (failures === 1 || failures % 10 === 0)
|
|
362
|
+
onError(error);
|
|
363
|
+
}
|
|
364
|
+
await sleep(delay, signal);
|
|
365
|
+
delay = Math.min(delay * 2, BASIN_RETRY_MAX_MS);
|
|
366
|
+
}
|
|
367
|
+
repairing = false;
|
|
368
|
+
})();
|
|
369
|
+
},
|
|
370
|
+
status: () => ({ ready, failures, lastError }),
|
|
371
|
+
};
|
|
372
|
+
}
|
|
373
|
+
/**
|
|
374
|
+
* Follows a signal stream from its tail at start, calling `onRecord` for each record, until
|
|
375
|
+
* `signal` aborts. Failed reads retry with backoff; a read that ends by itself (the basin was
|
|
376
|
+
* deleted) starts again from the new tail. Resolves once the first tail is known (or failed).
|
|
377
|
+
*/
|
|
378
|
+
function follow(streams, tenantId, stream, signal, onRecord, onError) {
|
|
379
|
+
let started;
|
|
380
|
+
const ready = new Promise((resolve) => (started = resolve));
|
|
381
|
+
void (async () => {
|
|
382
|
+
let from;
|
|
383
|
+
let delay = RETRY_MIN_MS;
|
|
384
|
+
while (!signal.aborted) {
|
|
385
|
+
try {
|
|
386
|
+
try {
|
|
387
|
+
from ??= await streams.tail(tenantId, stream);
|
|
388
|
+
}
|
|
389
|
+
finally {
|
|
390
|
+
started();
|
|
391
|
+
}
|
|
392
|
+
for await (const record of streams.read(tenantId, stream, from, {
|
|
393
|
+
signal,
|
|
394
|
+
})) {
|
|
395
|
+
from = record.seq + 1;
|
|
396
|
+
delay = RETRY_MIN_MS;
|
|
397
|
+
try {
|
|
398
|
+
onRecord(record.body);
|
|
399
|
+
}
|
|
400
|
+
catch (error) {
|
|
401
|
+
onError(error);
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
if (signal.aborted)
|
|
405
|
+
return;
|
|
406
|
+
from = undefined;
|
|
407
|
+
}
|
|
408
|
+
catch (error) {
|
|
409
|
+
if (signal.aborted)
|
|
410
|
+
return;
|
|
411
|
+
onError(error);
|
|
412
|
+
}
|
|
413
|
+
await sleep(delay, signal);
|
|
414
|
+
delay = Math.min(delay * 2, RETRY_MAX_MS);
|
|
415
|
+
}
|
|
416
|
+
started();
|
|
417
|
+
})();
|
|
418
|
+
return ready;
|
|
419
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { TenantContext } from "./context.js";
|
|
2
|
+
export declare function sweep(ctx: TenantContext, options?: {
|
|
3
|
+
afterOpen?: boolean;
|
|
4
|
+
}): Promise<void>;
|
|
5
|
+
/**
|
|
6
|
+
* Expire lapsed Action claims: hooks, fn and verify are offered again, tools become
|
|
7
|
+
* `uncertain`. One transaction per claim, so each locks only its own session.
|
|
8
|
+
*/
|
|
9
|
+
export declare function expireClaims(ctx: Pick<TenantContext, "store">, now?: Date): Promise<number>;
|
|
10
|
+
/** Offer claimed `fn`/`verify` Actions again, one transaction each. Returns how many. */
|
|
11
|
+
export declare function reofferFnVerifyClaims(ctx: Pick<TenantContext, "store">): Promise<number>;
|
|
12
|
+
/** Settle pending workflow `agent` effects whose linked turn already finished. */
|
|
13
|
+
export declare function reconcileLinkedAgents(ctx: Pick<TenantContext, "store" | "wake">): Promise<void>;
|
|
14
|
+
/** Wake `running`/`runnable` sessions without a live owner. Returns the ids woken. */
|
|
15
|
+
export declare function wakeOrphanedSessions(ctx: Pick<TenantContext, "store" | "wake">, now?: Date): Promise<string[]>;
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import { claimedFnVerifyActions, pendingAgentEffects, reconcilePendingAgentEffect, reofferFnVerifyClaim, } from "../core/flow-host.js";
|
|
2
|
+
const BATCH = 100;
|
|
3
|
+
export async function sweep(ctx, options = {}) {
|
|
4
|
+
if (ctx.closing || ctx.closed)
|
|
5
|
+
return;
|
|
6
|
+
const now = new Date();
|
|
7
|
+
const steps = [
|
|
8
|
+
...(options.afterOpen
|
|
9
|
+
? [["reoffer", () => reofferFnVerifyClaims(ctx)]]
|
|
10
|
+
: []),
|
|
11
|
+
["claims", () => expireClaims(ctx, now)],
|
|
12
|
+
["linked", () => reconcileLinkedAgents(ctx)],
|
|
13
|
+
["orphans", () => wakeOrphanedSessions(ctx, now)],
|
|
14
|
+
[
|
|
15
|
+
"sandboxes",
|
|
16
|
+
() => ctx.sandbox.sweep({
|
|
17
|
+
now: now.getTime(),
|
|
18
|
+
sessionExists: async (id) => !!(await ctx.store.tx((t) => t.get("sessions", id))),
|
|
19
|
+
}),
|
|
20
|
+
],
|
|
21
|
+
...[...ctx.sweepHooks].map((hook) => ["hook", hook]),
|
|
22
|
+
];
|
|
23
|
+
let failure;
|
|
24
|
+
for (const [step, run] of steps) {
|
|
25
|
+
if (ctx.closing || ctx.closed)
|
|
26
|
+
return;
|
|
27
|
+
try {
|
|
28
|
+
await run();
|
|
29
|
+
}
|
|
30
|
+
catch (error) {
|
|
31
|
+
ctx.config.logger.warn("tenant sweep step failed", {
|
|
32
|
+
step,
|
|
33
|
+
message: error instanceof Error ? error.message : String(error),
|
|
34
|
+
});
|
|
35
|
+
failure ??= { error };
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
if (failure)
|
|
39
|
+
throw failure.error;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Expire lapsed Action claims: hooks, fn and verify are offered again, tools become
|
|
43
|
+
* `uncertain`. One transaction per claim, so each locks only its own session.
|
|
44
|
+
*/
|
|
45
|
+
export async function expireClaims(ctx, now = new Date()) {
|
|
46
|
+
const { store } = ctx;
|
|
47
|
+
const seen = new Set();
|
|
48
|
+
let expiredCount = 0;
|
|
49
|
+
for (;;) {
|
|
50
|
+
const expired = (await store.tx((t) => t.expiredClaims(now, BATCH))).filter((action) => !seen.has(action.actionId));
|
|
51
|
+
for (const found of expired) {
|
|
52
|
+
seen.add(found.actionId);
|
|
53
|
+
const changed = await store.tx(async (t) => {
|
|
54
|
+
const s = await t.lockSession(found.sessionId);
|
|
55
|
+
// Read again under the session lock: a heartbeat or result may have won.
|
|
56
|
+
const action = await t.get("actions", found.actionId);
|
|
57
|
+
if (!action ||
|
|
58
|
+
action.status !== "claimed" ||
|
|
59
|
+
Date.parse(action.leaseExpiresAt) > now.getTime())
|
|
60
|
+
return false;
|
|
61
|
+
if (action.kind === "hook" ||
|
|
62
|
+
action.kind === "fn" ||
|
|
63
|
+
action.kind === "verify") {
|
|
64
|
+
action.status = "pending";
|
|
65
|
+
action.claimId = null;
|
|
66
|
+
action.leaseExpiresAt = null;
|
|
67
|
+
await t.put("actions", action.actionId, action);
|
|
68
|
+
t.signalWork();
|
|
69
|
+
return true;
|
|
70
|
+
}
|
|
71
|
+
action.status = "uncertain";
|
|
72
|
+
await t.put("actions", action.actionId, action);
|
|
73
|
+
const effect = await t.get("effects", action.actionId);
|
|
74
|
+
if (effect) {
|
|
75
|
+
effect.status = "uncertain";
|
|
76
|
+
await t.put("effects", action.actionId, effect);
|
|
77
|
+
}
|
|
78
|
+
if (s && s.status !== "cancelled" && s.activeTurnId === action.turnId) {
|
|
79
|
+
s.status = "uncertain";
|
|
80
|
+
await t.put("sessions", s.id, s);
|
|
81
|
+
await t.event(s.id, s.activeTurnId, "action.uncertain", {
|
|
82
|
+
actionId: action.actionId,
|
|
83
|
+
...(action.agent ? { agent: action.agent } : {}),
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
return true;
|
|
87
|
+
});
|
|
88
|
+
if (changed)
|
|
89
|
+
expiredCount += 1;
|
|
90
|
+
}
|
|
91
|
+
if (expired.length < BATCH)
|
|
92
|
+
return expiredCount;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
/** Offer claimed `fn`/`verify` Actions again, one transaction each. Returns how many. */
|
|
96
|
+
export async function reofferFnVerifyClaims(ctx) {
|
|
97
|
+
const claimed = await ctx.store.tx((t) => claimedFnVerifyActions(t));
|
|
98
|
+
let count = 0;
|
|
99
|
+
for (const action of claimed)
|
|
100
|
+
if (await ctx.store.tx((t) => reofferFnVerifyClaim(t, action.actionId)))
|
|
101
|
+
count += 1;
|
|
102
|
+
return count;
|
|
103
|
+
}
|
|
104
|
+
/** Settle pending workflow `agent` effects whose linked turn already finished. */
|
|
105
|
+
export async function reconcileLinkedAgents(ctx) {
|
|
106
|
+
const { store } = ctx;
|
|
107
|
+
const effects = await store.tx((t) => pendingAgentEffects(t));
|
|
108
|
+
for (const effect of effects)
|
|
109
|
+
await store.tx((t) => reconcilePendingAgentEffect({
|
|
110
|
+
t,
|
|
111
|
+
effectId: effect.request.effectId,
|
|
112
|
+
schedule: ctx.wake,
|
|
113
|
+
}));
|
|
114
|
+
}
|
|
115
|
+
/** Wake `running`/`runnable` sessions without a live owner. Returns the ids woken. */
|
|
116
|
+
export async function wakeOrphanedSessions(ctx, now = new Date()) {
|
|
117
|
+
const orphans = await ctx.store.tx((t) => t.orphanedSessions(now, BATCH));
|
|
118
|
+
const ids = orphans.map((session) => session.id);
|
|
119
|
+
for (const id of ids)
|
|
120
|
+
await ctx.wake(id, { reason: "recover" });
|
|
121
|
+
return ids;
|
|
122
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Transcript events (Host feature `transcript-events`): what a chat UI shows, written in the
|
|
3
|
+
* transaction that completes the effect, so a replayed effect never writes them again.
|
|
4
|
+
* Payload schemas: `@nylorun/core/contracts` (`parseTranscriptEvent`).
|
|
5
|
+
*/
|
|
6
|
+
import type { HostEffect } from "@nylorun/harness/run";
|
|
7
|
+
/** The model's tool call id and the harness invocation, for tool effects and tool actions. */
|
|
8
|
+
export declare function toolIds(context: unknown): {
|
|
9
|
+
callId: string;
|
|
10
|
+
invocationId: string;
|
|
11
|
+
} | Record<string, never>;
|
|
12
|
+
/**
|
|
13
|
+
* `message.assistant` for a completed model effect: the step's text and tool calls. The
|
|
14
|
+
* provider returns a `ModelCandidate` or a bare string; anything else writes no event.
|
|
15
|
+
*/
|
|
16
|
+
export declare function assistantMessage(request: HostEffect, value: unknown): {
|
|
17
|
+
agent: import("@nylorun/core").AgentRef;
|
|
18
|
+
invocationId: string;
|
|
19
|
+
text: string;
|
|
20
|
+
toolCalls: {
|
|
21
|
+
callId: string;
|
|
22
|
+
name: string;
|
|
23
|
+
input: unknown;
|
|
24
|
+
}[];
|
|
25
|
+
} | {
|
|
26
|
+
agent?: undefined;
|
|
27
|
+
invocationId: string;
|
|
28
|
+
text: string;
|
|
29
|
+
toolCalls: {
|
|
30
|
+
callId: string;
|
|
31
|
+
name: string;
|
|
32
|
+
input: unknown;
|
|
33
|
+
}[];
|
|
34
|
+
} | undefined;
|
|
35
|
+
/** `tool.completed` for an MCP or sandbox tool: its output, or the tool error it reported. */
|
|
36
|
+
export declare function toolCompleted(request: HostEffect, value: unknown): {
|
|
37
|
+
agent: import("@nylorun/core").AgentRef;
|
|
38
|
+
error: {
|
|
39
|
+
code: string;
|
|
40
|
+
message: string;
|
|
41
|
+
};
|
|
42
|
+
output?: undefined;
|
|
43
|
+
capabilityId: string;
|
|
44
|
+
toolName: string;
|
|
45
|
+
} | {
|
|
46
|
+
agent?: undefined;
|
|
47
|
+
error: {
|
|
48
|
+
code: string;
|
|
49
|
+
message: string;
|
|
50
|
+
};
|
|
51
|
+
output?: undefined;
|
|
52
|
+
capabilityId: string;
|
|
53
|
+
toolName: string;
|
|
54
|
+
} | {
|
|
55
|
+
agent: import("@nylorun/core").AgentRef;
|
|
56
|
+
error?: undefined;
|
|
57
|
+
output: unknown;
|
|
58
|
+
capabilityId: string;
|
|
59
|
+
toolName: string;
|
|
60
|
+
} | {
|
|
61
|
+
agent?: undefined;
|
|
62
|
+
error?: undefined;
|
|
63
|
+
output: unknown;
|
|
64
|
+
capabilityId: string;
|
|
65
|
+
toolName: string;
|
|
66
|
+
} | {
|
|
67
|
+
callId: string;
|
|
68
|
+
invocationId: string;
|
|
69
|
+
agent: import("@nylorun/core").AgentRef;
|
|
70
|
+
error: {
|
|
71
|
+
code: string;
|
|
72
|
+
message: string;
|
|
73
|
+
};
|
|
74
|
+
output?: undefined;
|
|
75
|
+
capabilityId: string;
|
|
76
|
+
toolName: string;
|
|
77
|
+
} | {
|
|
78
|
+
callId: string;
|
|
79
|
+
invocationId: string;
|
|
80
|
+
agent?: undefined;
|
|
81
|
+
error: {
|
|
82
|
+
code: string;
|
|
83
|
+
message: string;
|
|
84
|
+
};
|
|
85
|
+
output?: undefined;
|
|
86
|
+
capabilityId: string;
|
|
87
|
+
toolName: string;
|
|
88
|
+
} | {
|
|
89
|
+
callId: string;
|
|
90
|
+
invocationId: string;
|
|
91
|
+
agent: import("@nylorun/core").AgentRef;
|
|
92
|
+
error?: undefined;
|
|
93
|
+
output: unknown;
|
|
94
|
+
capabilityId: string;
|
|
95
|
+
toolName: string;
|
|
96
|
+
} | {
|
|
97
|
+
callId: string;
|
|
98
|
+
invocationId: string;
|
|
99
|
+
agent?: undefined;
|
|
100
|
+
error?: undefined;
|
|
101
|
+
output: unknown;
|
|
102
|
+
capabilityId: string;
|
|
103
|
+
toolName: string;
|
|
104
|
+
} | undefined;
|