@nylorun/runtime 0.12.0-beta → 0.14.0-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +361 -0
- package/README.md +69 -15
- package/dist/adapters/execution/restate.d.ts +7 -2
- package/dist/adapters/execution/restate.js +38 -8
- package/dist/adapters/replication/pgoutput.d.ts +16 -0
- package/dist/adapters/replication/pgoutput.js +169 -0
- package/dist/adapters/streams/s2.js +37 -20
- package/dist/admin-openapi.json +1071 -0
- package/dist/api/a2a/card.d.ts +15 -0
- package/dist/api/a2a/card.js +24 -0
- package/dist/api/a2a/endpoint.d.ts +8 -0
- package/dist/api/a2a/endpoint.js +81 -0
- package/dist/api/a2a/protocol.d.ts +144 -0
- package/dist/api/a2a/protocol.js +188 -0
- package/dist/api/a2a/routes.d.ts +19 -0
- package/dist/api/a2a/routes.js +289 -0
- package/dist/api/a2a/tasks.d.ts +50 -0
- package/dist/api/a2a/tasks.js +186 -0
- package/dist/api/ag-ui/endpoint.d.ts +8 -0
- package/dist/api/ag-ui/endpoint.js +126 -0
- package/dist/api/ag-ui/history.d.ts +7 -0
- package/dist/api/ag-ui/history.js +60 -0
- package/dist/api/ag-ui/routes.d.ts +32 -0
- package/dist/api/ag-ui/routes.js +305 -0
- package/dist/api/ag-ui/session-id.d.ts +1 -0
- package/dist/api/ag-ui/session-id.js +12 -0
- package/dist/api/ag-ui/sse.d.ts +9 -0
- package/dist/api/ag-ui/sse.js +7 -0
- package/dist/api/ag-ui/translate.d.ts +38 -0
- package/dist/api/ag-ui/translate.js +222 -0
- package/dist/api/components.d.ts +7825 -0
- package/dist/api/components.js +83 -0
- package/dist/api/http/app.d.ts +18 -0
- package/dist/api/http/app.js +58 -0
- package/dist/api/http/body.d.ts +4 -0
- package/dist/api/http/body.js +37 -0
- package/dist/api/http/define.d.ts +39 -0
- package/dist/api/http/define.js +130 -0
- package/dist/api/http/respond.d.ts +9 -0
- package/dist/api/http/respond.js +44 -0
- package/dist/api/http/routes/access.d.ts +24 -0
- package/dist/api/http/routes/access.js +259 -0
- package/dist/api/http/routes/actions.d.ts +8 -0
- package/dist/api/http/routes/actions.js +76 -0
- package/dist/api/http/routes/endpoints.d.ts +7 -0
- package/dist/api/http/routes/endpoints.js +100 -0
- package/dist/api/http/routes/sessions.d.ts +3 -0
- package/dist/api/http/routes/sessions.js +205 -0
- package/dist/api/http/routes/tenant.d.ts +10 -0
- package/dist/api/http/routes/tenant.js +172 -0
- package/dist/api/http/routes/vaults.d.ts +8 -0
- package/dist/api/http/routes/vaults.js +167 -0
- package/dist/api/openapi.d.ts +19 -0
- package/dist/api/openapi.js +138 -0
- package/dist/api/route.d.ts +12 -0
- package/dist/api/route.js +4 -0
- package/dist/core/bearer.d.ts +8 -0
- package/dist/core/bearer.js +11 -0
- package/dist/core/flow-host.d.ts +7 -15
- package/dist/core/flow-host.js +19 -41
- package/dist/core/sandbox-routes.d.ts +12 -7
- package/dist/core/sandbox-routes.js +24 -25
- package/dist/execution/memory.d.ts +8 -5
- package/dist/execution/memory.js +33 -13
- package/dist/execution/types.d.ts +38 -6
- package/dist/execution/types.js +12 -3
- package/dist/host/admin-api.d.ts +22 -0
- package/dist/host/admin-api.js +178 -0
- package/dist/host/app.d.ts +34 -0
- package/dist/host/app.js +185 -0
- package/dist/host/config-for.d.ts +2 -0
- package/dist/host/config-for.js +1 -0
- package/dist/host/config.d.ts +11 -0
- package/dist/host/cors.d.ts +27 -0
- package/dist/host/cors.js +64 -0
- package/dist/host/create-host.d.ts +31 -3
- package/dist/host/create-host.js +168 -327
- package/dist/host/http.d.ts +14 -3
- package/dist/host/http.js +64 -23
- package/dist/host/main.js +55 -4
- package/dist/host/stack-config.d.ts +20 -0
- package/dist/host/stack-config.js +55 -0
- package/dist/mcp/pool.d.ts +19 -1
- package/dist/mcp/pool.js +60 -10
- package/dist/model/catalog.d.ts +2 -10
- package/dist/model/classify.d.ts +13 -0
- package/dist/model/classify.js +61 -0
- package/dist/model/models.d.ts +10 -1
- package/dist/model/models.js +14 -6
- package/dist/model/pi-model.d.ts +17 -2
- package/dist/model/pi-model.js +366 -222
- package/dist/openapi.json +25696 -0
- package/dist/sandbox/manager.d.ts +13 -1
- package/dist/sandbox/manager.js +8 -11
- package/dist/sandbox/resolve.d.ts +33 -0
- package/dist/sandbox/resolve.js +81 -0
- package/dist/sandbox/session-sandbox.d.ts +11 -0
- package/dist/sandbox/session-sandbox.js +51 -0
- package/dist/sandbox/share.d.ts +14 -2
- package/dist/sandbox/share.js +33 -3
- package/dist/sandbox/tenant-config.d.ts +37 -0
- package/dist/sandbox/tenant-config.js +51 -0
- package/dist/sandbox/tools.d.ts +7 -9
- package/dist/sandbox/tools.js +11 -3
- package/dist/sandbox/types.d.ts +8 -2
- package/dist/store/event.d.ts +19 -0
- package/dist/store/event.js +49 -0
- package/dist/store/memory.d.ts +20 -8
- package/dist/store/memory.js +265 -100
- package/dist/store/postgres/connect.d.ts +13 -0
- package/dist/store/postgres/connect.js +5 -0
- package/dist/store/postgres/migrations/003_subject_tokens.d.ts +8 -0
- package/dist/store/postgres/migrations/003_subject_tokens.js +42 -0
- package/dist/store/postgres/migrations/004_publishable_keys.d.ts +6 -0
- package/dist/store/postgres/migrations/004_publishable_keys.js +18 -0
- package/dist/store/postgres/migrations/005_action_endpoints.d.ts +6 -0
- package/dist/store/postgres/migrations/005_action_endpoints.js +32 -0
- package/dist/store/postgres/migrations/006_drop_executors.d.ts +11 -0
- package/dist/store/postgres/migrations/006_drop_executors.js +32 -0
- package/dist/store/postgres/migrations/007_event_time.d.ts +7 -0
- package/dist/store/postgres/migrations/007_event_time.js +14 -0
- package/dist/store/postgres/migrations/008_durable_streams.d.ts +13 -0
- package/dist/store/postgres/migrations/008_durable_streams.js +35 -0
- package/dist/store/postgres/migrations/index.d.ts +5 -1
- package/dist/store/postgres/migrations/index.js +22 -2
- package/dist/store/postgres/migrations/shared/index.d.ts +31 -0
- package/dist/store/postgres/migrations/shared/index.js +92 -0
- package/dist/store/postgres/record.d.ts +11 -0
- package/dist/store/postgres/record.js +64 -0
- package/dist/store/postgres/store.js +337 -95
- package/dist/store/postgres/tenants.js +14 -0
- package/dist/store/types.d.ts +176 -81
- package/dist/streams/basin.d.ts +18 -1
- package/dist/streams/basin.js +34 -2
- package/dist/streams/control.d.ts +18 -0
- package/dist/streams/control.js +29 -0
- package/dist/streams/memory.d.ts +3 -1
- package/dist/streams/memory.js +14 -8
- package/dist/streams/relay/core.d.ts +35 -0
- package/dist/streams/relay/core.js +366 -0
- package/dist/streams/relay/memory.d.ts +69 -0
- package/dist/streams/relay/memory.js +176 -0
- package/dist/streams/relay/types.d.ts +77 -0
- package/dist/streams/relay/types.js +7 -0
- package/dist/streams/types.d.ts +66 -36
- package/dist/streams/types.js +18 -53
- package/dist/tenant/access-policy.d.ts +30 -0
- package/dist/tenant/access-policy.js +52 -0
- package/dist/tenant/actions.d.ts +10 -149
- package/dist/tenant/actions.js +5 -167
- package/dist/tenant/advance.js +32 -0
- package/dist/tenant/auth.d.ts +19 -17
- package/dist/tenant/auth.js +90 -93
- package/dist/tenant/browser.d.ts +15 -0
- package/dist/tenant/browser.js +39 -0
- package/dist/tenant/commands.d.ts +18 -2
- package/dist/tenant/commands.js +121 -129
- package/dist/tenant/context.d.ts +75 -18
- package/dist/tenant/context.js +14 -10
- package/dist/tenant/delivery-token.d.ts +41 -0
- package/dist/tenant/delivery-token.js +105 -0
- package/dist/tenant/delivery.d.ts +62 -0
- package/dist/tenant/delivery.js +476 -0
- package/dist/tenant/effects.d.ts +5 -0
- package/dist/tenant/effects.js +138 -48
- package/dist/tenant/endpoints.d.ts +67 -0
- package/dist/tenant/endpoints.js +90 -0
- package/dist/tenant/ephemeral.d.ts +6 -0
- package/dist/tenant/ephemeral.js +60 -36
- package/dist/tenant/http.d.ts +8 -9
- package/dist/tenant/http.js +13 -27
- package/dist/tenant/jwt.d.ts +16 -0
- package/dist/tenant/jwt.js +50 -0
- package/dist/tenant/module.d.ts +3 -0
- package/dist/tenant/module.js +6 -12
- package/dist/tenant/outbound.d.ts +44 -0
- package/dist/tenant/outbound.js +127 -0
- package/dist/tenant/pool.d.ts +0 -2
- package/dist/tenant/pool.js +0 -18
- package/dist/tenant/reset.d.ts +0 -4
- package/dist/tenant/reset.js +0 -5
- package/dist/tenant/runtime.d.ts +10 -20
- package/dist/tenant/runtime.js +59 -42
- package/dist/tenant/scheduler.d.ts +7 -2
- package/dist/tenant/scheduler.js +7 -3
- package/dist/tenant/session-streams.d.ts +125 -0
- package/dist/tenant/session-streams.js +399 -0
- package/dist/tenant/sessions.d.ts +28 -4
- package/dist/tenant/sessions.js +118 -24
- package/dist/tenant/signing-keys.d.ts +55 -0
- package/dist/tenant/signing-keys.js +172 -0
- package/dist/tenant/slim.d.ts +8 -0
- package/dist/tenant/slim.js +21 -0
- package/dist/tenant/status.d.ts +0 -3
- package/dist/tenant/status.js +19 -12
- package/dist/tenant/streams.d.ts +60 -67
- package/dist/tenant/streams.js +140 -208
- package/dist/tenant/subject-limits.d.ts +16 -0
- package/dist/tenant/subject-limits.js +45 -0
- package/dist/tenant/sweep.d.ts +1 -10
- package/dist/tenant/sweep.js +29 -69
- package/dist/tenant/tokens.d.ts +14 -0
- package/dist/tenant/tokens.js +178 -0
- package/dist/tenant/transcript.d.ts +48 -5
- package/dist/tenant/transcript.js +58 -1
- package/dist/tenant/types.d.ts +29 -7
- package/dist/tenant/worker.d.ts +3 -1
- package/dist/tenant/worker.js +9 -3
- package/dist/vault/service.d.ts +18 -2
- package/dist/vault/service.js +57 -13
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +20 -8
- package/dist/core/executors.d.ts +0 -44
- package/dist/core/executors.js +0 -95
- package/dist/streams/relay.d.ts +0 -37
- package/dist/streams/relay.js +0 -254
- package/dist/tenant/live.d.ts +0 -92
- package/dist/tenant/live.js +0 -290
- package/dist/tenant/routes-tenant.d.ts +0 -12
- package/dist/tenant/routes-tenant.js +0 -137
- package/dist/tenant/routes.d.ts +0 -9
- package/dist/tenant/routes.js +0 -151
package/dist/tenant/streams.js
CHANGED
|
@@ -1,145 +1,160 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
* and checked when it opens. A missing basin (S2 down at creation, a crash in between) is
|
|
7
|
-
* repaired on first use: a failed `ensureTenant` or relay append retries `ensureTenant` in the
|
|
8
|
-
* background with backoff, and `streamsStatus` reports the basin's state. Only Tenant deletion
|
|
9
|
-
* deletes the basin (`deleteTenantStreams`); a reset never does (s2-lite keeps a deleted basin's
|
|
10
|
-
* name for about a minute).
|
|
11
|
-
* - **Relay.** The outbox relay (`streams/relay.ts`) subscribes to the Session Store's commits,
|
|
12
|
-
* appends committed events to the session's stream, and appends `work_available` to
|
|
13
|
-
* `tenant/work` after a commit that called `t.signalWork()`. It is the only writer of either;
|
|
14
|
-
* business code never publishes or notifies.
|
|
15
|
-
* - **Readers.** History and session SSE read the session's stream (`tenant/live.ts`). This
|
|
16
|
-
* module runs one `tenant/work` reader per Tenant per process, which wakes connected
|
|
17
|
-
* executors, and one `tenant/control` reader, which calls `ctx.abortLocal` for each
|
|
18
|
-
* `session.cancel` and checks the session feeds for each `sessions.reset`.
|
|
19
|
-
* - **Incarnations.** A session's stream is `sessions/<id>/<incarnation>`, with the incarnation
|
|
20
|
-
* stored on the session when it is created (`streams/types.ts`). A reset abandons the
|
|
21
|
-
* streams of the sessions it deletes instead of deleting and re-creating them, so a session
|
|
22
|
-
* created again with the same id (during or after the reset, on any node) starts a new stream
|
|
23
|
-
* at sequence 0. Abandoned streams are deleted best effort by `collectSessionStreams`: right
|
|
24
|
-
* after the reset, and by the Tenant sweep until a collection succeeds. The pending
|
|
25
|
-
* collection is a Tenant setting written before the reset, so it survives a crash.
|
|
26
|
-
* - **Recovery.** `drainOutbox(ctx)` appends leftover outbox rows (after a crash or an S2
|
|
27
|
-
* outage). The Tenant sweep calls it.
|
|
28
|
-
*
|
|
29
|
-
* The caller passes the streams: the Host's S2 streams, or `MemoryStreams` for tests and a
|
|
30
|
-
* local development Host (not durable).
|
|
31
|
-
*/
|
|
32
|
-
import { randomBytes } from "node:crypto";
|
|
33
|
-
import { createRelay, signalCancel, signalSessionsReset, StreamGapError, } from "../streams/relay.js";
|
|
34
|
-
import { CONTROL_STREAM, SESSION_STREAM_PREFIX, WORK_STREAM, parseSessionStream, streamOfSession, } from "../streams/types.js";
|
|
35
|
-
import { announceWork, checkFeeds, sleep } from "./live.js";
|
|
1
|
+
import { basinOf } from "../streams/basin.js";
|
|
2
|
+
import { signalCancel, signalSessionsReset } from "../streams/control.js";
|
|
3
|
+
import { createStreamRelay, } from "../streams/relay/core.js";
|
|
4
|
+
import { CONTROL_STREAM, } from "../streams/types.js";
|
|
5
|
+
import { checkSessionStreams, currentBasin, endSubjectStreams, sleep, } from "./session-streams.js";
|
|
36
6
|
const RETRY_MIN_MS = 100;
|
|
37
7
|
const RETRY_MAX_MS = 2000;
|
|
38
8
|
/** The longest wait between two `ensureTenant` attempts while the basin is being repaired. */
|
|
39
9
|
const BASIN_RETRY_MAX_MS = 5000;
|
|
40
10
|
/** A repair started within this long after the last one succeeded waits for the next failure. */
|
|
41
11
|
const BASIN_REPAIR_COOLDOWN_MS = 1000;
|
|
42
|
-
/**
|
|
43
|
-
const OPEN_DRAIN_PAGE = 1000;
|
|
44
|
-
/** How often every session feed is checked against its session (backstop for lost signals). */
|
|
12
|
+
/** How often every session stream is checked against its session (backstop for lost signals). */
|
|
45
13
|
const FEED_CHECK_MS = 30_000;
|
|
46
|
-
/**
|
|
47
|
-
|
|
48
|
-
/** Sessions looked up per transaction while collecting. */
|
|
49
|
-
const COLLECT_LOOKUP_PAGE = 500;
|
|
50
|
-
/** Stream deletions in flight at once while collecting. */
|
|
51
|
-
const COLLECT_PARALLEL = 8;
|
|
14
|
+
/** How long a retired basin stays for readers that have not moved yet. */
|
|
15
|
+
const RETIRE_GRACE_MS = 60_000;
|
|
52
16
|
const messageOf = (error) => error instanceof Error ? error.message : String(error);
|
|
53
17
|
/**
|
|
54
|
-
* Wires the Tenant to Durable Streams:
|
|
55
|
-
*
|
|
56
|
-
*
|
|
18
|
+
* Wires the Tenant to Durable Streams: reads its basin generation, checks the basin, starts
|
|
19
|
+
* the Tenant's relay (unless the Host relays), the control reader and the retired basins'
|
|
20
|
+
* deletion, and records the handles on `ctx.sessionStreams.wiring`. Call it once, after
|
|
21
|
+
* `ctx` is built and before the Tenant serves requests.
|
|
57
22
|
*/
|
|
58
23
|
export async function wireStreams(ctx, options) {
|
|
59
|
-
if (ctx.
|
|
24
|
+
if (ctx.sessionStreams.wiring)
|
|
60
25
|
throw new Error("Streams are already wired");
|
|
61
26
|
const { store, streams, tenantId } = options;
|
|
62
27
|
const logger = ctx.config.logger;
|
|
63
28
|
const stop = new AbortController();
|
|
29
|
+
const graceMs = options.retireGraceMs ?? RETIRE_GRACE_MS;
|
|
64
30
|
// Nothing is reported once the wiring stops: the Tenant (and its log directory) may be gone,
|
|
65
31
|
// and a throwing report in a detached loop would be an unhandled rejection.
|
|
66
32
|
const report = (what) => (error) => {
|
|
67
33
|
if (!stop.signal.aborted)
|
|
68
34
|
logger.warn(what, { message: messageOf(error) });
|
|
69
35
|
};
|
|
70
|
-
const
|
|
36
|
+
const generations = await store.tx((t) => t.basinGenerations());
|
|
37
|
+
ctx.sessionStreams.generation = generations.current;
|
|
38
|
+
const basin = basinKeeper(streams, () => currentBasin(ctx), stop.signal, report("stream basin unavailable; retrying"));
|
|
71
39
|
// Wait for the first check so the first commits find the basin; a failure repairs it later.
|
|
72
40
|
await basin.ensure().catch((error) => {
|
|
73
41
|
report("stream basin unavailable; relaying later")(error);
|
|
74
42
|
basin.repair();
|
|
75
43
|
});
|
|
76
|
-
const relay = options.
|
|
77
|
-
|
|
78
|
-
|
|
44
|
+
const relay = options.hostRelay
|
|
45
|
+
? undefined
|
|
46
|
+
: createStreamRelay({
|
|
47
|
+
source: commitSource(store, tenantId),
|
|
48
|
+
record: store.record(),
|
|
79
49
|
streams,
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
// The basin may be missing (never created, or deleted underneath); make sure of it.
|
|
84
|
-
if (!(error instanceof StreamGapError))
|
|
85
|
-
basin.repair();
|
|
50
|
+
log: (message, fields) => {
|
|
51
|
+
if (!stop.signal.aborted)
|
|
52
|
+
logger.info(message, fields);
|
|
86
53
|
},
|
|
87
54
|
});
|
|
88
|
-
|
|
89
|
-
|
|
55
|
+
relay?.start();
|
|
56
|
+
// One control reader, on the current basin; moving to a new generation restarts it there.
|
|
57
|
+
let control = new AbortController();
|
|
58
|
+
const startControl = () => follow(streams, currentBasin(ctx), CONTROL_STREAM, AbortSignal.any([stop.signal, control.signal]), (body) => {
|
|
90
59
|
const signal = body;
|
|
91
60
|
if (signal?.type === "session.cancel" && typeof signal.sessionId === "string")
|
|
92
61
|
ctx.abortLocal(signal.sessionId);
|
|
93
62
|
else if (signal?.type === "sessions.reset")
|
|
94
|
-
void
|
|
63
|
+
void checkSessionStreams(ctx).catch(report("session stream check failed"));
|
|
64
|
+
else if (signal?.type === "subject.revoked" &&
|
|
65
|
+
typeof signal.subject === "string" &&
|
|
66
|
+
typeof signal.epoch === "number")
|
|
67
|
+
endSubjectStreams(ctx.sessionStreams, signal.subject, signal.epoch);
|
|
95
68
|
}, report("control stream read failed; retrying"));
|
|
96
69
|
// Signals appended from here on reach this process.
|
|
97
|
-
await
|
|
98
|
-
//
|
|
99
|
-
const feedCheck = setInterval(() => void
|
|
70
|
+
await startControl();
|
|
71
|
+
// Streams whose session was reset end on the `sessions.reset` signal; this catches lost ones.
|
|
72
|
+
const feedCheck = setInterval(() => void checkSessionStreams(ctx).catch(report("session stream check failed")), FEED_CHECK_MS);
|
|
100
73
|
feedCheck.unref();
|
|
101
|
-
|
|
74
|
+
const retiring = new Set();
|
|
75
|
+
const timers = new Set();
|
|
102
76
|
const wiring = {
|
|
103
77
|
streams,
|
|
104
78
|
relay,
|
|
105
79
|
basin: () => basin.status(),
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
80
|
+
moveTo(generation) {
|
|
81
|
+
if (stop.signal.aborted || generation === ctx.sessionStreams.generation)
|
|
82
|
+
return;
|
|
83
|
+
ctx.sessionStreams.generation = generation;
|
|
84
|
+
basin.repair();
|
|
85
|
+
control.abort();
|
|
86
|
+
control = new AbortController();
|
|
87
|
+
void startControl();
|
|
109
88
|
},
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
89
|
+
retire(retired) {
|
|
90
|
+
for (const generation of retired) {
|
|
91
|
+
if (generation === ctx.sessionStreams.generation || retiring.has(generation))
|
|
92
|
+
continue;
|
|
93
|
+
retiring.add(generation);
|
|
94
|
+
const timer = setTimeout(() => {
|
|
95
|
+
timers.delete(timer);
|
|
96
|
+
void deleteRetired(generation).finally(() => retiring.delete(generation));
|
|
97
|
+
}, graceMs);
|
|
98
|
+
timer.unref();
|
|
99
|
+
timers.add(timer);
|
|
100
|
+
}
|
|
115
101
|
},
|
|
116
102
|
async close() {
|
|
117
103
|
stop.abort();
|
|
118
104
|
clearInterval(feedCheck);
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
await collecting?.catch(() => undefined);
|
|
105
|
+
for (const timer of timers)
|
|
106
|
+
clearTimeout(timer);
|
|
107
|
+
await relay?.stop();
|
|
123
108
|
if (options.ownsStreams)
|
|
124
109
|
await streams.close();
|
|
125
110
|
},
|
|
126
111
|
};
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
;
|
|
136
|
-
|
|
112
|
+
/** Deletes a retired generation's basin, then forgets it. A failure retries on next open. */
|
|
113
|
+
async function deleteRetired(generation) {
|
|
114
|
+
try {
|
|
115
|
+
await withRetries(() => streams.deleteTenant(basinOf(tenantId, generation)), 5);
|
|
116
|
+
await store.tx((t) => t.forgetRetiredGeneration(generation));
|
|
117
|
+
logger.info("retired stream basin deleted", { generation });
|
|
118
|
+
}
|
|
119
|
+
catch (error) {
|
|
120
|
+
report("retired stream basin not deleted; retrying when the Tenant opens")(error);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
ctx.sessionStreams.wiring = wiring;
|
|
124
|
+
wiring.retire(generations.retired);
|
|
137
125
|
return wiring;
|
|
138
126
|
}
|
|
139
|
-
/**
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
127
|
+
/**
|
|
128
|
+
* The Tenant's own commits as a change source, for a Tenant without the Host's relay. It
|
|
129
|
+
* starts fresh every time (the relay reconciles the record first) and keeps nothing: a commit
|
|
130
|
+
* this process did not see is found by the next reconciliation.
|
|
131
|
+
*/
|
|
132
|
+
function commitSource(store, tenantId) {
|
|
133
|
+
let unsubscribe;
|
|
134
|
+
let n = 0;
|
|
135
|
+
return {
|
|
136
|
+
start(handlers) {
|
|
137
|
+
handlers.onActive({ fresh: true });
|
|
138
|
+
unsubscribe = store.onCommit((commit) => {
|
|
139
|
+
n += 1;
|
|
140
|
+
handlers.onTx({
|
|
141
|
+
endLsn: `0/${n}`,
|
|
142
|
+
rows: commit.events.map((event, i) => ({
|
|
143
|
+
tenantId,
|
|
144
|
+
sessionId: event.sessionId,
|
|
145
|
+
seq: event.seq,
|
|
146
|
+
generation: commit.generations[i],
|
|
147
|
+
body: event,
|
|
148
|
+
})),
|
|
149
|
+
});
|
|
150
|
+
});
|
|
151
|
+
},
|
|
152
|
+
acknowledge() { },
|
|
153
|
+
async reconciled() { },
|
|
154
|
+
async stop() {
|
|
155
|
+
unsubscribe?.();
|
|
156
|
+
},
|
|
157
|
+
};
|
|
143
158
|
}
|
|
144
159
|
/**
|
|
145
160
|
* Appends `session.cancel` for `sessionId` to `tenant/control`, so the process running its
|
|
@@ -148,126 +163,40 @@ export function drainOutbox(ctx, limit) {
|
|
|
148
163
|
* logged, never thrown.
|
|
149
164
|
*/
|
|
150
165
|
export function signalSessionCancel(ctx, sessionId) {
|
|
151
|
-
const streams = ctx.
|
|
166
|
+
const streams = ctx.sessionStreams.wiring?.streams;
|
|
152
167
|
if (!streams)
|
|
153
168
|
return;
|
|
154
|
-
void signalCancel(streams, ctx
|
|
169
|
+
void signalCancel(streams, currentBasin(ctx), sessionId).catch((error) => ctx.config.logger.warn("cancel signal failed", {
|
|
155
170
|
sessionId,
|
|
156
171
|
message: messageOf(error),
|
|
157
172
|
}));
|
|
158
173
|
}
|
|
159
174
|
/**
|
|
160
|
-
*
|
|
161
|
-
*
|
|
175
|
+
* After a reset deleted the Tenant's sessions and moved it to a new basin generation: tells
|
|
176
|
+
* every process with the Tenant open (`sessions.reset` on the old basin's control stream),
|
|
177
|
+
* moves this process's readers, and deletes the old basin after the grace period. Failures
|
|
178
|
+
* are logged; the periodic check and the next open finish the job.
|
|
162
179
|
*/
|
|
163
|
-
export async function
|
|
164
|
-
const
|
|
165
|
-
await ctx.store.tx((t) => t.putSetting(COLLECT_SETTING, token));
|
|
166
|
-
}
|
|
167
|
-
/**
|
|
168
|
-
* After a reset deleted sessions: tells every process with the Tenant open to check its
|
|
169
|
-
* session feeds (`sessions.reset`), and deletes the abandoned streams now, best effort.
|
|
170
|
-
* Failures are logged; the sweep retries the collection.
|
|
171
|
-
*/
|
|
172
|
-
export function sessionStreamsAbandoned(ctx) {
|
|
173
|
-
const wiring = ctx.live.wiring;
|
|
180
|
+
export async function tenantReset(ctx) {
|
|
181
|
+
const wiring = ctx.sessionStreams.wiring;
|
|
174
182
|
if (!wiring)
|
|
175
183
|
return;
|
|
176
|
-
const
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
* Deletes the Tenant's session streams that no session uses any more: streams of deleted
|
|
182
|
-
* sessions and of earlier incarnations. Runs only while a collection is pending
|
|
183
|
-
* (`requestStreamCollection`), and clears it once every deletion succeeded, unless another
|
|
184
|
-
* reset requested a new one meanwhile.
|
|
185
|
-
*
|
|
186
|
-
* Streams are listed before their sessions are read, so a session created meanwhile keeps
|
|
187
|
-
* its stream: its row committed before the relay created the stream.
|
|
188
|
-
*/
|
|
189
|
-
export async function collectSessionStreams(ctx, streams) {
|
|
190
|
-
const { store } = ctx;
|
|
191
|
-
const tenantId = ctx.config.tenantId;
|
|
192
|
-
const token = await store.tx((t) => t.getSetting(COLLECT_SETTING));
|
|
193
|
-
if (!token)
|
|
194
|
-
return undefined;
|
|
195
|
-
const bySession = new Map();
|
|
196
|
-
for (const name of await streams.listStreams(tenantId, SESSION_STREAM_PREFIX)) {
|
|
197
|
-
const parsed = parseSessionStream(name);
|
|
198
|
-
if (!parsed)
|
|
199
|
-
continue;
|
|
200
|
-
let names = bySession.get(parsed.sessionId);
|
|
201
|
-
if (!names)
|
|
202
|
-
bySession.set(parsed.sessionId, (names = []));
|
|
203
|
-
names.push(name);
|
|
204
|
-
}
|
|
205
|
-
const ids = [...bySession.keys()];
|
|
206
|
-
const current = new Map();
|
|
207
|
-
for (let i = 0; i < ids.length; i += COLLECT_LOOKUP_PAGE) {
|
|
208
|
-
const page = ids.slice(i, i + COLLECT_LOOKUP_PAGE);
|
|
209
|
-
await store.tx(async (t) => {
|
|
210
|
-
for (const id of page) {
|
|
211
|
-
const session = await t.get("sessions", id);
|
|
212
|
-
if (session)
|
|
213
|
-
current.set(id, streamOfSession(session));
|
|
214
|
-
}
|
|
215
|
-
});
|
|
216
|
-
}
|
|
217
|
-
const abandoned = [...bySession].flatMap(([id, names]) => names.filter((name) => current.get(id) !== name));
|
|
218
|
-
const result = {
|
|
219
|
-
deleted: 0,
|
|
220
|
-
kept: [...bySession.values()].flat().length - abandoned.length,
|
|
221
|
-
failed: 0,
|
|
222
|
-
};
|
|
223
|
-
let next = 0;
|
|
224
|
-
await Promise.all(Array.from({ length: Math.min(COLLECT_PARALLEL, abandoned.length) }, async () => {
|
|
225
|
-
while (next < abandoned.length) {
|
|
226
|
-
const name = abandoned[next++];
|
|
227
|
-
try {
|
|
228
|
-
await streams.deleteStream(tenantId, name);
|
|
229
|
-
result.deleted += 1;
|
|
230
|
-
}
|
|
231
|
-
catch (error) {
|
|
232
|
-
result.failed += 1;
|
|
233
|
-
ctx.config.logger.warn("session stream delete failed; the sweep retries", {
|
|
234
|
-
stream: name,
|
|
235
|
-
message: messageOf(error),
|
|
236
|
-
});
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
}));
|
|
240
|
-
if (result.failed === 0)
|
|
241
|
-
await store.tx(async (t) => {
|
|
242
|
-
if ((await t.getSetting(COLLECT_SETTING)) === token)
|
|
243
|
-
await t.putSetting(COLLECT_SETTING, "");
|
|
244
|
-
});
|
|
245
|
-
return result;
|
|
184
|
+
const previous = currentBasin(ctx);
|
|
185
|
+
const generations = await ctx.store.tx((t) => t.basinGenerations());
|
|
186
|
+
await signalSessionsReset(wiring.streams, previous, generations.current).catch((error) => ctx.config.logger.warn("sessions.reset signal failed", { message: messageOf(error) }));
|
|
187
|
+
wiring.moveTo(generations.current);
|
|
188
|
+
wiring.retire(generations.retired);
|
|
246
189
|
}
|
|
247
|
-
/**
|
|
248
|
-
* The streams seam's status for an open Tenant: S2 reachability, the basin, outbox depth and
|
|
249
|
-
* relay lag. Reads the Session Store and probes the streams (at most `probeTimeoutMs`).
|
|
250
|
-
*/
|
|
190
|
+
/** The streams seam's status for an open Tenant: S2 reachability, the basin and the relay. */
|
|
251
191
|
export async function streamsStatus(ctx, options = {}) {
|
|
252
|
-
const wiring = ctx.
|
|
253
|
-
const now = options.now ?? Date.now;
|
|
254
|
-
const [reachable, stored] = await Promise.all([
|
|
255
|
-
wiring
|
|
256
|
-
? probe(wiring.streams, options.probeTimeoutMs ?? 2000)
|
|
257
|
-
: Promise.resolve(false),
|
|
258
|
-
ctx.store.tx(async (t) => ({
|
|
259
|
-
outbox: await t.outboxStats(),
|
|
260
|
-
collect: await t.getSetting(COLLECT_SETTING),
|
|
261
|
-
})),
|
|
262
|
-
]);
|
|
263
|
-
const oldest = stored.outbox.oldestCreatedAt;
|
|
264
|
-
const oldestAgeMs = oldest === null ? null : Math.max(0, now() - Date.parse(oldest));
|
|
192
|
+
const wiring = ctx.sessionStreams.wiring;
|
|
265
193
|
return {
|
|
266
|
-
reachable
|
|
194
|
+
reachable: wiring
|
|
195
|
+
? await probe(wiring.streams, options.probeTimeoutMs ?? 2000)
|
|
196
|
+
: false,
|
|
267
197
|
basin: wiring?.basin() ?? { ready: false, failures: 0, lastError: null },
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
collectionPending: !!stored.collect,
|
|
198
|
+
generation: ctx.sessionStreams.generation,
|
|
199
|
+
relay: wiring?.relay?.status() ?? null,
|
|
271
200
|
};
|
|
272
201
|
}
|
|
273
202
|
/** Resolves true when the streams answer a probe within `timeoutMs` (in-process ones always do). */
|
|
@@ -289,15 +218,15 @@ export async function createTenantStreams(streams, tenantId, options = {}) {
|
|
|
289
218
|
await withRetries(() => streams.ensureTenant(tenantId), options.attempts ?? 3);
|
|
290
219
|
}
|
|
291
220
|
/**
|
|
292
|
-
* Deletes a Tenant
|
|
293
|
-
*
|
|
221
|
+
* Deletes every basin of a Tenant and every stream in them. Call it when the Tenant is
|
|
222
|
+
* deleted. Idempotent; retries transient failures, then throws.
|
|
294
223
|
*/
|
|
295
224
|
export async function deleteTenantStreams(streams, tenantId, options = {}) {
|
|
296
|
-
await withRetries(() => streams.deleteTenant(tenantId), options.attempts ?? 5);
|
|
225
|
+
await withRetries(() => streams.deleteTenant(tenantId, { allGenerations: true }), options.attempts ?? 5);
|
|
297
226
|
}
|
|
298
227
|
/** Close: stops the readers and relay; the observers were ended by `endAllStreams`. */
|
|
299
228
|
export async function closeStreams(ctx) {
|
|
300
|
-
await ctx.
|
|
229
|
+
await ctx.sessionStreams.wiring?.close();
|
|
301
230
|
}
|
|
302
231
|
// ---------------------------------------------------------------------------
|
|
303
232
|
async function withRetries(task, attempts) {
|
|
@@ -314,20 +243,23 @@ async function withRetries(task, attempts) {
|
|
|
314
243
|
}
|
|
315
244
|
}
|
|
316
245
|
}
|
|
317
|
-
function basinKeeper(streams,
|
|
246
|
+
function basinKeeper(streams, basinKey, signal, onError) {
|
|
318
247
|
let ready = false;
|
|
248
|
+
let readyFor;
|
|
319
249
|
let failures = 0;
|
|
320
250
|
let lastError = null;
|
|
321
251
|
let attempt;
|
|
322
252
|
let repairing = false;
|
|
323
253
|
let repairedAt = 0;
|
|
324
254
|
const ensure = () => {
|
|
325
|
-
|
|
255
|
+
const key = basinKey();
|
|
256
|
+
if (ready && readyFor === key)
|
|
326
257
|
return Promise.resolve();
|
|
327
258
|
attempt ??= streams
|
|
328
|
-
.ensureTenant(
|
|
259
|
+
.ensureTenant(key)
|
|
329
260
|
.then(() => {
|
|
330
261
|
ready = true;
|
|
262
|
+
readyFor = key;
|
|
331
263
|
failures = 0;
|
|
332
264
|
lastError = null;
|
|
333
265
|
}, (error) => {
|
|
@@ -345,7 +277,7 @@ function basinKeeper(streams, tenantId, signal, onError) {
|
|
|
345
277
|
repair() {
|
|
346
278
|
if (repairing || signal.aborted)
|
|
347
279
|
return;
|
|
348
|
-
if (ready && Date.now() - repairedAt < BASIN_REPAIR_COOLDOWN_MS)
|
|
280
|
+
if (ready && readyFor === basinKey() && Date.now() - repairedAt < BASIN_REPAIR_COOLDOWN_MS)
|
|
349
281
|
return;
|
|
350
282
|
repairing = true;
|
|
351
283
|
ready = false;
|
|
@@ -375,7 +307,7 @@ function basinKeeper(streams, tenantId, signal, onError) {
|
|
|
375
307
|
* `signal` aborts. Failed reads retry with backoff; a read that ends by itself (the basin was
|
|
376
308
|
* deleted) starts again from the new tail. Resolves once the first tail is known (or failed).
|
|
377
309
|
*/
|
|
378
|
-
function follow(streams,
|
|
310
|
+
function follow(streams, basin, stream, signal, onRecord, onError) {
|
|
379
311
|
let started;
|
|
380
312
|
const ready = new Promise((resolve) => (started = resolve));
|
|
381
313
|
void (async () => {
|
|
@@ -384,12 +316,12 @@ function follow(streams, tenantId, stream, signal, onRecord, onError) {
|
|
|
384
316
|
while (!signal.aborted) {
|
|
385
317
|
try {
|
|
386
318
|
try {
|
|
387
|
-
from ??= await streams.tail(
|
|
319
|
+
from ??= await streams.tail(basin, stream);
|
|
388
320
|
}
|
|
389
321
|
finally {
|
|
390
322
|
started();
|
|
391
323
|
}
|
|
392
|
-
for await (const record of streams.read(
|
|
324
|
+
for await (const record of streams.read(basin, stream, from, {
|
|
393
325
|
signal,
|
|
394
326
|
})) {
|
|
395
327
|
from = record.seq + 1;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Subject limits (Host feature `subject-tokens`): a subject token's role may cap how many turns
|
|
3
|
+
* its subject starts per hour and how many run at once. Every turn spends the Tenant's model
|
|
4
|
+
* budget, so these ship with the first credential that reaches the Runtime without an app
|
|
5
|
+
* server in between.
|
|
6
|
+
*
|
|
7
|
+
* Turns per hour is a token bucket per subject (capacity `turnsPerHour`, refilled
|
|
8
|
+
* continuously): no hourly rows and no burst at a window boundary. Concurrent turns counts the
|
|
9
|
+
* subject's sessions with work in flight; `paused` is not counted, so an unanswered approval
|
|
10
|
+
* never locks a person out. Checked inside the command's transaction, after the session row
|
|
11
|
+
* lock, so one subject's messages serialize on the bucket row.
|
|
12
|
+
*/
|
|
13
|
+
import type { RoleLimits } from "@nylorun/core/contracts";
|
|
14
|
+
import type { Tx } from "../store/types.js";
|
|
15
|
+
/** Charges one turn to `subject`, or answers `429 limit_exceeded`. */
|
|
16
|
+
export declare function chargeTurn(t: Tx, subject: string, limits: RoleLimits, now?: number): Promise<void>;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { fail } from "./http.js";
|
|
2
|
+
/** Session statuses that hold a turn in flight. */
|
|
3
|
+
const ACTIVE_STATUSES = ["runnable", "running", "waiting"];
|
|
4
|
+
const CONCURRENT_RETRY_SECONDS = 5;
|
|
5
|
+
/** Floating-point slack when a refill lands exactly on a whole turn. */
|
|
6
|
+
const EPSILON = 1e-9;
|
|
7
|
+
/** Charges one turn to `subject`, or answers `429 limit_exceeded`. */
|
|
8
|
+
export async function chargeTurn(t, subject, limits, now = Date.now()) {
|
|
9
|
+
if (limits.concurrentTurns !== undefined) {
|
|
10
|
+
const active = await t.countOwnerSessions(subject, ACTIVE_STATUSES);
|
|
11
|
+
if (active >= limits.concurrentTurns)
|
|
12
|
+
fail(429, `At most ${limits.concurrentTurns} turns may run at once`, {
|
|
13
|
+
code: "limit_exceeded",
|
|
14
|
+
details: {
|
|
15
|
+
limit: "concurrentTurns",
|
|
16
|
+
retryAfterSeconds: CONCURRENT_RETRY_SECONDS,
|
|
17
|
+
},
|
|
18
|
+
}, { "retry-after": String(CONCURRENT_RETRY_SECONDS) });
|
|
19
|
+
}
|
|
20
|
+
const capacity = limits.turnsPerHour;
|
|
21
|
+
if (capacity === undefined)
|
|
22
|
+
return;
|
|
23
|
+
const perMs = capacity / 3_600_000;
|
|
24
|
+
const usage = await t.lockSubjectUsage({
|
|
25
|
+
subject,
|
|
26
|
+
turnTokens: capacity,
|
|
27
|
+
refilledAt: new Date(now).toISOString(),
|
|
28
|
+
});
|
|
29
|
+
const elapsed = Math.max(0, now - Date.parse(usage.refilledAt));
|
|
30
|
+
const tokens = Math.min(capacity, usage.turnTokens + elapsed * perMs);
|
|
31
|
+
if (tokens < 1 - EPSILON) {
|
|
32
|
+
// Rounded before the ceiling so floating-point dust never adds a second.
|
|
33
|
+
const wait = Math.round(((1 - tokens) / perMs / 1000) * 1000) / 1000;
|
|
34
|
+
const retryAfterSeconds = Math.max(1, Math.ceil(wait));
|
|
35
|
+
fail(429, `At most ${capacity} turns per hour`, {
|
|
36
|
+
code: "limit_exceeded",
|
|
37
|
+
details: { limit: "turnsPerHour", retryAfterSeconds },
|
|
38
|
+
}, { "retry-after": String(retryAfterSeconds) });
|
|
39
|
+
}
|
|
40
|
+
await t.putSubjectUsage({
|
|
41
|
+
subject,
|
|
42
|
+
turnTokens: Math.max(0, tokens - 1),
|
|
43
|
+
refilledAt: new Date(now).toISOString(),
|
|
44
|
+
});
|
|
45
|
+
}
|
package/dist/tenant/sweep.d.ts
CHANGED
|
@@ -1,14 +1,5 @@
|
|
|
1
1
|
import type { TenantContext } from "./context.js";
|
|
2
|
-
export declare function sweep(ctx: TenantContext
|
|
3
|
-
afterOpen?: boolean;
|
|
4
|
-
}): Promise<void>;
|
|
5
|
-
/**
|
|
6
|
-
* Expire lapsed Action claims: hooks, fn and verify are offered again, tools become
|
|
7
|
-
* `uncertain`. One transaction per claim, so each locks only its own session.
|
|
8
|
-
*/
|
|
9
|
-
export declare function expireClaims(ctx: Pick<TenantContext, "store">, now?: Date): Promise<number>;
|
|
10
|
-
/** Offer claimed `fn`/`verify` Actions again, one transaction each. Returns how many. */
|
|
11
|
-
export declare function reofferFnVerifyClaims(ctx: Pick<TenantContext, "store">): Promise<number>;
|
|
2
|
+
export declare function sweep(ctx: TenantContext): Promise<void>;
|
|
12
3
|
/** Settle pending workflow `agent` effects whose linked turn already finished. */
|
|
13
4
|
export declare function reconcileLinkedAgents(ctx: Pick<TenantContext, "store" | "wake">): Promise<void>;
|
|
14
5
|
/** Wake `running`/`runnable` sessions without a live owner. Returns the ids woken. */
|