@mostra/runtime 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin.js +2879 -8
- package/dist/bin.js.map +1 -1
- package/dist/builder/child/main.js +2 -2
- package/dist/{child-Chiavhdx.js → child-BmNZ4a70.js} +2 -2
- package/dist/{child-Chiavhdx.js.map → child-BmNZ4a70.js.map} +1 -1
- package/dist/deps/child/install.js +1 -1
- package/dist/{directories-mJ1qlVht.js → directories-BLJjeSPD.js} +5 -5
- package/dist/directories-BLJjeSPD.js.map +1 -0
- package/dist/{files-FygSJIzK.js → files-DGeWuE4I.js} +2 -2
- package/dist/{files-FygSJIzK.js.map → files-DGeWuE4I.js.map} +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/{instance-EnaZPs14.js → instance-FExcyENK.js} +3341 -396
- package/dist/instance-FExcyENK.js.map +1 -0
- package/package.json +7 -5
- package/src/actions/index.ts +163 -10
- package/src/bin.ts +78 -1
- package/src/gateway/actions.ts +20 -4
- package/src/gateway/browser.ts +118 -95
- package/src/gateway/capabilities.ts +15 -6
- package/src/gateway/index.ts +112 -15
- package/src/gateway/routes.ts +15 -7
- package/src/gc/index.ts +14 -0
- package/src/instance/index.ts +41 -2
- package/src/interaction/ephemeral.ts +32 -23
- package/src/interaction/events.ts +21 -1
- package/src/interaction/imports.ts +317 -0
- package/src/interaction/index.ts +172 -19
- package/src/interaction/outbox.ts +9 -0
- package/src/interaction/results.ts +8 -1
- package/src/interaction/runs.ts +52 -2
- package/src/interaction/submissions.ts +17 -1
- package/src/interaction/tally.ts +129 -0
- package/src/ledger/index.ts +104 -9
- package/src/ledger/migrations.ts +183 -0
- package/src/ledger/types.ts +8 -0
- package/src/link/binding.ts +289 -0
- package/src/link/hosted.ts +65 -0
- package/src/link/http.ts +136 -0
- package/src/link/index.ts +8 -0
- package/src/link/manager.ts +1385 -0
- package/src/link/relay.ts +512 -0
- package/src/live/index.ts +112 -2
- package/src/platform/filesystems.ts +78 -0
- package/src/platform/index.ts +1 -0
- package/src/protocol-server/index.ts +18 -1
- package/src/protocol-server/server.ts +38 -7
- package/src/protocol-server/websocket.ts +161 -0
- package/src/publish/bundle.ts +398 -0
- package/src/publish/manifest.ts +9 -0
- package/src/publish/methods.ts +12 -1
- package/src/publish/publisher.ts +127 -0
- package/src/registry/index.ts +35 -14
- package/src/server/api.ts +439 -0
- package/src/server/auth.ts +613 -0
- package/src/server/config.ts +221 -0
- package/src/server/data.ts +369 -0
- package/src/server/directory.ts +141 -0
- package/src/server/hosted.ts +822 -0
- package/src/server/index.ts +368 -0
- package/src/server/relay.ts +1163 -0
- package/src/share/index.ts +110 -6
- package/dist/directories-mJ1qlVht.js.map +0 -1
- package/dist/instance-EnaZPs14.js.map +0 -1
|
@@ -0,0 +1,1163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The relay on a hosted home (D117, D165 item 4; ARCHITECTURE §15.5, INTERFACES §18.3, FAILURE_MODEL
|
|
3
|
+
* §11.1): a hosted page's runs and plain calls run on the artifact's owner instance, over that
|
|
4
|
+
* instance's binding. The hosted home records each run as a local instance does, then sends it to
|
|
5
|
+
* the owner, which records it under the same ID, runs it in its own pool with its own secrets and
|
|
6
|
+
* reports its progress, log, result and end back (`relay/run.update`). While the owner is away a
|
|
7
|
+
* run waits (`waiting`), as the artifact's offline policy says, in a capped queue that drains in
|
|
8
|
+
* order at a bounded rate when the owner returns; after every reconnect, before anything drains,
|
|
9
|
+
* the runs the owner accepted and hasn't finished are reconciled with it. A run whose end the owner
|
|
10
|
+
* never reported within the offline limit becomes `unknown`, never `failed`, since it may have
|
|
11
|
+
* taken effect; the owner's later report replaces it once.
|
|
12
|
+
*
|
|
13
|
+
* Plain calls aren't durable: they go out only while the owner is connected.
|
|
14
|
+
*/
|
|
15
|
+
import {
|
|
16
|
+
ACTION_ARGS_MAX,
|
|
17
|
+
ArtifactManifest,
|
|
18
|
+
type Batches,
|
|
19
|
+
type CallLine,
|
|
20
|
+
checkArgumentFrames,
|
|
21
|
+
createDiagnostic,
|
|
22
|
+
durationMs,
|
|
23
|
+
HOSTED_OFFLINE_LIMIT_DEFAULT,
|
|
24
|
+
isFinalRunStatus,
|
|
25
|
+
type JsonRpcParams,
|
|
26
|
+
type PortRelayReconcileResult,
|
|
27
|
+
portRelayCallMethod,
|
|
28
|
+
portRelayCancelMethod,
|
|
29
|
+
portRelayLiveMethod,
|
|
30
|
+
portRelayReconcileMethod,
|
|
31
|
+
portRelayRunMethod,
|
|
32
|
+
RpcError,
|
|
33
|
+
type RunState,
|
|
34
|
+
relayCallBatchMethod,
|
|
35
|
+
relayEffectMethod,
|
|
36
|
+
relayLiveValueMethod,
|
|
37
|
+
relayRunUpdateMethod,
|
|
38
|
+
type ServerLiveValueParams,
|
|
39
|
+
TALLY_LIVE_PREFIX,
|
|
40
|
+
} from "@mostra/protocol";
|
|
41
|
+
import type { Actions } from "../actions/index.ts";
|
|
42
|
+
import {
|
|
43
|
+
appendRunOutput,
|
|
44
|
+
type EphemeralBus,
|
|
45
|
+
finishRun,
|
|
46
|
+
type Interaction,
|
|
47
|
+
markRunning,
|
|
48
|
+
markUnknown,
|
|
49
|
+
markWaiting,
|
|
50
|
+
type RunRow,
|
|
51
|
+
runState,
|
|
52
|
+
} from "../interaction/index.ts";
|
|
53
|
+
import type { Ledger, LedgerDb } from "../ledger/index.ts";
|
|
54
|
+
import type { Log } from "../log.ts";
|
|
55
|
+
import { handler, type MethodHandler, type Session } from "../protocol-server/index.ts";
|
|
56
|
+
|
|
57
|
+
/** How long a relay port call may take: the owner answers once it recorded the run. */
|
|
58
|
+
const PORT_CALL_MS = 30_000;
|
|
59
|
+
/** How long a run its connected owner never acknowledged waits before it is sent again. */
|
|
60
|
+
const RESEND_MS = PORT_CALL_MS + 5_000;
|
|
61
|
+
/** Failures of the connection rather than answers of the owner: the run may yet be recorded. */
|
|
62
|
+
const TRANSPORT_FAILURES: ReadonlySet<string> = new Set([
|
|
63
|
+
"MOSTRA_HOST_CONNECTION_CLOSED",
|
|
64
|
+
"MOSTRA_HOST_TIMEOUT",
|
|
65
|
+
]);
|
|
66
|
+
/** The most waiting runs per artifact, by default (a server setting, D165 item 4). */
|
|
67
|
+
export const WAITING_RUNS_DEFAULT = 100;
|
|
68
|
+
/** How many waiting runs go out per second per artifact when the owner returns, by default. */
|
|
69
|
+
export const DRAIN_PER_SECOND_DEFAULT = 2;
|
|
70
|
+
|
|
71
|
+
/** A version's relay facts: its server functions and its offline policy (`hosted.offline`). */
|
|
72
|
+
interface RelayVersion {
|
|
73
|
+
readonly functions: ReadonlySet<string>;
|
|
74
|
+
readonly policy: "queue" | "fail";
|
|
75
|
+
readonly limitMs: number;
|
|
76
|
+
/** Who may trigger its relayed actions (`hosted.actions`, D159): default signed-in viewers. */
|
|
77
|
+
readonly who: "signed-in" | "link";
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Options for {@link createHostedRelay}. */
|
|
81
|
+
export interface HostedRelayOptions {
|
|
82
|
+
readonly ledger: Ledger;
|
|
83
|
+
readonly interaction: Interaction;
|
|
84
|
+
readonly bus: EphemeralBus;
|
|
85
|
+
readonly log: Log;
|
|
86
|
+
/** The artifact's owner link (`lnk_`), the one instance its runs go to (D165 item 1b). */
|
|
87
|
+
readonly ownerOf: (artifactId: string) => string | undefined;
|
|
88
|
+
/** The open session of an owner link that registered `relay`, if one is open. */
|
|
89
|
+
readonly sessionOf: (linkId: string) => number | undefined;
|
|
90
|
+
/** Calls a port method on one session. */
|
|
91
|
+
readonly call: (
|
|
92
|
+
session: number,
|
|
93
|
+
method: string,
|
|
94
|
+
params: JsonRpcParams,
|
|
95
|
+
timeoutMs: number,
|
|
96
|
+
) => Promise<unknown>;
|
|
97
|
+
/** The server functions a version's registry lists. */
|
|
98
|
+
readonly functions: (versionId: string) => ReadonlySet<string>;
|
|
99
|
+
readonly waitingRunsPerArtifact?: number;
|
|
100
|
+
readonly drainPerSecond?: number;
|
|
101
|
+
/**
|
|
102
|
+
* Relayed runs and calls per link (`rateLimits.perLink.relayedRuns`, D159): past it a page is
|
|
103
|
+
* refused before anything is recorded or relayed.
|
|
104
|
+
*/
|
|
105
|
+
readonly perLink?: { readonly count: number; readonly perMs: number } | undefined;
|
|
106
|
+
/** How often waiting and unacknowledged runs are checked against their limits; default 1 s. */
|
|
107
|
+
readonly sweepMs?: number;
|
|
108
|
+
/** An artifact's latest version here, whose producers its viewers want. */
|
|
109
|
+
readonly latestVersion?: (artifactId: string) => string | undefined;
|
|
110
|
+
/** Keeps a value an owner's producer made (`relay/live.value`). */
|
|
111
|
+
readonly storeLive?: (
|
|
112
|
+
artifactId: string,
|
|
113
|
+
versionId: string,
|
|
114
|
+
produced: ServerLiveValueParams,
|
|
115
|
+
) => { readonly revision?: string };
|
|
116
|
+
/** How often viewers' interest in live sources is renewed with the owner; default 20 s. */
|
|
117
|
+
readonly liveRenewMs?: number;
|
|
118
|
+
readonly now?: () => number;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** The relay: the gateway's actions on a hosted home, and the owner's methods. */
|
|
122
|
+
export interface HostedRelay extends Pick<Actions, "call" | "startRun" | "attach" | "cancel"> {
|
|
123
|
+
/** `relay/run.update` and `relay/call.batch`, for the hosted home's protocol server. */
|
|
124
|
+
methods(): MethodHandler[];
|
|
125
|
+
/** An owner session opened: reconcile its accepted runs, then resend and drain (§18.3). */
|
|
126
|
+
connected(session: Session): void;
|
|
127
|
+
/** A session closed: its calls end, its runs show their owner unreachable. */
|
|
128
|
+
disconnected(session: Session): void;
|
|
129
|
+
/** Whether runs wait or calls run: the home stays open while they do. */
|
|
130
|
+
busy(): boolean;
|
|
131
|
+
/** A page read live sources: their producers are asked for on the owner now. */
|
|
132
|
+
liveWanted(artifactId: string): void;
|
|
133
|
+
/** Whether an artifact's owner is out of reach, so its live values can't be refreshed. */
|
|
134
|
+
ownerOffline(artifactId: string): boolean;
|
|
135
|
+
stop(): void;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** A `relay_run` row. */
|
|
139
|
+
interface RelayRow {
|
|
140
|
+
readonly run_id: string;
|
|
141
|
+
readonly owner_link_id: string;
|
|
142
|
+
readonly relay_seq: number;
|
|
143
|
+
readonly attempt: number;
|
|
144
|
+
readonly sent_at: string | null;
|
|
145
|
+
readonly accepted_at: string | null;
|
|
146
|
+
readonly owner_heard_at: string | null;
|
|
147
|
+
readonly offline_limit_at: string;
|
|
148
|
+
readonly last_seq: number;
|
|
149
|
+
readonly cancel_requested_at: string | null;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** A relayed plain call in flight: its lines as they come. */
|
|
153
|
+
interface PendingCall {
|
|
154
|
+
readonly session: number;
|
|
155
|
+
readonly lines: CallLine[];
|
|
156
|
+
wake?: () => void;
|
|
157
|
+
ended: boolean;
|
|
158
|
+
lastSeq: number;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function failed(code: string, message: string): RpcError {
|
|
162
|
+
return new RpcError(createDiagnostic(code as never, { message }));
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const iso = (ms: number): string => new Date(ms).toISOString();
|
|
166
|
+
|
|
167
|
+
/** Creates a hosted home's relay. */
|
|
168
|
+
export function createHostedRelay(options: HostedRelayOptions): HostedRelay {
|
|
169
|
+
const { ledger, interaction, bus, log } = options;
|
|
170
|
+
const now = options.now ?? Date.now;
|
|
171
|
+
const cap = options.waitingRunsPerArtifact ?? WAITING_RUNS_DEFAULT;
|
|
172
|
+
const drainPerSecond = Math.max(1, options.drainPerSecond ?? DRAIN_PER_SECOND_DEFAULT);
|
|
173
|
+
const calls = new Map<string, PendingCall>();
|
|
174
|
+
/** Owner links whose reconcile finished on their current session: only then may runs go out. */
|
|
175
|
+
const ready = new Map<string, number>();
|
|
176
|
+
/** Runs being sent now, so a sweep or a second drain never sends one twice at once. */
|
|
177
|
+
const sending = new Set<string>();
|
|
178
|
+
/** When each run was last sent, for the sweep's resend while the owner stays connected. */
|
|
179
|
+
const lastTry = new Map<string, number>();
|
|
180
|
+
const draining = new Set<string>();
|
|
181
|
+
const versions = new Map<string, RelayVersion>();
|
|
182
|
+
/**
|
|
183
|
+
* The highest relay sequence each owner link's ledger has accepted, as its sessions report it
|
|
184
|
+
* (`relayMark`): a home restored from a backup continues above it, so no sequence is reused.
|
|
185
|
+
*/
|
|
186
|
+
const marks = new Map<string, number>();
|
|
187
|
+
/** The relay mark each open owner session reported, for a resume run again for it. */
|
|
188
|
+
const sessionMarks = new Map<number, number>();
|
|
189
|
+
/** A resume that failed is tried again for its session, with backoff (Opus review of WP-29, 1). */
|
|
190
|
+
const retries = new Map<number, { timer: ReturnType<typeof setTimeout>; attempt: number }>();
|
|
191
|
+
let stopped = false;
|
|
192
|
+
|
|
193
|
+
const versionOf = (versionId: string): RelayVersion => {
|
|
194
|
+
const known = versions.get(versionId);
|
|
195
|
+
if (known) return known;
|
|
196
|
+
const manifestJson = ledger.read(
|
|
197
|
+
(db) =>
|
|
198
|
+
db.get<{ manifest_json: string }>(
|
|
199
|
+
"SELECT manifest_json FROM version WHERE id = ?",
|
|
200
|
+
versionId,
|
|
201
|
+
)?.manifest_json,
|
|
202
|
+
);
|
|
203
|
+
const manifest = manifestJson
|
|
204
|
+
? ArtifactManifest.safeParse(JSON.parse(manifestJson)).data
|
|
205
|
+
: undefined;
|
|
206
|
+
const offline = manifest?.hosted?.offline;
|
|
207
|
+
const version: RelayVersion = {
|
|
208
|
+
functions: options.functions(versionId),
|
|
209
|
+
policy: offline?.actions ?? "queue",
|
|
210
|
+
limitMs: durationMs(offline?.limit ?? HOSTED_OFFLINE_LIMIT_DEFAULT),
|
|
211
|
+
who: manifest?.hosted?.actions ?? "signed-in",
|
|
212
|
+
};
|
|
213
|
+
versions.set(versionId, version);
|
|
214
|
+
return version;
|
|
215
|
+
};
|
|
216
|
+
|
|
217
|
+
const functionOf = (versionId: string, fnId: string): RelayVersion => {
|
|
218
|
+
const version = versionOf(versionId);
|
|
219
|
+
if (!version.functions.has(fnId)) {
|
|
220
|
+
throw failed(
|
|
221
|
+
"MOSTRA_ACTION_UNKNOWN_FUNCTION",
|
|
222
|
+
`This version has no server function ${fnId.slice(0, 200)}.`,
|
|
223
|
+
);
|
|
224
|
+
}
|
|
225
|
+
return version;
|
|
226
|
+
};
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* D159: a relayed action runs on its owner's machine with its owner's secrets, so by default
|
|
230
|
+
* only a signed-in viewer or an expected respondent (a link minted for a named person) may
|
|
231
|
+
* trigger one; `hosted.actions: "link"` opens it to anyone holding a link with the scope.
|
|
232
|
+
* Refused before anything is recorded or relayed.
|
|
233
|
+
*/
|
|
234
|
+
const mayTrigger = (viewerId: string, version: RelayVersion): void => {
|
|
235
|
+
if (version.who === "link") return;
|
|
236
|
+
const viewer = interaction.viewer(viewerId);
|
|
237
|
+
const named =
|
|
238
|
+
viewer.kind === "token" && viewer.ref !== undefined && !viewer.ref.startsWith("link:");
|
|
239
|
+
if (viewer.kind === "account" || named) return;
|
|
240
|
+
throw failed(
|
|
241
|
+
"MOSTRA_REMOTE_SIGN_IN_REQUIRED",
|
|
242
|
+
"Only signed-in viewers can run this page's actions; sign in, or ask its owner for a link of your own.",
|
|
243
|
+
);
|
|
244
|
+
};
|
|
245
|
+
|
|
246
|
+
/** Each link's recent relayed runs and calls, for the per-link limit. */
|
|
247
|
+
const linkUses = new Map<string, number[]>();
|
|
248
|
+
/**
|
|
249
|
+
* Counts a relayed run or call against its link (D159), refusing it past the limit with
|
|
250
|
+
* `MOSTRA_REMOTE_RATE_LIMITED` and when to try again. A request that joins a run already
|
|
251
|
+
* recorded (its client key, or its key) starts nothing, so it is never counted.
|
|
252
|
+
*/
|
|
253
|
+
const limitLink = (
|
|
254
|
+
tokenId: string | undefined,
|
|
255
|
+
artifactId?: string,
|
|
256
|
+
input?: {
|
|
257
|
+
readonly clientKey: string;
|
|
258
|
+
readonly key?: string | undefined;
|
|
259
|
+
readonly fnId: string;
|
|
260
|
+
},
|
|
261
|
+
): void => {
|
|
262
|
+
const limit = options.perLink;
|
|
263
|
+
if (!limit || tokenId === undefined) return;
|
|
264
|
+
if (artifactId !== undefined && input !== undefined) {
|
|
265
|
+
const joins = ledger.read(
|
|
266
|
+
(db) =>
|
|
267
|
+
db.get(
|
|
268
|
+
"SELECT 1 FROM action_run WHERE artifact_id = ? AND (client_key = ? OR (key IS NOT NULL AND key = ? AND action = ?))",
|
|
269
|
+
artifactId,
|
|
270
|
+
input.clientKey,
|
|
271
|
+
input.key ?? null,
|
|
272
|
+
input.fnId,
|
|
273
|
+
) !== undefined,
|
|
274
|
+
);
|
|
275
|
+
if (joins) return;
|
|
276
|
+
}
|
|
277
|
+
const at = now();
|
|
278
|
+
const recent = (linkUses.get(tokenId) ?? []).filter((used) => at - used < limit.perMs);
|
|
279
|
+
if (recent.length >= limit.count) {
|
|
280
|
+
const retryAfter = Math.max(1, Math.ceil(((recent[0] as number) + limit.perMs - at) / 1000));
|
|
281
|
+
const error = failed(
|
|
282
|
+
"MOSTRA_REMOTE_RATE_LIMITED",
|
|
283
|
+
`This link ran ${limit.count} actions recently; try again in ${retryAfter} s.`,
|
|
284
|
+
) as RpcError & { retryAfter?: number };
|
|
285
|
+
error.retryAfter = retryAfter;
|
|
286
|
+
linkUses.set(tokenId, recent);
|
|
287
|
+
throw error;
|
|
288
|
+
}
|
|
289
|
+
recent.push(at);
|
|
290
|
+
linkUses.set(tokenId, recent);
|
|
291
|
+
};
|
|
292
|
+
|
|
293
|
+
const checkArgs = (args: Batches): void => {
|
|
294
|
+
if (JSON.stringify(args).length > ACTION_ARGS_MAX) {
|
|
295
|
+
throw failed(
|
|
296
|
+
"MOSTRA_ACTION_TOO_LARGE",
|
|
297
|
+
"The serialized arguments are larger than 16 MiB; pass less data, such as identifiers.",
|
|
298
|
+
);
|
|
299
|
+
}
|
|
300
|
+
try {
|
|
301
|
+
checkArgumentFrames(args);
|
|
302
|
+
} catch (error) {
|
|
303
|
+
throw failed("MOSTRA_ACTION_UNSERIALIZABLE", (error as Error).message);
|
|
304
|
+
}
|
|
305
|
+
};
|
|
306
|
+
|
|
307
|
+
/** Tells a run's views what changed: the state as a delta from nothing held. */
|
|
308
|
+
const publish = (state: RunState, artifactId: string): void =>
|
|
309
|
+
bus.publishRun(artifactId, {
|
|
310
|
+
kind: "run",
|
|
311
|
+
run: {
|
|
312
|
+
runId: state.runId,
|
|
313
|
+
status: state.status,
|
|
314
|
+
...(state.reason !== undefined ? { reason: state.reason } : {}),
|
|
315
|
+
...(state.progress ? { progress: state.progress } : {}),
|
|
316
|
+
logFrom: state.logFrom ?? 0,
|
|
317
|
+
log: state.log,
|
|
318
|
+
batchFrom: state.batchFrom ?? 0,
|
|
319
|
+
batches: state.batches,
|
|
320
|
+
...(state.error ? { error: state.error } : {}),
|
|
321
|
+
...(state.finishedAt ? { finishedAt: state.finishedAt } : {}),
|
|
322
|
+
},
|
|
323
|
+
});
|
|
324
|
+
|
|
325
|
+
const relayRow = (db: LedgerDb, runId: string): RelayRow | undefined =>
|
|
326
|
+
db.get<RelayRow>("SELECT * FROM relay_run WHERE run_id = ?", runId);
|
|
327
|
+
const runRow = (db: LedgerDb, runId: string): RunRow | undefined =>
|
|
328
|
+
db.get<RunRow>("SELECT * FROM action_run WHERE id = ?", runId);
|
|
329
|
+
|
|
330
|
+
/** Whether the owner of a run's artifact is connected and reconciled. */
|
|
331
|
+
const sessionFor = (linkId: string): number | undefined => {
|
|
332
|
+
const session = options.sessionOf(linkId);
|
|
333
|
+
return session !== undefined && ready.get(linkId) === session ? session : undefined;
|
|
334
|
+
};
|
|
335
|
+
|
|
336
|
+
/** The owner never heard the run: it ends `failed` with reason `owner-offline` (never sent). */
|
|
337
|
+
const endOffline = (runId: string): void => {
|
|
338
|
+
const state = ledger.write((db) => {
|
|
339
|
+
const row = runRow(db, runId);
|
|
340
|
+
if (!row || isFinalRunStatus(row.status) || row.status === "unknown") return undefined;
|
|
341
|
+
const ended = finishRun(db, runId, {
|
|
342
|
+
status: "failed",
|
|
343
|
+
reason: "owner-offline",
|
|
344
|
+
error: {
|
|
345
|
+
code: "MOSTRA_REMOTE_OWNER_OFFLINE",
|
|
346
|
+
message: "The page's owner was offline, so the action never ran.",
|
|
347
|
+
},
|
|
348
|
+
});
|
|
349
|
+
return ended ? { state: runState(db, ended), artifactId: ended.artifact_id } : undefined;
|
|
350
|
+
});
|
|
351
|
+
if (state) publish(state.state, state.artifactId);
|
|
352
|
+
};
|
|
353
|
+
|
|
354
|
+
/** Sends one run to its owner; the answer records that the owner holds it. */
|
|
355
|
+
const send = async (runId: string, session: number): Promise<void> => {
|
|
356
|
+
if (sending.has(runId)) return;
|
|
357
|
+
const prepared = ledger.write((db) => {
|
|
358
|
+
const relay = relayRow(db, runId);
|
|
359
|
+
const row = runRow(db, runId);
|
|
360
|
+
if (!relay || !row || isFinalRunStatus(row.status) || row.status === "unknown")
|
|
361
|
+
return undefined;
|
|
362
|
+
// A run never sent whose offline limit passed never happens, however the owner came back
|
|
363
|
+
// (a home closed while it waited ran no sweep): it ends `failed`, `owner-offline` (Opus
|
|
364
|
+
// review of WP-29, 7).
|
|
365
|
+
if (relay.sent_at === null && Date.parse(relay.offline_limit_at) <= now()) return "expired";
|
|
366
|
+
db.run(
|
|
367
|
+
"UPDATE relay_run SET attempt = attempt + 1, sent_at = COALESCE(sent_at, ?) WHERE run_id = ?",
|
|
368
|
+
iso(now()),
|
|
369
|
+
runId,
|
|
370
|
+
);
|
|
371
|
+
return { relay, row };
|
|
372
|
+
});
|
|
373
|
+
if (!prepared) return;
|
|
374
|
+
if (prepared === "expired") {
|
|
375
|
+
endOffline(runId);
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
const { relay, row } = prepared;
|
|
379
|
+
sending.add(runId);
|
|
380
|
+
try {
|
|
381
|
+
const viewer = row.viewer_id ? interaction.viewer(row.viewer_id) : undefined;
|
|
382
|
+
await options.call(
|
|
383
|
+
session,
|
|
384
|
+
portRelayRunMethod.method,
|
|
385
|
+
{
|
|
386
|
+
runId,
|
|
387
|
+
seq: relay.relay_seq,
|
|
388
|
+
artifactId: row.artifact_id,
|
|
389
|
+
versionId: row.version_id,
|
|
390
|
+
fnId: row.action,
|
|
391
|
+
args: JSON.parse(row.args_json) as Batches,
|
|
392
|
+
...(row.key !== null ? { key: row.key } : {}),
|
|
393
|
+
...(viewer
|
|
394
|
+
? { viewer: { id: viewer.id, ...(viewer.name ? { name: viewer.name } : {}) } }
|
|
395
|
+
: {}),
|
|
396
|
+
...(row.view_id !== null ? { viewId: row.view_id } : {}),
|
|
397
|
+
},
|
|
398
|
+
PORT_CALL_MS,
|
|
399
|
+
);
|
|
400
|
+
const at = iso(now());
|
|
401
|
+
const cancelled = ledger.write((db) => {
|
|
402
|
+
db.run(
|
|
403
|
+
"UPDATE relay_run SET accepted_at = COALESCE(accepted_at, ?), owner_heard_at = ? WHERE run_id = ?",
|
|
404
|
+
at,
|
|
405
|
+
at,
|
|
406
|
+
runId,
|
|
407
|
+
);
|
|
408
|
+
return relayRow(db, runId)?.cancel_requested_at ?? null;
|
|
409
|
+
});
|
|
410
|
+
// A cancel pressed before the owner had it, or while its acknowledgement was on its way,
|
|
411
|
+
// reaches the run now (Opus review of WP-29).
|
|
412
|
+
if (cancelled !== null) void cancelOnOwner(runId, session);
|
|
413
|
+
} catch (error) {
|
|
414
|
+
const code = error instanceof RpcError ? error.diagnostic.code : undefined;
|
|
415
|
+
if (code !== undefined && !TRANSPORT_FAILURES.has(code)) {
|
|
416
|
+
// The owner answered and refused the run: it records a run before it answers, so a
|
|
417
|
+
// refusal means the run never started there, and it ends failed with the owner's reason.
|
|
418
|
+
refused(runId, (error as RpcError).diagnostic);
|
|
419
|
+
} else {
|
|
420
|
+
// Sent and not acknowledged: it is sent again (after the next reconnect, or by the sweep
|
|
421
|
+
// while the owner stays connected), which joins it if the owner recorded it (§11.1).
|
|
422
|
+
log.warn("relay.send_failed", { runId, error });
|
|
423
|
+
}
|
|
424
|
+
} finally {
|
|
425
|
+
lastTry.set(runId, now());
|
|
426
|
+
sending.delete(runId);
|
|
427
|
+
}
|
|
428
|
+
};
|
|
429
|
+
|
|
430
|
+
/** A run its owner refused before recording it ends failed, with the owner's diagnostic. */
|
|
431
|
+
const refused = (runId: string, diagnostic: { code: string; message: string }): void => {
|
|
432
|
+
log.warn("relay.run_refused", { runId, code: diagnostic.code });
|
|
433
|
+
const ended = ledger.write((db) => {
|
|
434
|
+
const row = runRow(db, runId);
|
|
435
|
+
if (!row || isFinalRunStatus(row.status) || row.status === "unknown") return undefined;
|
|
436
|
+
const done = finishRun(db, runId, {
|
|
437
|
+
status: "failed",
|
|
438
|
+
reason: "owner-refused",
|
|
439
|
+
error: { code: diagnostic.code, message: diagnostic.message },
|
|
440
|
+
});
|
|
441
|
+
return done ? { state: runState(db, done), artifactId: done.artifact_id } : undefined;
|
|
442
|
+
});
|
|
443
|
+
if (ended) publish(ended.state, ended.artifactId);
|
|
444
|
+
};
|
|
445
|
+
|
|
446
|
+
const cancelOnOwner = async (runId: string, session: number): Promise<void> => {
|
|
447
|
+
try {
|
|
448
|
+
await options.call(session, portRelayCancelMethod.method, { runId }, PORT_CALL_MS);
|
|
449
|
+
ledger.write((db) =>
|
|
450
|
+
db.run("UPDATE relay_run SET cancel_requested_at = NULL WHERE run_id = ?", runId),
|
|
451
|
+
);
|
|
452
|
+
} catch (error) {
|
|
453
|
+
log.warn("relay.cancel_failed", { runId, error });
|
|
454
|
+
}
|
|
455
|
+
};
|
|
456
|
+
|
|
457
|
+
/**
|
|
458
|
+
* Applies a run's state as the owner reports it, from what the hosted home holds on: new log
|
|
459
|
+
* lines and batches by position, `running`, and the end (which replaces `unknown` once).
|
|
460
|
+
*/
|
|
461
|
+
const apply = (
|
|
462
|
+
runId: string,
|
|
463
|
+
report: {
|
|
464
|
+
readonly running?: boolean;
|
|
465
|
+
readonly progress?: RunState["progress"];
|
|
466
|
+
readonly log?: readonly string[];
|
|
467
|
+
readonly logFrom?: number;
|
|
468
|
+
readonly batches?: Batches;
|
|
469
|
+
readonly batchFrom?: number;
|
|
470
|
+
readonly end?: {
|
|
471
|
+
readonly status: "succeeded" | "failed" | "cancelled";
|
|
472
|
+
readonly reason?: string;
|
|
473
|
+
readonly error?: { readonly message: string; readonly code?: string };
|
|
474
|
+
};
|
|
475
|
+
},
|
|
476
|
+
): void => {
|
|
477
|
+
const state = ledger.write((db) => {
|
|
478
|
+
const row = runRow(db, runId);
|
|
479
|
+
if (!row || isFinalRunStatus(row.status)) return undefined;
|
|
480
|
+
db.run("UPDATE relay_run SET owner_heard_at = ? WHERE run_id = ?", iso(now()), runId);
|
|
481
|
+
// An update from past what this home holds means one before it was lost: nothing of it is
|
|
482
|
+
// kept, its end least of all, and the owner is asked for the run from what is held, so a
|
|
483
|
+
// run never ends without its result (Opus review of WP-29, 4).
|
|
484
|
+
if (
|
|
485
|
+
(report.logFrom !== undefined && report.logFrom > row.log_lines) ||
|
|
486
|
+
(report.batchFrom !== undefined && report.batchFrom > row.batches)
|
|
487
|
+
) {
|
|
488
|
+
return "gap" as const;
|
|
489
|
+
}
|
|
490
|
+
if (
|
|
491
|
+
(report.running || report.log || report.batches || report.end) &&
|
|
492
|
+
row.status !== "unknown"
|
|
493
|
+
) {
|
|
494
|
+
markRunning(db, runId);
|
|
495
|
+
}
|
|
496
|
+
// What the hosted home holds already is skipped, so a repeat appends nothing.
|
|
497
|
+
const lines = report.log ?? [];
|
|
498
|
+
const linesFrom = report.logFrom ?? row.log_lines;
|
|
499
|
+
const freshLines = linesFrom <= row.log_lines ? lines.slice(row.log_lines - linesFrom) : [];
|
|
500
|
+
const batches = report.batches ?? [];
|
|
501
|
+
const batchesFrom = report.batchFrom ?? row.batches;
|
|
502
|
+
const freshBatches =
|
|
503
|
+
batchesFrom <= row.batches ? batches.slice(row.batches - batchesFrom) : [];
|
|
504
|
+
if (freshLines.length > 0 || freshBatches.length > 0 || report.progress) {
|
|
505
|
+
appendRunOutput(db, runId, {
|
|
506
|
+
lines: freshLines,
|
|
507
|
+
batches: freshBatches,
|
|
508
|
+
...(report.progress ? { progress: report.progress } : {}),
|
|
509
|
+
});
|
|
510
|
+
}
|
|
511
|
+
if (report.end) {
|
|
512
|
+
finishRun(db, runId, {
|
|
513
|
+
status: report.end.status,
|
|
514
|
+
...(report.end.reason !== undefined ? { reason: report.end.reason } : {}),
|
|
515
|
+
...(report.end.error ? { error: report.end.error } : {}),
|
|
516
|
+
});
|
|
517
|
+
}
|
|
518
|
+
const after = runRow(db, runId);
|
|
519
|
+
return after ? { state: runState(db, after), artifactId: after.artifact_id } : undefined;
|
|
520
|
+
});
|
|
521
|
+
if (state === "gap") {
|
|
522
|
+
log.warn("relay.update_gap", { runId });
|
|
523
|
+
void catchUp(runId);
|
|
524
|
+
return;
|
|
525
|
+
}
|
|
526
|
+
if (state) publish(state.state, state.artifactId);
|
|
527
|
+
};
|
|
528
|
+
|
|
529
|
+
/** Runs being asked for again after a gap, so one gap asks once. */
|
|
530
|
+
const catchingUp = new Set<string>();
|
|
531
|
+
/** Asks the owner for one run's state from what this home holds (a reconcile of that run). */
|
|
532
|
+
const catchUp = async (runId: string): Promise<void> => {
|
|
533
|
+
if (catchingUp.has(runId)) return;
|
|
534
|
+
catchingUp.add(runId);
|
|
535
|
+
try {
|
|
536
|
+
const entry = ledger.read((db) => {
|
|
537
|
+
const relay = relayRow(db, runId);
|
|
538
|
+
const run = runRow(db, runId);
|
|
539
|
+
return relay && run ? { relay, run } : undefined;
|
|
540
|
+
});
|
|
541
|
+
if (!entry) return;
|
|
542
|
+
const session = sessionFor(entry.relay.owner_link_id);
|
|
543
|
+
if (session === undefined) return;
|
|
544
|
+
const answer = (await options.call(
|
|
545
|
+
session,
|
|
546
|
+
portRelayReconcileMethod.method,
|
|
547
|
+
{
|
|
548
|
+
runs: [runId],
|
|
549
|
+
held: [{ runId, log: entry.run.log_lines, batches: entry.run.batches }],
|
|
550
|
+
},
|
|
551
|
+
PORT_CALL_MS,
|
|
552
|
+
)) as PortRelayReconcileResult;
|
|
553
|
+
for (const reconciled of answer.runs) applyReconciled(reconciled);
|
|
554
|
+
} catch (error) {
|
|
555
|
+
// The next reconnect's reconcile asks again.
|
|
556
|
+
log.warn("relay.catch_up_failed", { runId, error });
|
|
557
|
+
} finally {
|
|
558
|
+
catchingUp.delete(runId);
|
|
559
|
+
}
|
|
560
|
+
};
|
|
561
|
+
|
|
562
|
+
/** Applies the owner's answer for one run of a reconcile. */
|
|
563
|
+
const applyReconciled = (entry: PortRelayReconcileResult["runs"][number]): void => {
|
|
564
|
+
if (entry.state === "unknown-to-owner") {
|
|
565
|
+
unknown(entry.runId);
|
|
566
|
+
return;
|
|
567
|
+
}
|
|
568
|
+
const state = entry.state;
|
|
569
|
+
apply(entry.runId, {
|
|
570
|
+
running: state.status !== "queued",
|
|
571
|
+
...(state.progress ? { progress: state.progress } : {}),
|
|
572
|
+
log: state.log,
|
|
573
|
+
logFrom: state.logFrom ?? 0,
|
|
574
|
+
batches: state.batches,
|
|
575
|
+
batchFrom: state.batchFrom ?? 0,
|
|
576
|
+
...(state.status === "succeeded" || state.status === "failed" || state.status === "cancelled"
|
|
577
|
+
? {
|
|
578
|
+
end: {
|
|
579
|
+
status: state.status,
|
|
580
|
+
...(state.reason !== undefined ? { reason: state.reason } : {}),
|
|
581
|
+
...(state.error ? { error: state.error } : {}),
|
|
582
|
+
},
|
|
583
|
+
}
|
|
584
|
+
: {}),
|
|
585
|
+
});
|
|
586
|
+
};
|
|
587
|
+
|
|
588
|
+
/** Moves a run to `unknown`, and tells its views. */
|
|
589
|
+
const unknown = (runId: string): void => {
|
|
590
|
+
const state = ledger.write((db) => {
|
|
591
|
+
if (!markUnknown(db, runId)) return undefined;
|
|
592
|
+
const row = runRow(db, runId);
|
|
593
|
+
return row ? { state: runState(db, row), artifactId: row.artifact_id } : undefined;
|
|
594
|
+
});
|
|
595
|
+
if (state) publish(state.state, state.artifactId);
|
|
596
|
+
};
|
|
597
|
+
|
|
598
|
+
/**
|
|
599
|
+
* The runs of a link the owner accepted and the hosted home holds no end for, and every run sent
|
|
600
|
+
* that became `unknown` (its acknowledgement may have been lost): the owner's report replaces
|
|
601
|
+
* `unknown` once, and asking starts nothing (Opus review of WP-29, 8).
|
|
602
|
+
*/
|
|
603
|
+
const acceptedOpen = (linkId: string): { relay: RelayRow; run: RunRow }[] =>
|
|
604
|
+
ledger.read((db) =>
|
|
605
|
+
db
|
|
606
|
+
.all<RelayRow & { status: string }>(
|
|
607
|
+
`SELECT r.*, a.status FROM relay_run r JOIN action_run a ON a.id = r.run_id
|
|
608
|
+
WHERE r.owner_link_id = ?
|
|
609
|
+
AND ((r.accepted_at IS NOT NULL AND a.status IN ('queued', 'running', 'waiting'))
|
|
610
|
+
OR (r.sent_at IS NOT NULL AND a.status = 'unknown'))
|
|
611
|
+
ORDER BY r.relay_seq`,
|
|
612
|
+
linkId,
|
|
613
|
+
)
|
|
614
|
+
.map((relay) => ({ relay, run: runRow(db, relay.run_id) as RunRow })),
|
|
615
|
+
);
|
|
616
|
+
|
|
617
|
+
/** After a reconnect: reconcile, then resend, then drain the waiting runs in order (§18.3). */
|
|
618
|
+
const resume = async (linkId: string, session: number, relayMark: number): Promise<void> => {
|
|
619
|
+
try {
|
|
620
|
+
// The owner's home was restored to before runs it accepted: they, and every run sent but
|
|
621
|
+
// never acknowledged, become `unknown` and are never sent again.
|
|
622
|
+
const high = ledger.read(
|
|
623
|
+
(db) =>
|
|
624
|
+
db.get<{ high: number | null }>(
|
|
625
|
+
"SELECT MAX(relay_seq) AS high FROM relay_run WHERE owner_link_id = ? AND accepted_at IS NOT NULL",
|
|
626
|
+
linkId,
|
|
627
|
+
)?.high ?? 0,
|
|
628
|
+
);
|
|
629
|
+
if (relayMark < high) {
|
|
630
|
+
const lost = ledger.read((db) =>
|
|
631
|
+
db
|
|
632
|
+
.all<{ run_id: string }>(
|
|
633
|
+
`SELECT r.run_id FROM relay_run r JOIN action_run a ON a.id = r.run_id
|
|
634
|
+
WHERE r.owner_link_id = ? AND r.sent_at IS NOT NULL
|
|
635
|
+
AND a.status IN ('queued', 'running', 'waiting')`,
|
|
636
|
+
linkId,
|
|
637
|
+
)
|
|
638
|
+
.map((row) => row.run_id),
|
|
639
|
+
);
|
|
640
|
+
log.warn("relay.owner_regressed", { linkId, relayMark, high, runs: lost.length });
|
|
641
|
+
for (const runId of lost) unknown(runId);
|
|
642
|
+
} else {
|
|
643
|
+
const open = acceptedOpen(linkId);
|
|
644
|
+
if (open.length > 0) {
|
|
645
|
+
const answer = (await options.call(
|
|
646
|
+
session,
|
|
647
|
+
portRelayReconcileMethod.method,
|
|
648
|
+
{
|
|
649
|
+
runs: open.map((entry) => entry.run.id),
|
|
650
|
+
held: open.map((entry) => ({
|
|
651
|
+
runId: entry.run.id,
|
|
652
|
+
log: entry.run.log_lines,
|
|
653
|
+
batches: entry.run.batches,
|
|
654
|
+
})),
|
|
655
|
+
},
|
|
656
|
+
PORT_CALL_MS,
|
|
657
|
+
)) as PortRelayReconcileResult;
|
|
658
|
+
for (const entry of answer.runs) applyReconciled(entry);
|
|
659
|
+
// A cancel made while the owner was away reaches it now.
|
|
660
|
+
for (const entry of open) {
|
|
661
|
+
if (entry.relay.cancel_requested_at !== null) void cancelOnOwner(entry.run.id, session);
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
if (options.sessionOf(linkId) !== session) return;
|
|
666
|
+
ready.set(linkId, session);
|
|
667
|
+
// The sources its viewers want start again on the owner that came back.
|
|
668
|
+
asked.clear();
|
|
669
|
+
renewLive();
|
|
670
|
+
// Sent and never acknowledged: sent again, which joins the owner's record if it has one.
|
|
671
|
+
const unacknowledged = ledger.read((db) =>
|
|
672
|
+
db
|
|
673
|
+
.all<{ run_id: string }>(
|
|
674
|
+
`SELECT r.run_id FROM relay_run r JOIN action_run a ON a.id = r.run_id
|
|
675
|
+
WHERE r.owner_link_id = ? AND r.sent_at IS NOT NULL AND r.accepted_at IS NULL
|
|
676
|
+
AND a.status IN ('queued', 'running', 'waiting') ORDER BY r.relay_seq`,
|
|
677
|
+
linkId,
|
|
678
|
+
)
|
|
679
|
+
.map((row) => row.run_id),
|
|
680
|
+
);
|
|
681
|
+
for (const runId of unacknowledged) await send(runId, session);
|
|
682
|
+
retries.delete(session);
|
|
683
|
+
void drain(linkId);
|
|
684
|
+
} catch (error) {
|
|
685
|
+
log.warn("relay.resume_failed", { linkId, error });
|
|
686
|
+
// A reconcile that timed out or failed leaves the owner unready: tried again for the same
|
|
687
|
+
// session, 1 s doubling to 60 s, while it stays its link's session.
|
|
688
|
+
if (stopped || options.sessionOf(linkId) !== session) return;
|
|
689
|
+
const attempt = (retries.get(session)?.attempt ?? 0) + 1;
|
|
690
|
+
const timer = setTimeout(
|
|
691
|
+
() => {
|
|
692
|
+
if (options.sessionOf(linkId) === session && ready.get(linkId) !== session) {
|
|
693
|
+
void resume(linkId, session, relayMark);
|
|
694
|
+
}
|
|
695
|
+
},
|
|
696
|
+
Math.min(60_000, 1000 * 2 ** (attempt - 1)),
|
|
697
|
+
);
|
|
698
|
+
timer.unref();
|
|
699
|
+
retries.set(session, { timer, attempt });
|
|
700
|
+
}
|
|
701
|
+
};
|
|
702
|
+
|
|
703
|
+
/** Sends a link's waiting runs, oldest first, at most `drainPerSecond` a second per artifact. */
|
|
704
|
+
const drain = async (linkId: string): Promise<void> => {
|
|
705
|
+
if (draining.has(linkId)) return;
|
|
706
|
+
draining.add(linkId);
|
|
707
|
+
try {
|
|
708
|
+
const sentAt = new Map<string, number[]>();
|
|
709
|
+
for (;;) {
|
|
710
|
+
if (stopped) return;
|
|
711
|
+
const session = sessionFor(linkId);
|
|
712
|
+
if (session === undefined) return;
|
|
713
|
+
const next = ledger.read((db) =>
|
|
714
|
+
db.get<{ run_id: string; artifact_id: string }>(
|
|
715
|
+
`SELECT r.run_id, a.artifact_id FROM relay_run r JOIN action_run a ON a.id = r.run_id
|
|
716
|
+
WHERE r.owner_link_id = ? AND r.sent_at IS NULL
|
|
717
|
+
AND a.status IN ('queued', 'waiting') ORDER BY r.relay_seq LIMIT 1`,
|
|
718
|
+
linkId,
|
|
719
|
+
),
|
|
720
|
+
);
|
|
721
|
+
if (!next) return;
|
|
722
|
+
const times = (sentAt.get(next.artifact_id) ?? []).filter((at) => now() - at < 1000);
|
|
723
|
+
if (times.length >= drainPerSecond) {
|
|
724
|
+
await new Promise((resolve) =>
|
|
725
|
+
setTimeout(resolve, 1000 - (now() - (times[0] as number))),
|
|
726
|
+
);
|
|
727
|
+
continue;
|
|
728
|
+
}
|
|
729
|
+
times.push(now());
|
|
730
|
+
sentAt.set(next.artifact_id, times);
|
|
731
|
+
await send(next.run_id, session);
|
|
732
|
+
}
|
|
733
|
+
} finally {
|
|
734
|
+
draining.delete(linkId);
|
|
735
|
+
}
|
|
736
|
+
};
|
|
737
|
+
|
|
738
|
+
/** Checks waiting and unacknowledged runs against their offline limit's two clocks. */
|
|
739
|
+
const sweep = (): void => {
|
|
740
|
+
const rows = ledger.read((db) =>
|
|
741
|
+
db.all<RelayRow & { status: string; version_id: string; started_at: string }>(
|
|
742
|
+
`SELECT r.*, a.status, a.version_id, a.started_at FROM relay_run r
|
|
743
|
+
JOIN action_run a ON a.id = r.run_id WHERE a.status IN ('queued', 'running', 'waiting')`,
|
|
744
|
+
),
|
|
745
|
+
);
|
|
746
|
+
const open = new Set(rows.map((row) => row.run_id));
|
|
747
|
+
for (const id of lastTry.keys()) if (!open.has(id)) lastTry.delete(id);
|
|
748
|
+
for (const row of rows) {
|
|
749
|
+
if (sending.has(row.run_id)) continue;
|
|
750
|
+
if (row.sent_at === null) {
|
|
751
|
+
// Never sent: counted from when it was recorded, and it never happened.
|
|
752
|
+
if (Date.parse(row.offline_limit_at) <= now()) endOffline(row.run_id);
|
|
753
|
+
continue;
|
|
754
|
+
}
|
|
755
|
+
// Sent: counted from when the owner was last heard from, and only while it is away. While
|
|
756
|
+
// it stays connected, a run it never acknowledged (its answer timed out) is sent again.
|
|
757
|
+
const session = sessionFor(row.owner_link_id);
|
|
758
|
+
if (session !== undefined) {
|
|
759
|
+
if (row.accepted_at === null && now() - (lastTry.get(row.run_id) ?? 0) >= RESEND_MS) {
|
|
760
|
+
void send(row.run_id, session);
|
|
761
|
+
}
|
|
762
|
+
continue;
|
|
763
|
+
}
|
|
764
|
+
const heard = Date.parse(row.owner_heard_at ?? row.sent_at);
|
|
765
|
+
if (heard + versionOf(row.version_id).limitMs <= now()) unknown(row.run_id);
|
|
766
|
+
}
|
|
767
|
+
};
|
|
768
|
+
const sweeper = setInterval(sweep, options.sweepMs ?? 1000);
|
|
769
|
+
sweeper.unref();
|
|
770
|
+
|
|
771
|
+
/** What each artifact's owner was last asked to produce, so a source no one wants is stopped. */
|
|
772
|
+
const asked = new Map<string, ReadonlySet<string>>();
|
|
773
|
+
/** Asks owners for the live sources viewers want now, and stops the ones no one does (§18.3). */
|
|
774
|
+
const renewLive = (only?: string): void => {
|
|
775
|
+
const wanted = new Map<string, Set<string>>();
|
|
776
|
+
for (const entry of bus.liveInterest()) {
|
|
777
|
+
if (entry.key.startsWith(TALLY_LIVE_PREFIX)) continue;
|
|
778
|
+
const keys = wanted.get(entry.artifactId) ?? new Set<string>();
|
|
779
|
+
keys.add(entry.key);
|
|
780
|
+
wanted.set(entry.artifactId, keys);
|
|
781
|
+
}
|
|
782
|
+
const artifacts = new Set([...wanted.keys(), ...asked.keys()]);
|
|
783
|
+
for (const artifactId of artifacts) {
|
|
784
|
+
if (only !== undefined && artifactId !== only) continue;
|
|
785
|
+
const owner = options.ownerOf(artifactId);
|
|
786
|
+
const session = owner === undefined ? undefined : sessionFor(owner);
|
|
787
|
+
const versionId = options.latestVersion?.(artifactId);
|
|
788
|
+
if (session === undefined || versionId === undefined) continue;
|
|
789
|
+
const keys = wanted.get(artifactId) ?? new Set<string>();
|
|
790
|
+
const before = asked.get(artifactId) ?? new Set<string>();
|
|
791
|
+
const stopped = [...before].filter((key) => !keys.has(key));
|
|
792
|
+
const send = (interest: "start" | "renew" | "stop", list: readonly string[]): void => {
|
|
793
|
+
if (list.length === 0) return;
|
|
794
|
+
void options
|
|
795
|
+
.call(
|
|
796
|
+
session,
|
|
797
|
+
portRelayLiveMethod.method,
|
|
798
|
+
{ artifactId, versionId, interest, keys: list },
|
|
799
|
+
PORT_CALL_MS,
|
|
800
|
+
)
|
|
801
|
+
.catch((error: unknown) => log.warn("relay.live_interest_failed", { artifactId, error }));
|
|
802
|
+
};
|
|
803
|
+
send("renew", [...keys]);
|
|
804
|
+
send("stop", stopped);
|
|
805
|
+
if (keys.size > 0) asked.set(artifactId, keys);
|
|
806
|
+
else asked.delete(artifactId);
|
|
807
|
+
}
|
|
808
|
+
};
|
|
809
|
+
const liveRenewer = setInterval(() => renewLive(), options.liveRenewMs ?? 20_000);
|
|
810
|
+
liveRenewer.unref();
|
|
811
|
+
|
|
812
|
+
const decorate = (state: RunState, artifactId: string): RunState => {
|
|
813
|
+
if (state.status !== "running") return state;
|
|
814
|
+
const owner = options.ownerOf(artifactId);
|
|
815
|
+
return owner !== undefined && sessionFor(owner) !== undefined
|
|
816
|
+
? state
|
|
817
|
+
: { ...state, ownerUnreachable: true };
|
|
818
|
+
};
|
|
819
|
+
|
|
820
|
+
return {
|
|
821
|
+
async call(page, input, signal) {
|
|
822
|
+
checkArgs(input.args);
|
|
823
|
+
const versionId = interaction.viewVersion(page, input.viewId);
|
|
824
|
+
mayTrigger(page.viewerId, functionOf(versionId, input.fnId));
|
|
825
|
+
limitLink(page.tokenId);
|
|
826
|
+
const owner = options.ownerOf(page.artifactId);
|
|
827
|
+
const session = owner === undefined ? undefined : sessionFor(owner);
|
|
828
|
+
if (session === undefined) {
|
|
829
|
+
throw failed(
|
|
830
|
+
"MOSTRA_REMOTE_OWNER_OFFLINE",
|
|
831
|
+
"The page's owner is offline, so its server code can't answer now; try again later.",
|
|
832
|
+
);
|
|
833
|
+
}
|
|
834
|
+
const callId = `call_${crypto.randomUUID()}`;
|
|
835
|
+
const pending: PendingCall = { session, lines: [], ended: false, lastSeq: 0 };
|
|
836
|
+
calls.set(callId, pending);
|
|
837
|
+
const viewer = interaction.viewer(page.viewerId);
|
|
838
|
+
try {
|
|
839
|
+
await options.call(
|
|
840
|
+
session,
|
|
841
|
+
portRelayCallMethod.method,
|
|
842
|
+
{
|
|
843
|
+
callId,
|
|
844
|
+
artifactId: page.artifactId,
|
|
845
|
+
versionId,
|
|
846
|
+
fnId: input.fnId,
|
|
847
|
+
args: input.args,
|
|
848
|
+
viewer: { id: viewer.id, ...(viewer.name ? { name: viewer.name } : {}) },
|
|
849
|
+
viewId: input.viewId,
|
|
850
|
+
},
|
|
851
|
+
PORT_CALL_MS,
|
|
852
|
+
);
|
|
853
|
+
} catch (error) {
|
|
854
|
+
calls.delete(callId);
|
|
855
|
+
throw error instanceof RpcError
|
|
856
|
+
? failed(
|
|
857
|
+
"MOSTRA_REMOTE_OWNER_OFFLINE",
|
|
858
|
+
`The page's owner couldn't take the call: ${error.diagnostic.message}`,
|
|
859
|
+
)
|
|
860
|
+
: error;
|
|
861
|
+
}
|
|
862
|
+
const lines = async function* (): AsyncGenerator<CallLine> {
|
|
863
|
+
try {
|
|
864
|
+
for (;;) {
|
|
865
|
+
const line = pending.lines.shift();
|
|
866
|
+
if (line) {
|
|
867
|
+
yield line;
|
|
868
|
+
if ("end" in line) return;
|
|
869
|
+
continue;
|
|
870
|
+
}
|
|
871
|
+
if (pending.ended || signal.aborted) return;
|
|
872
|
+
await new Promise<void>((resolve) => {
|
|
873
|
+
pending.wake = resolve;
|
|
874
|
+
});
|
|
875
|
+
}
|
|
876
|
+
} finally {
|
|
877
|
+
calls.delete(callId);
|
|
878
|
+
}
|
|
879
|
+
};
|
|
880
|
+
return lines();
|
|
881
|
+
},
|
|
882
|
+
|
|
883
|
+
startRun(page, input) {
|
|
884
|
+
checkArgs(input.args);
|
|
885
|
+
const versionId = interaction.viewVersion(page, input.viewId);
|
|
886
|
+
const version = functionOf(versionId, input.fnId);
|
|
887
|
+
mayTrigger(page.viewerId, version);
|
|
888
|
+
limitLink(page.tokenId, page.artifactId, input);
|
|
889
|
+
const owner = options.ownerOf(page.artifactId);
|
|
890
|
+
if (owner === undefined) {
|
|
891
|
+
throw failed("MOSTRA_REMOTE_OWNER_OFFLINE", "This page has no owner instance to run it.");
|
|
892
|
+
}
|
|
893
|
+
const connected = sessionFor(owner) !== undefined;
|
|
894
|
+
// Past the cap a run is refused before anything is recorded (D165 item 4). A request that
|
|
895
|
+
// joins a run already recorded is never refused.
|
|
896
|
+
if (!connected && version.policy === "queue") {
|
|
897
|
+
const waiting = ledger.read(
|
|
898
|
+
(db) =>
|
|
899
|
+
db.get<{ n: number }>(
|
|
900
|
+
"SELECT COUNT(*) AS n FROM action_run WHERE artifact_id = ? AND status = 'waiting'",
|
|
901
|
+
page.artifactId,
|
|
902
|
+
)?.n ?? 0,
|
|
903
|
+
);
|
|
904
|
+
const joins = ledger.read(
|
|
905
|
+
(db) =>
|
|
906
|
+
db.get<{ id: string }>(
|
|
907
|
+
`SELECT id FROM action_run WHERE artifact_id = ? AND (client_key = ? OR (key IS NOT NULL AND key = ? AND action = ?))`,
|
|
908
|
+
page.artifactId,
|
|
909
|
+
input.clientKey,
|
|
910
|
+
input.key ?? null,
|
|
911
|
+
input.fnId,
|
|
912
|
+
) !== undefined,
|
|
913
|
+
);
|
|
914
|
+
if (waiting >= cap && !joins) {
|
|
915
|
+
throw failed(
|
|
916
|
+
"MOSTRA_REMOTE_QUEUE_FULL",
|
|
917
|
+
"Too many actions are waiting for this page's owner; try again once it is back.",
|
|
918
|
+
);
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
const { state, row, started } = interaction.startRun(page, input);
|
|
922
|
+
bus.attachRuns(page.artifactId, input.viewId, [state.runId]);
|
|
923
|
+
if (!started) return decorate(state, page.artifactId);
|
|
924
|
+
// The relay's record, in the order runs were made: its sequence is unique per owner link and
|
|
925
|
+
// continues above the owner's mark after a restore (§18.2).
|
|
926
|
+
const recorded = Date.parse(row.started_at);
|
|
927
|
+
const waitingState = ledger.write((db) => {
|
|
928
|
+
const seq =
|
|
929
|
+
(db.get<{ high: number | null }>(
|
|
930
|
+
"SELECT MAX(relay_seq) AS high FROM relay_run WHERE owner_link_id = ?",
|
|
931
|
+
owner,
|
|
932
|
+
)?.high ?? 0) + 1;
|
|
933
|
+
const floor = Math.max(
|
|
934
|
+
marks.get(owner) ?? 0,
|
|
935
|
+
db.get<{ floor: number }>("SELECT floor FROM relay_floor WHERE link_id = ?", owner)
|
|
936
|
+
?.floor ?? 0,
|
|
937
|
+
);
|
|
938
|
+
db.run(
|
|
939
|
+
"INSERT INTO relay_run (run_id, owner_link_id, relay_seq, offline_limit_at) VALUES (?, ?, ?, ?)",
|
|
940
|
+
row.id,
|
|
941
|
+
owner,
|
|
942
|
+
Math.max(seq, floor + 1),
|
|
943
|
+
iso(recorded + version.limitMs),
|
|
944
|
+
);
|
|
945
|
+
if (connected) return undefined;
|
|
946
|
+
markWaiting(db, row.id);
|
|
947
|
+
const after = runRow(db, row.id);
|
|
948
|
+
return after ? runState(db, after) : undefined;
|
|
949
|
+
});
|
|
950
|
+
if (connected) {
|
|
951
|
+
const session = sessionFor(owner);
|
|
952
|
+
if (session !== undefined) void send(row.id, session);
|
|
953
|
+
return state;
|
|
954
|
+
}
|
|
955
|
+
if (version.policy === "fail") {
|
|
956
|
+
endOffline(row.id);
|
|
957
|
+
return interaction.runState(row.id) ?? state;
|
|
958
|
+
}
|
|
959
|
+
if (waitingState) publish(waitingState, page.artifactId);
|
|
960
|
+
return waitingState ?? state;
|
|
961
|
+
},
|
|
962
|
+
|
|
963
|
+
attach(page, input) {
|
|
964
|
+
const states = interaction.runsFor(page, input);
|
|
965
|
+
bus.attachRuns(
|
|
966
|
+
page.artifactId,
|
|
967
|
+
input.viewId,
|
|
968
|
+
states.map((state) => state.runId),
|
|
969
|
+
);
|
|
970
|
+
return states.map((state) => decorate(state, page.artifactId));
|
|
971
|
+
},
|
|
972
|
+
|
|
973
|
+
cancel(page, viewId, runId) {
|
|
974
|
+
const row = interaction.runFor(page, viewId, runId);
|
|
975
|
+
const relay = ledger.read((db) => relayRow(db, row.id));
|
|
976
|
+
if (!relay || relay.sent_at === null) {
|
|
977
|
+
// Never sent: it ends here without contacting the owner.
|
|
978
|
+
if (!isFinalRunStatus(row.status) && row.status !== "unknown") {
|
|
979
|
+
const ended = interaction.endRun(row.id, { status: "cancelled", reason: "cancelled" });
|
|
980
|
+
if (ended) publish(ended, row.artifact_id);
|
|
981
|
+
}
|
|
982
|
+
return interaction.runState(row.id) as RunState;
|
|
983
|
+
}
|
|
984
|
+
ledger.write((db) =>
|
|
985
|
+
db.run("UPDATE relay_run SET cancel_requested_at = ? WHERE run_id = ?", iso(now()), row.id),
|
|
986
|
+
);
|
|
987
|
+
const session = sessionFor(relay.owner_link_id);
|
|
988
|
+
if (session !== undefined && relay.accepted_at !== null) void cancelOnOwner(row.id, session);
|
|
989
|
+
return decorate(interaction.runState(row.id) as RunState, row.artifact_id);
|
|
990
|
+
},
|
|
991
|
+
|
|
992
|
+
methods() {
|
|
993
|
+
return [
|
|
994
|
+
handler(relayRunUpdateMethod, (params, session) => {
|
|
995
|
+
const known = ledger.read((db) => relayRow(db, params.runId));
|
|
996
|
+
if (!known || known.owner_link_id !== session.link?.instanceId) {
|
|
997
|
+
// A home restored to before this run: the owner keeps and finishes it locally.
|
|
998
|
+
log.warn("relay.update_dropped", { runId: params.runId });
|
|
999
|
+
return {};
|
|
1000
|
+
}
|
|
1001
|
+
if (params.seq <= known.last_seq) return {};
|
|
1002
|
+
ledger.write((db) =>
|
|
1003
|
+
db.run("UPDATE relay_run SET last_seq = ? WHERE run_id = ?", params.seq, params.runId),
|
|
1004
|
+
);
|
|
1005
|
+
apply(params.runId, params);
|
|
1006
|
+
return {};
|
|
1007
|
+
}),
|
|
1008
|
+
handler(relayLiveValueMethod, (params, session) => {
|
|
1009
|
+
if (options.ownerOf(params.artifactId) !== session.link?.instanceId) {
|
|
1010
|
+
throw failed(
|
|
1011
|
+
"MOSTRA_REMOTE_NOT_OWNER_INSTANCE",
|
|
1012
|
+
"This instance doesn't own that artifact here.",
|
|
1013
|
+
);
|
|
1014
|
+
}
|
|
1015
|
+
return (
|
|
1016
|
+
options.storeLive?.(params.artifactId, params.versionId, {
|
|
1017
|
+
key: params.key,
|
|
1018
|
+
...(params.value !== undefined ? { value: params.value } : {}),
|
|
1019
|
+
...(params.error !== undefined ? { error: params.error } : {}),
|
|
1020
|
+
}) ?? {}
|
|
1021
|
+
);
|
|
1022
|
+
}),
|
|
1023
|
+
handler(relayEffectMethod, (params, session) => {
|
|
1024
|
+
// Only the artifact's owner instance speaks for its runs (D165 item 1b).
|
|
1025
|
+
if (options.ownerOf(params.artifactId) !== session.link?.instanceId) {
|
|
1026
|
+
throw failed(
|
|
1027
|
+
"MOSTRA_REMOTE_NOT_OWNER_INSTANCE",
|
|
1028
|
+
"This instance doesn't own that artifact here.",
|
|
1029
|
+
);
|
|
1030
|
+
}
|
|
1031
|
+
const row =
|
|
1032
|
+
params.runId !== undefined
|
|
1033
|
+
? ledger.read((db) => runRow(db, params.runId as string))
|
|
1034
|
+
: undefined;
|
|
1035
|
+
if (params.runId !== undefined && row?.artifact_id !== params.artifactId) {
|
|
1036
|
+
throw failed("MOSTRA_ACTION_RUN_NOT_FOUND", "This home has no such run.");
|
|
1037
|
+
}
|
|
1038
|
+
const viewerId = row?.viewer_id ?? undefined;
|
|
1039
|
+
const common = {
|
|
1040
|
+
...(params.payload !== undefined ? { payload: params.payload } : {}),
|
|
1041
|
+
...(params.runId !== undefined ? { runId: params.runId } : {}),
|
|
1042
|
+
...(viewerId !== undefined ? { viewerId } : {}),
|
|
1043
|
+
};
|
|
1044
|
+
if (params.kind === "signal") {
|
|
1045
|
+
return {
|
|
1046
|
+
cursor: interaction.serverSignal(params.artifactId, { name: params.name, ...common }),
|
|
1047
|
+
};
|
|
1048
|
+
}
|
|
1049
|
+
if (!/^host\.[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/.test(params.name)) {
|
|
1050
|
+
throw failed(
|
|
1051
|
+
"MOSTRA_ACTION_EVENT_INVALID",
|
|
1052
|
+
`${params.name.slice(0, 80)} isn't a host event type.`,
|
|
1053
|
+
);
|
|
1054
|
+
}
|
|
1055
|
+
return {
|
|
1056
|
+
cursor: interaction.serverEmit(params.artifactId, { type: params.name, ...common }),
|
|
1057
|
+
};
|
|
1058
|
+
}),
|
|
1059
|
+
handler(relayCallBatchMethod, (params, session) => {
|
|
1060
|
+
const pending = calls.get(params.callId);
|
|
1061
|
+
if (!pending || pending.session !== session.id || params.seq <= pending.lastSeq)
|
|
1062
|
+
return {};
|
|
1063
|
+
pending.lastSeq = params.seq;
|
|
1064
|
+
for (const batch of params.batches ?? []) pending.lines.push({ batch: [...batch] });
|
|
1065
|
+
if (params.end) {
|
|
1066
|
+
pending.lines.push(
|
|
1067
|
+
params.end.status === "succeeded"
|
|
1068
|
+
? { end: { status: "succeeded" } }
|
|
1069
|
+
: {
|
|
1070
|
+
end: {
|
|
1071
|
+
status: "failed",
|
|
1072
|
+
error: params.end.error ?? { message: "The call failed on its owner." },
|
|
1073
|
+
},
|
|
1074
|
+
},
|
|
1075
|
+
);
|
|
1076
|
+
pending.ended = true;
|
|
1077
|
+
}
|
|
1078
|
+
pending.wake?.();
|
|
1079
|
+
return {};
|
|
1080
|
+
}),
|
|
1081
|
+
];
|
|
1082
|
+
},
|
|
1083
|
+
|
|
1084
|
+
connected(session) {
|
|
1085
|
+
const linkId = session.link?.instanceId;
|
|
1086
|
+
if (linkId === undefined || !session.ports.has("relay")) return;
|
|
1087
|
+
const mark = session.link?.relayMark ?? 0;
|
|
1088
|
+
marks.set(linkId, Math.max(marks.get(linkId) ?? 0, mark));
|
|
1089
|
+
sessionMarks.set(session.id, mark);
|
|
1090
|
+
void resume(linkId, session.id, mark);
|
|
1091
|
+
},
|
|
1092
|
+
|
|
1093
|
+
disconnected(session) {
|
|
1094
|
+
const linkId = session.link?.instanceId;
|
|
1095
|
+
sessionMarks.delete(session.id);
|
|
1096
|
+
const retry = retries.get(session.id);
|
|
1097
|
+
if (retry) clearTimeout(retry.timer);
|
|
1098
|
+
retries.delete(session.id);
|
|
1099
|
+
if (linkId !== undefined && ready.get(linkId) === session.id) ready.delete(linkId);
|
|
1100
|
+
// The link's session now, if it isn't ready (it opened while this one was still the link's,
|
|
1101
|
+
// so its resume stopped short), is resumed: nothing else would (Opus review of WP-29, 1).
|
|
1102
|
+
if (linkId !== undefined) {
|
|
1103
|
+
const live = options.sessionOf(linkId);
|
|
1104
|
+
if (live !== undefined && live !== session.id && ready.get(linkId) !== live) {
|
|
1105
|
+
void resume(linkId, live, sessionMarks.get(live) ?? marks.get(linkId) ?? 0);
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
for (const [callId, pending] of calls) {
|
|
1109
|
+
if (pending.session !== session.id) continue;
|
|
1110
|
+
pending.lines.push({
|
|
1111
|
+
end: {
|
|
1112
|
+
status: "failed",
|
|
1113
|
+
error: {
|
|
1114
|
+
code: "MOSTRA_REMOTE_OWNER_OFFLINE",
|
|
1115
|
+
message: "The page's owner went offline before the call ended.",
|
|
1116
|
+
},
|
|
1117
|
+
},
|
|
1118
|
+
});
|
|
1119
|
+
pending.ended = true;
|
|
1120
|
+
pending.wake?.();
|
|
1121
|
+
calls.delete(callId);
|
|
1122
|
+
}
|
|
1123
|
+
},
|
|
1124
|
+
|
|
1125
|
+
busy() {
|
|
1126
|
+
return (
|
|
1127
|
+
calls.size > 0 ||
|
|
1128
|
+
ledger.read(
|
|
1129
|
+
(db) =>
|
|
1130
|
+
(db.get<{ n: number }>(
|
|
1131
|
+
"SELECT COUNT(*) AS n FROM action_run WHERE status IN ('queued', 'running', 'waiting')",
|
|
1132
|
+
)?.n ?? 0) > 0,
|
|
1133
|
+
)
|
|
1134
|
+
);
|
|
1135
|
+
},
|
|
1136
|
+
|
|
1137
|
+
liveWanted(artifactId) {
|
|
1138
|
+
const known = asked.get(artifactId);
|
|
1139
|
+
const fresh = bus
|
|
1140
|
+
.liveInterest()
|
|
1141
|
+
.some(
|
|
1142
|
+
(entry) =>
|
|
1143
|
+
entry.artifactId === artifactId &&
|
|
1144
|
+
!entry.key.startsWith(TALLY_LIVE_PREFIX) &&
|
|
1145
|
+
!(known?.has(entry.key) ?? false),
|
|
1146
|
+
);
|
|
1147
|
+
if (fresh) renewLive(artifactId);
|
|
1148
|
+
},
|
|
1149
|
+
|
|
1150
|
+
ownerOffline(artifactId) {
|
|
1151
|
+
const owner = options.ownerOf(artifactId);
|
|
1152
|
+
return owner === undefined || sessionFor(owner) === undefined;
|
|
1153
|
+
},
|
|
1154
|
+
|
|
1155
|
+
stop() {
|
|
1156
|
+
stopped = true;
|
|
1157
|
+
for (const retry of retries.values()) clearTimeout(retry.timer);
|
|
1158
|
+
retries.clear();
|
|
1159
|
+
clearInterval(sweeper);
|
|
1160
|
+
clearInterval(liveRenewer);
|
|
1161
|
+
},
|
|
1162
|
+
};
|
|
1163
|
+
}
|