@serfab/cadre-core 1.17.0 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/cadre-invite-protocol.d.ts +35 -13
- package/dist/cadre-invite-protocol.js +90 -25
- package/dist/cadre-invite-protocol.js.map +1 -1
- package/dist/cadre-node.d.ts +586 -255
- package/dist/cadre-node.js +1176 -564
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-cohort.d.ts +66 -9
- package/dist/control-cohort.js +67 -9
- package/dist/control-cohort.js.map +1 -1
- package/dist/control-database.d.ts +451 -193
- package/dist/control-database.js +701 -359
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +46 -43
- package/dist/control-formation-recorder.js +77 -78
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-protocol-guard.d.ts +6 -0
- package/dist/control-protocol-guard.js +14 -2
- package/dist/control-protocol-guard.js.map +1 -1
- package/dist/control-schema-version.d.ts +48 -31
- package/dist/control-schema-version.js +67 -45
- package/dist/control-schema-version.js.map +1 -1
- package/dist/control-schema.d.ts +8 -8
- package/dist/control-schema.js +16 -16
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +42 -7
- package/dist/control-write-retry.js +87 -19
- package/dist/control-write-retry.js.map +1 -1
- package/dist/enrolled-machine-store.d.ts +3 -3
- package/dist/enrolled-machine-store.js +3 -3
- package/dist/formation-token.d.ts +48 -0
- package/dist/formation-token.js +26 -0
- package/dist/formation-token.js.map +1 -0
- package/dist/index.d.ts +14 -13
- package/dist/index.js +16 -13
- package/dist/index.js.map +1 -1
- package/dist/joined-strand-store.d.ts +26 -9
- package/dist/joined-strand-store.js +63 -24
- package/dist/joined-strand-store.js.map +1 -1
- package/dist/link-budget.d.ts +9 -7
- package/dist/link-budget.js +9 -7
- package/dist/link-budget.js.map +1 -1
- package/dist/member-redemption.d.ts +15 -9
- package/dist/member-redemption.js +13 -8
- package/dist/member-redemption.js.map +1 -1
- package/dist/membership-view.d.ts +91 -0
- package/dist/membership-view.js +110 -0
- package/dist/membership-view.js.map +1 -0
- package/dist/owner-grant-protocol.d.ts +11 -7
- package/dist/owner-grant-protocol.js +31 -17
- package/dist/owner-grant-protocol.js.map +1 -1
- package/dist/peer-addr-book.d.ts +10 -1
- package/dist/peer-addr-book.js +4 -3
- package/dist/peer-addr-book.js.map +1 -1
- package/dist/peer-join-backfill.d.ts +89 -20
- package/dist/peer-join-backfill.js +265 -108
- package/dist/peer-join-backfill.js.map +1 -1
- package/dist/pending-join-runner.d.ts +48 -18
- package/dist/pending-join-runner.js +74 -56
- package/dist/pending-join-runner.js.map +1 -1
- package/dist/seed-bootstrap.d.ts +47 -40
- package/dist/seed-bootstrap.js +66 -61
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/strand-database.d.ts +23 -2
- package/dist/strand-database.js +26 -6
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-first-sync-gate.d.ts +176 -89
- package/dist/strand-first-sync-gate.js +288 -122
- package/dist/strand-first-sync-gate.js.map +1 -1
- package/dist/strand-formation-deadlines.d.ts +9 -9
- package/dist/strand-formation-deadlines.js +9 -9
- package/dist/strand-formation-manager.d.ts +23 -17
- package/dist/strand-formation-manager.js +46 -44
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +5 -8
- package/dist/strand-formation-protocol.js +101 -16
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +26 -17
- package/dist/strand-instance-manager.js +119 -50
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-reconciler.d.ts +16 -13
- package/dist/strand-membership-reconciler.js +30 -22
- package/dist/strand-membership-reconciler.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +40 -28
- package/dist/strand-membership-writer.js +212 -153
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-move.js +5 -2
- package/dist/strand-move.js.map +1 -1
- package/dist/strand-revocation-enforcer.d.ts +134 -60
- package/dist/strand-revocation-enforcer.js +144 -71
- package/dist/strand-revocation-enforcer.js.map +1 -1
- package/dist/strand-solicitation.d.ts +50 -39
- package/dist/strand-solicitation.js +10 -3
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-watcher.d.ts +80 -13
- package/dist/strand-watcher.js +152 -48
- package/dist/strand-watcher.js.map +1 -1
- package/dist/table-change-watch.d.ts +63 -0
- package/dist/table-change-watch.js +209 -0
- package/dist/table-change-watch.js.map +1 -0
- package/dist/timeout-scheduler.d.ts +4 -3
- package/dist/timeout-scheduler.js +4 -3
- package/dist/timeout-scheduler.js.map +1 -1
- package/dist/types.d.ts +90 -49
- package/dist/types.js +4 -4
- package/dist/types.js.map +1 -1
- package/package.json +8 -8
- package/src/cadre-invite-protocol.ts +119 -31
- package/src/cadre-node.ts +1282 -584
- package/src/control-cohort.ts +86 -9
- package/src/control-database.ts +848 -403
- package/src/control-formation-recorder.ts +359 -353
- package/src/control-protocol-guard.ts +16 -2
- package/src/control-schema-version.ts +101 -51
- package/src/control-schema.ts +16 -16
- package/src/control-write-retry.ts +91 -19
- package/src/delegate-admission.ts +351 -351
- package/src/enrolled-machine-store.ts +3 -3
- package/src/formation-token.ts +63 -0
- package/src/index.ts +30 -17
- package/src/joined-strand-store.ts +69 -24
- package/src/link-budget.ts +9 -7
- package/src/member-redemption.ts +20 -13
- package/src/membership-view.ts +144 -0
- package/src/owner-grant-protocol.ts +33 -18
- package/src/peer-addr-book.ts +4 -3
- package/src/peer-join-backfill.ts +971 -761
- package/src/pending-join-runner.ts +76 -53
- package/src/seed-bootstrap.ts +72 -67
- package/src/strand-database.ts +352 -322
- package/src/strand-first-sync-gate.ts +334 -134
- package/src/strand-formation-deadlines.ts +227 -227
- package/src/strand-formation-manager.ts +731 -725
- package/src/strand-formation-protocol.ts +122 -24
- package/src/strand-instance-manager.ts +2092 -2026
- package/src/strand-membership-reconciler.ts +33 -25
- package/src/strand-membership-writer.ts +2093 -2019
- package/src/strand-move.ts +5 -2
- package/src/strand-revocation-enforcer.ts +740 -654
- package/src/strand-solicitation.ts +61 -34
- package/src/strand-watcher.ts +175 -56
- package/src/table-change-watch.ts +285 -0
- package/src/timeout-scheduler.ts +29 -28
- package/src/types.ts +89 -51
|
@@ -24,6 +24,7 @@ import {
|
|
|
24
24
|
isValidResponderCreatesResult,
|
|
25
25
|
type FormationResultMessage
|
|
26
26
|
} from './strand-formation-protocol.js';
|
|
27
|
+
import { classifyTokenRead, type FormationTokenRead } from './formation-token.js';
|
|
27
28
|
|
|
28
29
|
const log = debug('sereus:cadre:solicitation');
|
|
29
30
|
|
|
@@ -40,6 +41,25 @@ export interface DisclosureValidator {
|
|
|
40
41
|
validateDisclosure(token: string, disclosure: StrandFormationDisclosure): Promise<boolean>;
|
|
41
42
|
}
|
|
42
43
|
|
|
44
|
+
/**
|
|
45
|
+
* A bound invite's host strand as {@link FormationUsageRecorder.resolveStrand} read it, once
|
|
46
|
+
* per session: the membership invitation is issued from it and the consent row recorded
|
|
47
|
+
* against it without reading the `Strand` row again.
|
|
48
|
+
*/
|
|
49
|
+
export interface BoundHostStrand {
|
|
50
|
+
strandId: string;
|
|
51
|
+
/** `'c'` closed: the joiner gets a membership invitation; `'o'` open: nothing to invite into. */
|
|
52
|
+
strandType: 'o' | 'c';
|
|
53
|
+
/**
|
|
54
|
+
* The row's one-off `StampId`, which the consent row names (`FormationUsage.StrandStampId`).
|
|
55
|
+
* Fixed for the life of the row; a row removed and re-seated in between gets a new one, and
|
|
56
|
+
* the `StrandExists` CHECK then refuses the consent row.
|
|
57
|
+
*/
|
|
58
|
+
strandStampId: string;
|
|
59
|
+
/** The closed-strand read-gating secret; null for an open strand. */
|
|
60
|
+
memberPrivateKey: string | null;
|
|
61
|
+
}
|
|
62
|
+
|
|
43
63
|
/**
|
|
44
64
|
* Discriminated result of resolving the host strand an invite binds to (responder side):
|
|
45
65
|
*
|
|
@@ -52,16 +72,13 @@ export interface DisclosureValidator {
|
|
|
52
72
|
* non-existent strand, which would fail the deferred `FormationUsage.StrandExists` CHECK at
|
|
53
73
|
* commit and drop the result frame.
|
|
54
74
|
*
|
|
55
|
-
* Replaces the older `{ strandId, memberPrivateKey } | null` shape, which conflated `bound`
|
|
56
|
-
* and `missing` (both produced a non-null result whenever the invite had a `StrandId`).
|
|
57
|
-
*
|
|
58
75
|
* The invite's `ValidationUrl` (the outside approval hook) is deliberately NOT surfaced here:
|
|
59
|
-
* the recorder that performs the write
|
|
60
|
-
* never travels through a caller.
|
|
76
|
+
* the recorder that performs the write takes it from the session's {@link FormationTokenRead},
|
|
77
|
+
* so the choice of which hook to trust never travels through a caller.
|
|
61
78
|
*/
|
|
62
79
|
export type ResolvedHostStrand =
|
|
63
80
|
| { kind: 'unbound' }
|
|
64
|
-
| { kind: 'bound'
|
|
81
|
+
| ({ kind: 'bound' } & BoundHostStrand)
|
|
65
82
|
| { kind: 'missing'; strandId: string };
|
|
66
83
|
|
|
67
84
|
/** One redemption of an invite against an already-existing host strand (the record-only consent write). */
|
|
@@ -87,48 +104,52 @@ export interface AuthorizedFormationUsage {
|
|
|
87
104
|
}
|
|
88
105
|
|
|
89
106
|
/**
|
|
90
|
-
*
|
|
107
|
+
* The responder's view of the formation-invite tables. A session reads its token once
|
|
108
|
+
* ({@link readToken}) and hands that read to every later step, so no step reads the invite again.
|
|
91
109
|
*/
|
|
92
110
|
export interface FormationUsageRecorder {
|
|
93
111
|
/**
|
|
94
|
-
* Record that a formation invite was used
|
|
95
|
-
* is the same as
|
|
112
|
+
* Record that a formation invite was used, reading the invite for itself. Where
|
|
113
|
+
* {@link authorizeUsage} is implemented, this is the same as reading the token, authorizing,
|
|
114
|
+
* and calling `record()` straight away.
|
|
96
115
|
*/
|
|
97
116
|
recordUsage(params: FormationUsageParams): Promise<void>;
|
|
98
117
|
|
|
99
118
|
/**
|
|
100
119
|
* Everything {@link recordUsage} does short of the write: obtain the outside approval (when
|
|
101
|
-
* the invite demands one) and
|
|
102
|
-
* writes the consent row with exactly the fields
|
|
103
|
-
* before it writes anything into the host strand,
|
|
120
|
+
* `read` says the invite demands one) and, after an obtained approval, re-check the seat
|
|
121
|
+
* budget, returning a handle whose `record()` writes the consent row with exactly the fields
|
|
122
|
+
* that were approved. Lets the manager ask before it writes anything into the host strand,
|
|
123
|
+
* and write consent only after. `host` is the resolved host strand the consent row names.
|
|
104
124
|
* Optional: a recorder without it is treated as having nothing to ask up front.
|
|
105
125
|
*/
|
|
106
|
-
authorizeUsage?(
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
isTokenUsed(token: string): Promise<boolean>;
|
|
126
|
+
authorizeUsage?(
|
|
127
|
+
params: FormationUsageParams,
|
|
128
|
+
read: FormationTokenRead,
|
|
129
|
+
host: BoundHostStrand
|
|
130
|
+
): Promise<AuthorizedFormationUsage>;
|
|
112
131
|
|
|
113
132
|
/**
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
133
|
+
* Read one token as a formation session acts on it: the invite row (null when there is none
|
|
134
|
+
* here — it may not have replicated yet) and, for a metered unexpired invite, its recorded
|
|
135
|
+
* use count. One read per session; `classifyTokenRead` turns it into the token check.
|
|
117
136
|
*/
|
|
118
|
-
|
|
137
|
+
readToken(token: string): Promise<FormationTokenRead>;
|
|
119
138
|
|
|
120
139
|
/**
|
|
121
|
-
* Resolve the host strand
|
|
122
|
-
* missing (see {@link ResolvedHostStrand}).
|
|
123
|
-
*
|
|
124
|
-
*
|
|
125
|
-
*
|
|
140
|
+
* Resolve the host strand the read invite binds to, classifying it as unbound / bound /
|
|
141
|
+
* missing (see {@link ResolvedHostStrand}). Reads the `Strand` row, never the invite.
|
|
142
|
+
* Optional: a recorder that does not bind strands omits it, and
|
|
143
|
+
* {@link StrandFormationManager} treats every invite as `unbound` and falls back to its
|
|
144
|
+
* provisioner. Keeping resolution behind this interface keeps the manager DB-agnostic and
|
|
145
|
+
* unit-testable with an in-memory fake.
|
|
126
146
|
*/
|
|
127
|
-
resolveStrand?(
|
|
147
|
+
resolveStrand?(read: FormationTokenRead): Promise<ResolvedHostStrand>;
|
|
128
148
|
|
|
129
149
|
/**
|
|
130
150
|
* Provision a NEW strand for an UNBOUND invite and record consent against it
|
|
131
151
|
* ATOMICALLY (single `FormationUsage` row → single-use enforced on the next redemption).
|
|
152
|
+
* Takes the invite's approval hook and seat budget from `read`.
|
|
132
153
|
* Optional: a recorder that only supports provision-then-record omits it, and the
|
|
133
154
|
* manager falls back to its {@link StrandProvisioner}. Mirrors the create-strand-by-consent
|
|
134
155
|
* path the schema's `Strand.AuthorizedInsert` `FormationUsage` branch authorizes. Returns the
|
|
@@ -147,14 +168,14 @@ export interface FormationUsageRecorder {
|
|
|
147
168
|
disclosure: string;
|
|
148
169
|
/** Aborted when the caller has given up; observed BEFORE any write so the invite stays unspent. */
|
|
149
170
|
signal?: AbortSignal;
|
|
150
|
-
}): Promise<{ strandId: string; memberPrivateKey: string | null }>;
|
|
171
|
+
}, read: FormationTokenRead): Promise<{ strandId: string; memberPrivateKey: string | null }>;
|
|
151
172
|
|
|
152
173
|
/**
|
|
153
174
|
* Is ANY invitation this recorder knows about unexpired and not fully
|
|
154
175
|
* consumed? Optional: a recorder that cannot enumerate invitations omits it,
|
|
155
176
|
* and only locally-minted invitations hold the connection gate's formation
|
|
156
|
-
* exemption open. Distinct from the per-token {@link
|
|
157
|
-
*
|
|
177
|
+
* exemption open. Distinct from the per-token {@link readToken} — the
|
|
178
|
+
* connection gate has no token to ask about.
|
|
158
179
|
*/
|
|
159
180
|
hasOutstandingInvitation?(): Promise<boolean>;
|
|
160
181
|
}
|
|
@@ -470,8 +491,8 @@ export class StrandSolicitationService {
|
|
|
470
491
|
* in-memory and dies with the process. Expired entries are dropped. With
|
|
471
492
|
* no {@link FormationUsageRecorder} configured there is no consumption
|
|
472
493
|
* oracle, so an unexpired minted token counts as outstanding by
|
|
473
|
-
* construction; with one, a token it
|
|
474
|
-
* is permanent, so the registry never reconsiders it).
|
|
494
|
+
* construction; with one, a token whose row it reads as expired or used up
|
|
495
|
+
* is deleted (both are permanent, so the registry never reconsiders it).
|
|
475
496
|
* 2. The recorder's optional {@link FormationUsageRecorder.hasOutstandingInvitation},
|
|
476
497
|
* which survives restarts and sees invitations replicated in from siblings.
|
|
477
498
|
*
|
|
@@ -488,7 +509,13 @@ export class StrandSolicitationService {
|
|
|
488
509
|
if (!this.formationUsageRecorder) {
|
|
489
510
|
return true;
|
|
490
511
|
}
|
|
491
|
-
|
|
512
|
+
// NOTE: a read of its own, not shared with a formation session's: this runs while a
|
|
513
|
+
// stranger's connection is being admitted, before any stream or token exists, under the
|
|
514
|
+
// gate's fail-open deadline, and the session that follows reads on a different call.
|
|
515
|
+
// An unknown token stays outstanding (its row may not be published yet); an expired or
|
|
516
|
+
// used-up row is dropped, the same `token-spent` the formation handler would answer.
|
|
517
|
+
const check = classifyTokenRead(await this.formationUsageRecorder.readToken(token), now);
|
|
518
|
+
if (check.valid || check.code === 'token-unknown') {
|
|
492
519
|
return true;
|
|
493
520
|
}
|
|
494
521
|
this.mintedInvitations.delete(token);
|
package/src/strand-watcher.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import debug from 'debug';
|
|
2
|
+
import { scheduleRuns } from './table-change-watch.js';
|
|
3
|
+
import type { TableChangeWatch, TableChangeWatchOptions } from './table-change-watch.js';
|
|
2
4
|
import type { StrandFilter, StrandRow } from './types.js';
|
|
3
5
|
|
|
4
6
|
const log = debug('sereus:cadre:strand-watcher');
|
|
@@ -7,7 +9,7 @@ const log = debug('sereus:cadre:strand-watcher');
|
|
|
7
9
|
* Outcome of evaluating a strand against the current filter.
|
|
8
10
|
* - `pass`: admit (final)
|
|
9
11
|
* - `reject`: exclude
|
|
10
|
-
* - `defer`: admit provisionally; sAppId not yet known, re-evaluate
|
|
12
|
+
* - `defer`: admit provisionally; sAppId not yet known, re-evaluate when it is learned
|
|
11
13
|
*/
|
|
12
14
|
type FilterDecision = 'pass' | 'reject' | 'defer';
|
|
13
15
|
|
|
@@ -15,10 +17,34 @@ type FilterDecision = 'pass' | 'reject' | 'defer';
|
|
|
15
17
|
* Ceiling on the retry delay for a strand whose launch keeps failing. Attempts
|
|
16
18
|
* are never abandoned — the failure this backoff exists for (a transient network
|
|
17
19
|
* or storage fault) can last a long time — but a permanently-unlaunchable strand
|
|
18
|
-
* must not re-attempt on every
|
|
20
|
+
* must not re-attempt on every run and storm the host app with `strand:error`.
|
|
19
21
|
*/
|
|
20
22
|
const MAX_RETRY_BACKOFF_MS = 5 * 60 * 1000;
|
|
21
23
|
|
|
24
|
+
/**
|
|
25
|
+
* Default backstop: the longest the watcher goes without re-reading when no change to
|
|
26
|
+
* `Strand` or `JoinedStrand` is heard (`docs/architecture.md` → "Table change subscriptions"
|
|
27
|
+
* says what the change subscription misses).
|
|
28
|
+
*
|
|
29
|
+
* NOTE: on a cadre larger than `CONTROL_REPLICATION_BREADTH` (16), a machine outside the
|
|
30
|
+
* `Strand` block's cohort hears no change event, so this backstop is its only path and a
|
|
31
|
+
* strand founded elsewhere reaches it up to 5 minutes later. If that matters, shorten the
|
|
32
|
+
* backstop for such machines rather than for everyone.
|
|
33
|
+
*/
|
|
34
|
+
export const DEFAULT_STRAND_WATCH_BACKSTOP_MS = 5 * 60 * 1000;
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* First step of the launch-retry and read-retry ladders, unless the backstop is shorter: a
|
|
38
|
+
* test or scenario that passes a short interval keeps the short base it always had.
|
|
39
|
+
*/
|
|
40
|
+
const LAUNCH_RETRY_BASE_MS = 5000;
|
|
41
|
+
|
|
42
|
+
/** Ceiling on the read-retry ladder (see `StrandWatcher.scheduleReadRetry`), unless the backstop is shorter. */
|
|
43
|
+
const READ_RETRY_CAP_MS = 60_000;
|
|
44
|
+
|
|
45
|
+
/** Delay before the first run, so `start()` never reads synchronously. */
|
|
46
|
+
const INITIAL_RUN_DELAY_MS = 100;
|
|
47
|
+
|
|
22
48
|
/** Per-strand consecutive-failure state driving the launch retry backoff. */
|
|
23
49
|
interface StrandFailureState {
|
|
24
50
|
/** Consecutive `onStrandAdded` failures since the last success. */
|
|
@@ -45,10 +71,14 @@ export interface StrandWatcherCallbacks {
|
|
|
45
71
|
}
|
|
46
72
|
|
|
47
73
|
/**
|
|
48
|
-
*
|
|
74
|
+
* What the watcher reads, and how it hears that what it reads changed.
|
|
49
75
|
*/
|
|
50
76
|
export interface StrandQueryable {
|
|
51
77
|
queryStrands(): Promise<StrandRow[]>;
|
|
78
|
+
/** Subscribe to changes of the tables {@link queryStrands} reads. Absent: runs come from `trigger` and the backstop only. */
|
|
79
|
+
watchChanges?(onChange: () => Promise<void>, options: TableChangeWatchOptions): TableChangeWatch;
|
|
80
|
+
/** True when the last {@link queryStrands} answered part of its result from a stand-in after a failed read. */
|
|
81
|
+
lastReadPartial?(): boolean;
|
|
52
82
|
}
|
|
53
83
|
|
|
54
84
|
/**
|
|
@@ -63,11 +93,16 @@ export interface SAppIdLookup {
|
|
|
63
93
|
* Watches the control network's Strand table for changes and triggers
|
|
64
94
|
* strand instance start/stop via callbacks.
|
|
65
95
|
*
|
|
66
|
-
*
|
|
96
|
+
* Every read is one run on a {@link TableChangeWatch}: once at start, on a change to the
|
|
97
|
+
* tables the queryable reads, on the watcher's own retries, and on the backstop.
|
|
67
98
|
*/
|
|
68
99
|
export class StrandWatcher {
|
|
69
100
|
private readonly filter: StrandFilter;
|
|
70
|
-
|
|
101
|
+
/** Longest gap between runs when no change is heard. */
|
|
102
|
+
private readonly backstopMs: number;
|
|
103
|
+
/** First step of the launch-retry and read-retry ladders. */
|
|
104
|
+
private readonly retryBaseMs: number;
|
|
105
|
+
private readonly readRetryCapMs: number;
|
|
71
106
|
private readonly callbacks: StrandWatcherCallbacks;
|
|
72
107
|
private readonly queryable: StrandQueryable;
|
|
73
108
|
private readonly sAppIdLookup?: SAppIdLookup;
|
|
@@ -81,7 +116,7 @@ export class StrandWatcher {
|
|
|
81
116
|
private readonly now: () => number;
|
|
82
117
|
|
|
83
118
|
private knownStrands: Map<string, StrandRow> = new Map();
|
|
84
|
-
/** Ids admitted under a `defer` decision; re-evaluated each
|
|
119
|
+
/** Ids admitted under a `defer` decision; re-evaluated each run until they resolve. */
|
|
85
120
|
private provisional: Set<string> = new Set();
|
|
86
121
|
/** Ids whose last launch attempt threw, with the backoff gating their retry. */
|
|
87
122
|
private failureStates: Map<string, StrandFailureState> = new Map();
|
|
@@ -92,31 +127,39 @@ export class StrandWatcher {
|
|
|
92
127
|
* strand, the permanence has to be recorded explicitly.
|
|
93
128
|
*/
|
|
94
129
|
private suppressed: Set<string> = new Set();
|
|
95
|
-
|
|
96
|
-
private
|
|
130
|
+
/** Consecutive runs whose read failed or was partial; drives {@link scheduleReadRetry}. */
|
|
131
|
+
private readFailures = 0;
|
|
132
|
+
/** Built by {@link start}, disposed by {@link stop}; a restart builds a new one. */
|
|
133
|
+
private watch: TableChangeWatch | null = null;
|
|
97
134
|
private running = false;
|
|
98
135
|
|
|
136
|
+
/**
|
|
137
|
+
* @param backstopMs - `CadreNodeConfig.strandWatchInterval`: the longest the watcher goes
|
|
138
|
+
* without re-reading when no change is heard. Also caps the retry ladders' first step.
|
|
139
|
+
*/
|
|
99
140
|
constructor(
|
|
100
141
|
queryable: StrandQueryable,
|
|
101
142
|
callbacks: StrandWatcherCallbacks,
|
|
102
143
|
filter: StrandFilter = { mode: 'all' },
|
|
103
|
-
|
|
144
|
+
backstopMs: number = DEFAULT_STRAND_WATCH_BACKSTOP_MS,
|
|
104
145
|
sAppIdLookup?: SAppIdLookup,
|
|
105
146
|
now: () => number = Date.now
|
|
106
147
|
) {
|
|
107
148
|
this.queryable = queryable;
|
|
108
149
|
this.callbacks = callbacks;
|
|
109
150
|
this.filter = filter;
|
|
110
|
-
this.
|
|
151
|
+
this.backstopMs = backstopMs;
|
|
152
|
+
this.retryBaseMs = Math.min(backstopMs, LAUNCH_RETRY_BASE_MS);
|
|
153
|
+
this.readRetryCapMs = Math.min(backstopMs, READ_RETRY_CAP_MS);
|
|
111
154
|
this.sAppIdLookup = sAppIdLookup;
|
|
112
155
|
this.now = now;
|
|
113
|
-
log('StrandWatcher created with filter: %o,
|
|
156
|
+
log('StrandWatcher created with filter: %o, backstop: %dms', filter, backstopMs);
|
|
114
157
|
}
|
|
115
158
|
|
|
116
159
|
/**
|
|
117
160
|
* Evaluate a strand against the current filter, distinguishing a not-yet-known
|
|
118
161
|
* sAppId (`defer`) from a known non-match (`reject`). A `defer` admission is
|
|
119
|
-
* provisional and re-checked on subsequent
|
|
162
|
+
* provisional and re-checked on subsequent runs (see {@link sAppIdLearned}).
|
|
120
163
|
*/
|
|
121
164
|
private evaluateFilter(strand: StrandRow): FilterDecision {
|
|
122
165
|
switch (this.filter.mode) {
|
|
@@ -133,7 +176,7 @@ export class StrandWatcher {
|
|
|
133
176
|
}
|
|
134
177
|
const sAppId = this.sAppIdLookup.getSAppId(strand.Id);
|
|
135
178
|
if (sAppId === undefined) {
|
|
136
|
-
// sAppId not yet known - admit provisionally and re-evaluate
|
|
179
|
+
// sAppId not yet known - admit provisionally and re-evaluate once it is learned.
|
|
137
180
|
log('sAppId unknown for strand %s - deferring filter decision', strand.Id);
|
|
138
181
|
return 'defer';
|
|
139
182
|
}
|
|
@@ -149,20 +192,73 @@ export class StrandWatcher {
|
|
|
149
192
|
|
|
150
193
|
/**
|
|
151
194
|
* Record a failed launch attempt and schedule when the strand may be retried:
|
|
152
|
-
* `
|
|
195
|
+
* `retryBaseMs * 2^(failures-1)`, capped at {@link MAX_RETRY_BACKOFF_MS}.
|
|
153
196
|
*/
|
|
154
197
|
private recordFailure(strandId: string): void {
|
|
155
198
|
const failures = (this.failureStates.get(strandId)?.failures ?? 0) + 1;
|
|
156
|
-
const delay = Math.min(this.
|
|
199
|
+
const delay = Math.min(this.retryBaseMs * 2 ** (failures - 1), MAX_RETRY_BACKOFF_MS);
|
|
157
200
|
this.failureStates.set(strandId, { failures, nextAttemptAt: this.now() + delay });
|
|
158
201
|
log('Strand %s launch failed (%d consecutive) - next attempt in %dms', strandId, failures, delay);
|
|
159
202
|
}
|
|
160
203
|
|
|
161
204
|
/**
|
|
162
|
-
*
|
|
205
|
+
* One run: read, reconcile, then arm the follow-up runs this run needs. A run the watch
|
|
206
|
+
* starts clears every pending wake, so each run re-arms the retries still owed. A
|
|
207
|
+
* {@link forcePoll} run clears none, so what it arms may sit beside an equivalent wake;
|
|
208
|
+
* whichever fires first runs and clears the other.
|
|
163
209
|
*/
|
|
164
|
-
private async
|
|
165
|
-
|
|
210
|
+
private async runOnce(): Promise<void> {
|
|
211
|
+
const startedAt = this.now();
|
|
212
|
+
const readComplete = await this.poll();
|
|
213
|
+
if (this.running) {
|
|
214
|
+
this.scheduleLaunchRetry(startedAt);
|
|
215
|
+
this.scheduleReadRetry(readComplete);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Wake for the earliest launch retry falling due after `since`. A backoff that expired at
|
|
221
|
+
* or before `since` belongs to a strand the run starting then either attempted or skipped
|
|
222
|
+
* for a reason other than backoff (suppressed, filtered out); waking for it would spin.
|
|
223
|
+
* Without this a failed launch would wait for an unrelated change or the backstop.
|
|
224
|
+
*/
|
|
225
|
+
private scheduleLaunchRetry(since: number): void {
|
|
226
|
+
let earliest = Infinity;
|
|
227
|
+
for (const { nextAttemptAt } of this.failureStates.values()) {
|
|
228
|
+
if (nextAttemptAt > since) {
|
|
229
|
+
earliest = Math.min(earliest, nextAttemptAt);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
if (earliest !== Infinity) {
|
|
233
|
+
this.watch?.trigger(Math.max(0, earliest - this.now()));
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* After a failed or partial read, run again on a ladder from `retryBaseMs`, doubling per
|
|
239
|
+
* consecutive failed run up to `readRetryCapMs`; a complete read resets it. Not left to
|
|
240
|
+
* the backstop because the plugin arms a table's replicated-change bridge on the table's
|
|
241
|
+
* first successful initialization: a machine whose first read failed (cut off from its
|
|
242
|
+
* party, or its sibling unreachable for a never-written `JoinedStrand`) would otherwise
|
|
243
|
+
* hear nothing until then.
|
|
244
|
+
*/
|
|
245
|
+
private scheduleReadRetry(readComplete: boolean): void {
|
|
246
|
+
if (readComplete) {
|
|
247
|
+
this.readFailures = 0;
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
this.readFailures++;
|
|
251
|
+
const delay = Math.min(this.retryBaseMs * 2 ** (this.readFailures - 1), this.readRetryCapMs);
|
|
252
|
+
log('Strand read incomplete (%d consecutive) - re-reading in %dms', this.readFailures, delay);
|
|
253
|
+
this.watch?.trigger(delay);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* Read and reconcile. Resolves true when the read answered in full: false when it threw,
|
|
258
|
+
* or when the queryable answered part of it from a stand-in.
|
|
259
|
+
*/
|
|
260
|
+
private async poll(): Promise<boolean> {
|
|
261
|
+
if (!this.running) return true;
|
|
166
262
|
|
|
167
263
|
try {
|
|
168
264
|
const currentStrands = await this.queryable.queryStrands();
|
|
@@ -173,17 +269,17 @@ export class StrandWatcher {
|
|
|
173
269
|
if (this.suppressed.has(strand.Id)) continue; // deliberately stopped locally
|
|
174
270
|
if (this.knownStrands.has(strand.Id)) continue;
|
|
175
271
|
const failure = this.failureStates.get(strand.Id);
|
|
176
|
-
// Clock read per candidate, not once per
|
|
272
|
+
// Clock read per candidate, not once per run: an earlier strand's launch
|
|
177
273
|
// may have taken a long time, and this one's gate must reflect that.
|
|
178
274
|
if (failure && this.now() < failure.nextAttemptAt) continue; // backing off after a failed launch
|
|
179
275
|
const decision = this.evaluateFilter(strand);
|
|
180
276
|
if (decision === 'reject') continue;
|
|
181
277
|
log('Strand added: %s', strand.Id);
|
|
182
|
-
// Set BEFORE the await: this doubles as the in-flight guard that stops
|
|
183
|
-
//
|
|
278
|
+
// Set BEFORE the await: this doubles as the in-flight guard that stops a
|
|
279
|
+
// concurrent run (`forcePoll` beside a scheduled one) from launching the strand twice.
|
|
184
280
|
this.knownStrands.set(strand.Id, strand);
|
|
185
281
|
if (decision === 'defer') {
|
|
186
|
-
// Provisional admission: sAppId unknown, re-check on later
|
|
282
|
+
// Provisional admission: sAppId unknown, re-check on later runs.
|
|
187
283
|
this.provisional.add(strand.Id);
|
|
188
284
|
}
|
|
189
285
|
try {
|
|
@@ -215,7 +311,7 @@ export class StrandWatcher {
|
|
|
215
311
|
log('Error handling strand remove for %s: %o', strandId, error);
|
|
216
312
|
}
|
|
217
313
|
}
|
|
218
|
-
// decision === 'defer': still unknown, leave provisional
|
|
314
|
+
// decision === 'defer': still unknown, leave provisional until sAppIdLearned wakes a run.
|
|
219
315
|
}
|
|
220
316
|
|
|
221
317
|
// Find removed strands. A strand whose launch failed is no longer in
|
|
@@ -242,8 +338,8 @@ export class StrandWatcher {
|
|
|
242
338
|
this.failureStates.delete(strandId);
|
|
243
339
|
}
|
|
244
340
|
}
|
|
245
|
-
// NOTE: the clear needs a
|
|
246
|
-
// unpublishes and re-publishes a locally-stopped id
|
|
341
|
+
// NOTE: the clear needs a run that actually SEES the row absent, so a sibling that
|
|
342
|
+
// unpublishes and re-publishes a locally-stopped id between two runs leaves
|
|
247
343
|
// it suppressed for the session. Contrived today — re-seating a removed id is
|
|
248
344
|
// owner-gated and manual — and there is no row version to distinguish the new row
|
|
249
345
|
// from the old. If re-publishing a removed id ever becomes routine, give the row a
|
|
@@ -253,14 +349,17 @@ export class StrandWatcher {
|
|
|
253
349
|
this.suppressed.delete(strandId);
|
|
254
350
|
}
|
|
255
351
|
}
|
|
352
|
+
return !(this.queryable.lastReadPartial?.() ?? false);
|
|
256
353
|
} catch (error) {
|
|
257
354
|
// NOTE: a machine cut off from its party before it ever received the `Strand` block
|
|
258
|
-
// fails this read `cohort-unreachable` on every
|
|
259
|
-
// `control-cohort-edge-carries-data`). Harmless: nothing was ever
|
|
260
|
-
// this machine never held, and this catch keeps the known set. Do
|
|
261
|
-
// empty the way `ControlDatabase.queryRevokedStamps` does for
|
|
262
|
-
// answer here runs the removed-strand loop above. Revisit if
|
|
355
|
+
// fails this read `cohort-unreachable` on every step of the read-retry ladder and logs
|
|
356
|
+
// here (observed in `control-cohort-edge-carries-data`). Harmless: nothing was ever
|
|
357
|
+
// launched from a block this machine never held, and this catch keeps the known set. Do
|
|
358
|
+
// not answer it as empty the way `ControlDatabase.queryRevokedStamps` does for
|
|
359
|
+
// `Revocation` — an empty answer here runs the removed-strand loop above. Revisit if
|
|
360
|
+
// the log volume matters.
|
|
263
361
|
log('Error polling strands: %o', error);
|
|
362
|
+
return false;
|
|
264
363
|
}
|
|
265
364
|
}
|
|
266
365
|
|
|
@@ -274,23 +373,24 @@ export class StrandWatcher {
|
|
|
274
373
|
}
|
|
275
374
|
|
|
276
375
|
log('Starting StrandWatcher');
|
|
376
|
+
// Subscribed before the first read, so a commit landing between them is either seen by
|
|
377
|
+
// that read or wakes a follow-up run.
|
|
378
|
+
this.watch = this.subscribe();
|
|
277
379
|
this.running = true;
|
|
278
|
-
|
|
279
|
-
//
|
|
280
|
-
this.
|
|
281
|
-
void this.poll();
|
|
282
|
-
}, this.pollInterval);
|
|
283
|
-
|
|
284
|
-
// Defer the first poll so start() doesn't block the caller.
|
|
285
|
-
// Strands added via addStrand() typically arrive after start() completes.
|
|
286
|
-
this.initialPollTimer = setTimeout(() => {
|
|
287
|
-
this.initialPollTimer = null;
|
|
288
|
-
void this.poll();
|
|
289
|
-
}, 100);
|
|
380
|
+
// Deferred so start() doesn't block the caller; strands added via addStrand()
|
|
381
|
+
// typically arrive after start() completes.
|
|
382
|
+
this.watch.trigger(INITIAL_RUN_DELAY_MS);
|
|
290
383
|
|
|
291
384
|
log('StrandWatcher started');
|
|
292
385
|
}
|
|
293
386
|
|
|
387
|
+
/** The watch every run goes through: the queryable's change subscription, or a timer-only one. */
|
|
388
|
+
private subscribe(): TableChangeWatch {
|
|
389
|
+
const onChange = (): Promise<void> => this.runOnce();
|
|
390
|
+
const options: TableChangeWatchOptions = { label: 'strand-watcher', backstopMs: this.backstopMs };
|
|
391
|
+
return this.queryable.watchChanges?.(onChange, options) ?? scheduleRuns(onChange, options);
|
|
392
|
+
}
|
|
393
|
+
|
|
294
394
|
/**
|
|
295
395
|
* Stop watching for strand changes
|
|
296
396
|
*/
|
|
@@ -301,27 +401,22 @@ export class StrandWatcher {
|
|
|
301
401
|
|
|
302
402
|
log('Stopping StrandWatcher');
|
|
303
403
|
this.running = false;
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
this.initialPollTimer = null;
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
if (this.pollTimer) {
|
|
311
|
-
clearInterval(this.pollTimer);
|
|
312
|
-
this.pollTimer = null;
|
|
313
|
-
}
|
|
404
|
+
// A run already in flight finishes; `running` keeps it from arming anything further.
|
|
405
|
+
this.watch?.dispose();
|
|
406
|
+
this.watch = null;
|
|
314
407
|
|
|
315
408
|
this.knownStrands.clear();
|
|
316
409
|
this.provisional.clear();
|
|
317
410
|
this.failureStates.clear();
|
|
318
411
|
this.suppressed.clear();
|
|
412
|
+
this.readFailures = 0;
|
|
319
413
|
log('StrandWatcher stopped');
|
|
320
414
|
}
|
|
321
415
|
|
|
322
416
|
/**
|
|
323
|
-
* Forget a strand whose launch failed outside this watcher, so a later
|
|
324
|
-
* re-offers it — gated by the same backoff a watcher-driven failure gets
|
|
417
|
+
* Forget a strand whose launch failed outside this watcher, so a later run
|
|
418
|
+
* re-offers it — gated by the same backoff a watcher-driven failure gets, and woken
|
|
419
|
+
* when that backoff expires.
|
|
325
420
|
*
|
|
326
421
|
* A failed launch normally leaves nothing running (StrandInstanceManager drops the
|
|
327
422
|
* record), which is what makes re-offering it correct. One case does leave something
|
|
@@ -335,6 +430,27 @@ export class StrandWatcher {
|
|
|
335
430
|
this.knownStrands.delete(strandId);
|
|
336
431
|
this.provisional.delete(strandId);
|
|
337
432
|
this.recordFailure(strandId);
|
|
433
|
+
if (this.running) {
|
|
434
|
+
this.scheduleLaunchRetry(this.now());
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
/**
|
|
439
|
+
* Re-read soon: something the queryable merges in beside the watched tables changed
|
|
440
|
+
* (`CadreNode`'s machine-local joined-strand records). A no-op once stopped.
|
|
441
|
+
*/
|
|
442
|
+
refresh(): void {
|
|
443
|
+
this.watch?.trigger();
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
/**
|
|
447
|
+
* The host just learned `strandId`'s sAppId: if the strand was admitted provisionally
|
|
448
|
+
* under an `sAppId` filter, run so the admission resolves now rather than on the backstop.
|
|
449
|
+
*/
|
|
450
|
+
sAppIdLearned(strandId: string): void {
|
|
451
|
+
if (this.provisional.has(strandId)) {
|
|
452
|
+
this.watch?.trigger();
|
|
453
|
+
}
|
|
338
454
|
}
|
|
339
455
|
|
|
340
456
|
/**
|
|
@@ -379,10 +495,13 @@ export class StrandWatcher {
|
|
|
379
495
|
}
|
|
380
496
|
|
|
381
497
|
/**
|
|
382
|
-
*
|
|
498
|
+
* Run once now, outside the watch's schedule (tests, and callers that just wrote a row
|
|
499
|
+
* and want this node to converge before returning). May overlap a scheduled run:
|
|
500
|
+
* `knownStrands` is updated before each callback's await, so neither launches nor
|
|
501
|
+
* removes a strand twice.
|
|
383
502
|
*/
|
|
384
503
|
async forcePoll(): Promise<void> {
|
|
385
|
-
await this.
|
|
504
|
+
await this.runOnce();
|
|
386
505
|
}
|
|
387
506
|
}
|
|
388
507
|
|