@serfab/cadre-core 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/cadre-node.d.ts +32 -19
- package/dist/cadre-node.js +120 -31
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +4 -4
- package/dist/control-formation-recorder.js +6 -6
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +5 -3
- package/dist/index.js.map +1 -1
- package/dist/membership-connection-gater.d.ts +18 -5
- package/dist/membership-connection-gater.js +50 -10
- package/dist/membership-connection-gater.js.map +1 -1
- package/dist/storage-scope.d.ts +50 -12
- package/dist/storage-scope.js +85 -12
- package/dist/storage-scope.js.map +1 -1
- package/dist/strand-formation-manager.js +2 -1
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-id.d.ts +35 -0
- package/dist/strand-id.js +46 -0
- package/dist/strand-id.js.map +1 -0
- package/dist/strand-instance-manager.js +19 -2
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-network-config.d.ts +3 -3
- package/dist/strand-network-config.js +3 -3
- package/dist/strand-solicitation.js +2 -1
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-watcher.d.ts +45 -0
- package/dist/strand-watcher.js +74 -13
- package/dist/strand-watcher.js.map +1 -1
- package/dist/types.d.ts +94 -15
- package/dist/types.js +59 -0
- package/dist/types.js.map +1 -1
- package/package.json +7 -7
- package/src/cadre-node.ts +126 -31
- package/src/control-formation-recorder.ts +6 -6
- package/src/index.ts +563 -554
- package/src/membership-connection-gater.ts +56 -11
- package/src/storage-scope.ts +163 -78
- package/src/strand-formation-manager.ts +647 -646
- package/src/strand-id.ts +48 -0
- package/src/strand-instance-manager.ts +20 -2
- package/src/strand-network-config.ts +250 -250
- package/src/strand-solicitation.ts +500 -499
- package/src/strand-watcher.ts +388 -325
- package/src/types.ts +97 -15
package/src/strand-watcher.ts
CHANGED
|
@@ -1,325 +1,388 @@
|
|
|
1
|
-
import debug from 'debug';
|
|
2
|
-
import type { StrandFilter, StrandRow } from './types.js';
|
|
3
|
-
|
|
4
|
-
const log = debug('sereus:cadre:strand-watcher');
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Outcome of evaluating a strand against the current filter.
|
|
8
|
-
* - `pass`: admit (final)
|
|
9
|
-
* - `reject`: exclude
|
|
10
|
-
* - `defer`: admit provisionally; sAppId not yet known, re-evaluate next poll
|
|
11
|
-
*/
|
|
12
|
-
type FilterDecision = 'pass' | 'reject' | 'defer';
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* Ceiling on the retry delay for a strand whose launch keeps failing. Attempts
|
|
16
|
-
* are never abandoned — the failure this backoff exists for (a transient network
|
|
17
|
-
* or storage fault) can last a long time — but a permanently-unlaunchable strand
|
|
18
|
-
* must not re-attempt on every poll and storm the host app with `strand:error`.
|
|
19
|
-
*/
|
|
20
|
-
const MAX_RETRY_BACKOFF_MS = 5 * 60 * 1000;
|
|
21
|
-
|
|
22
|
-
/** Per-strand consecutive-failure state driving the launch retry backoff. */
|
|
23
|
-
interface StrandFailureState {
|
|
24
|
-
/** Consecutive `onStrandAdded` failures since the last success. */
|
|
25
|
-
failures: number;
|
|
26
|
-
/** Earliest wall-clock time (ms) at which the next attempt may run. */
|
|
27
|
-
nextAttemptAt: number;
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* Extended strand row that includes the sAppId for filtering purposes.
|
|
32
|
-
* The sAppId is provided by the hosting application, not from the control network.
|
|
33
|
-
*/
|
|
34
|
-
export interface StrandRowWithApp extends StrandRow {
|
|
35
|
-
/** sApp ID if known (for filtering) */
|
|
36
|
-
sAppId?: string;
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
/**
|
|
40
|
-
* Callback for strand changes
|
|
41
|
-
*/
|
|
42
|
-
export interface StrandWatcherCallbacks {
|
|
43
|
-
onStrandAdded: (strand: StrandRow) => Promise<void>;
|
|
44
|
-
onStrandRemoved: (strandId: string) => Promise<void>;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
/**
|
|
48
|
-
* Interface for querying strands from control network
|
|
49
|
-
*/
|
|
50
|
-
export interface StrandQueryable {
|
|
51
|
-
queryStrands(): Promise<StrandRow[]>;
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
/**
|
|
55
|
-
* Interface for looking up sAppId for a strand (for filtering)
|
|
56
|
-
*/
|
|
57
|
-
export interface SAppIdLookup {
|
|
58
|
-
/** Get the sAppId for a strand, if known */
|
|
59
|
-
getSAppId(strandId: string): string | undefined;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
/**
|
|
63
|
-
* Watches the control network's Strand table for changes and triggers
|
|
64
|
-
* strand instance start/stop via callbacks.
|
|
65
|
-
*
|
|
66
|
-
* Uses polling until Optimystic supports reactive subscriptions.
|
|
67
|
-
*/
|
|
68
|
-
export class StrandWatcher {
|
|
69
|
-
private readonly filter: StrandFilter;
|
|
70
|
-
private readonly pollInterval: number;
|
|
71
|
-
private readonly callbacks: StrandWatcherCallbacks;
|
|
72
|
-
private readonly queryable: StrandQueryable;
|
|
73
|
-
private readonly sAppIdLookup?: SAppIdLookup;
|
|
74
|
-
/**
|
|
75
|
-
* Wall-clock source for the retry backoff. Read fresh at each use rather than
|
|
76
|
-
* snapshotted per poll: a poll awaits `onStrandAdded`, and a launch that fails
|
|
77
|
-
* slowly (e.g. a dial timeout) would otherwise be scheduled off the clock as it
|
|
78
|
-
* read *before* the attempt, putting `nextAttemptAt` in the past. Injectable so
|
|
79
|
-
* tests can advance time deterministically without fake timers.
|
|
80
|
-
*/
|
|
81
|
-
private readonly now: () => number;
|
|
82
|
-
|
|
83
|
-
private knownStrands: Map<string, StrandRow> = new Map();
|
|
84
|
-
/** Ids admitted under a `defer` decision; re-evaluated each poll until they resolve. */
|
|
85
|
-
private provisional: Set<string> = new Set();
|
|
86
|
-
/** Ids whose last launch attempt threw, with the backoff gating their retry. */
|
|
87
|
-
private failureStates: Map<string, StrandFailureState> = new Map();
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
//
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
this.
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
const
|
|
204
|
-
if (
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
//
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
//
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
//
|
|
248
|
-
//
|
|
249
|
-
//
|
|
250
|
-
//
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
log
|
|
263
|
-
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
this.
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
1
|
+
import debug from 'debug';
|
|
2
|
+
import type { StrandFilter, StrandRow } from './types.js';
|
|
3
|
+
|
|
4
|
+
const log = debug('sereus:cadre:strand-watcher');
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Outcome of evaluating a strand against the current filter.
|
|
8
|
+
* - `pass`: admit (final)
|
|
9
|
+
* - `reject`: exclude
|
|
10
|
+
* - `defer`: admit provisionally; sAppId not yet known, re-evaluate next poll
|
|
11
|
+
*/
|
|
12
|
+
type FilterDecision = 'pass' | 'reject' | 'defer';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Ceiling on the retry delay for a strand whose launch keeps failing. Attempts
|
|
16
|
+
* are never abandoned — the failure this backoff exists for (a transient network
|
|
17
|
+
* or storage fault) can last a long time — but a permanently-unlaunchable strand
|
|
18
|
+
* must not re-attempt on every poll and storm the host app with `strand:error`.
|
|
19
|
+
*/
|
|
20
|
+
const MAX_RETRY_BACKOFF_MS = 5 * 60 * 1000;
|
|
21
|
+
|
|
22
|
+
/** Per-strand consecutive-failure state driving the launch retry backoff. */
|
|
23
|
+
interface StrandFailureState {
|
|
24
|
+
/** Consecutive `onStrandAdded` failures since the last success. */
|
|
25
|
+
failures: number;
|
|
26
|
+
/** Earliest wall-clock time (ms) at which the next attempt may run. */
|
|
27
|
+
nextAttemptAt: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Extended strand row that includes the sAppId for filtering purposes.
|
|
32
|
+
* The sAppId is provided by the hosting application, not from the control network.
|
|
33
|
+
*/
|
|
34
|
+
export interface StrandRowWithApp extends StrandRow {
|
|
35
|
+
/** sApp ID if known (for filtering) */
|
|
36
|
+
sAppId?: string;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Callback for strand changes
|
|
41
|
+
*/
|
|
42
|
+
export interface StrandWatcherCallbacks {
|
|
43
|
+
onStrandAdded: (strand: StrandRow) => Promise<void>;
|
|
44
|
+
onStrandRemoved: (strandId: string) => Promise<void>;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Interface for querying strands from control network
|
|
49
|
+
*/
|
|
50
|
+
export interface StrandQueryable {
|
|
51
|
+
queryStrands(): Promise<StrandRow[]>;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Interface for looking up sAppId for a strand (for filtering)
|
|
56
|
+
*/
|
|
57
|
+
export interface SAppIdLookup {
|
|
58
|
+
/** Get the sAppId for a strand, if known */
|
|
59
|
+
getSAppId(strandId: string): string | undefined;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Watches the control network's Strand table for changes and triggers
|
|
64
|
+
* strand instance start/stop via callbacks.
|
|
65
|
+
*
|
|
66
|
+
* Uses polling until Optimystic supports reactive subscriptions.
|
|
67
|
+
*/
|
|
68
|
+
export class StrandWatcher {
|
|
69
|
+
private readonly filter: StrandFilter;
|
|
70
|
+
private readonly pollInterval: number;
|
|
71
|
+
private readonly callbacks: StrandWatcherCallbacks;
|
|
72
|
+
private readonly queryable: StrandQueryable;
|
|
73
|
+
private readonly sAppIdLookup?: SAppIdLookup;
|
|
74
|
+
/**
|
|
75
|
+
* Wall-clock source for the retry backoff. Read fresh at each use rather than
|
|
76
|
+
* snapshotted per poll: a poll awaits `onStrandAdded`, and a launch that fails
|
|
77
|
+
* slowly (e.g. a dial timeout) would otherwise be scheduled off the clock as it
|
|
78
|
+
* read *before* the attempt, putting `nextAttemptAt` in the past. Injectable so
|
|
79
|
+
* tests can advance time deterministically without fake timers.
|
|
80
|
+
*/
|
|
81
|
+
private readonly now: () => number;
|
|
82
|
+
|
|
83
|
+
private knownStrands: Map<string, StrandRow> = new Map();
|
|
84
|
+
/** Ids admitted under a `defer` decision; re-evaluated each poll until they resolve. */
|
|
85
|
+
private provisional: Set<string> = new Set();
|
|
86
|
+
/** Ids whose last launch attempt threw, with the backoff gating their retry. */
|
|
87
|
+
private failureStates: Map<string, StrandFailureState> = new Map();
|
|
88
|
+
/**
|
|
89
|
+
* Ids a deliberate local stop has withdrawn from offer for the rest of the session
|
|
90
|
+
* (see {@link suppressStrand}). Before {@link forgetStrand} existed, `knownStrands`
|
|
91
|
+
* retention alone made a stop permanent; now that a failed claim can un-know a
|
|
92
|
+
* strand, the permanence has to be recorded explicitly.
|
|
93
|
+
*/
|
|
94
|
+
private suppressed: Set<string> = new Set();
|
|
95
|
+
private pollTimer: ReturnType<typeof setInterval> | null = null;
|
|
96
|
+
private initialPollTimer: ReturnType<typeof setTimeout> | null = null;
|
|
97
|
+
private running = false;
|
|
98
|
+
|
|
99
|
+
constructor(
|
|
100
|
+
queryable: StrandQueryable,
|
|
101
|
+
callbacks: StrandWatcherCallbacks,
|
|
102
|
+
filter: StrandFilter = { mode: 'all' },
|
|
103
|
+
pollInterval: number = 5000,
|
|
104
|
+
sAppIdLookup?: SAppIdLookup,
|
|
105
|
+
now: () => number = Date.now
|
|
106
|
+
) {
|
|
107
|
+
this.queryable = queryable;
|
|
108
|
+
this.callbacks = callbacks;
|
|
109
|
+
this.filter = filter;
|
|
110
|
+
this.pollInterval = pollInterval;
|
|
111
|
+
this.sAppIdLookup = sAppIdLookup;
|
|
112
|
+
this.now = now;
|
|
113
|
+
log('StrandWatcher created with filter: %o, interval: %dms', filter, pollInterval);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Evaluate a strand against the current filter, distinguishing a not-yet-known
|
|
118
|
+
* sAppId (`defer`) from a known non-match (`reject`). A `defer` admission is
|
|
119
|
+
* provisional and re-checked on subsequent polls.
|
|
120
|
+
*/
|
|
121
|
+
private evaluateFilter(strand: StrandRow): FilterDecision {
|
|
122
|
+
switch (this.filter.mode) {
|
|
123
|
+
case 'all':
|
|
124
|
+
return 'pass';
|
|
125
|
+
case 'none':
|
|
126
|
+
return 'reject';
|
|
127
|
+
case 'strandId':
|
|
128
|
+
return strand.Id === this.filter.strandId ? 'pass' : 'reject';
|
|
129
|
+
case 'sAppId': {
|
|
130
|
+
if (!this.sAppIdLookup) {
|
|
131
|
+
// No way to ever decide - admit permanently (no lookup configured).
|
|
132
|
+
return 'pass';
|
|
133
|
+
}
|
|
134
|
+
const sAppId = this.sAppIdLookup.getSAppId(strand.Id);
|
|
135
|
+
if (sAppId === undefined) {
|
|
136
|
+
// sAppId not yet known - admit provisionally and re-evaluate next poll.
|
|
137
|
+
log('sAppId unknown for strand %s - deferring filter decision', strand.Id);
|
|
138
|
+
return 'defer';
|
|
139
|
+
}
|
|
140
|
+
const matches = sAppId === this.filter.sAppId;
|
|
141
|
+
log('sAppId filter: strand %s has sAppId %s, filter wants %s, match=%s',
|
|
142
|
+
strand.Id, sAppId, this.filter.sAppId, matches);
|
|
143
|
+
return matches ? 'pass' : 'reject';
|
|
144
|
+
}
|
|
145
|
+
default:
|
|
146
|
+
return 'pass';
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Record a failed launch attempt and schedule when the strand may be retried:
|
|
152
|
+
* `pollInterval * 2^(failures-1)`, capped at {@link MAX_RETRY_BACKOFF_MS}.
|
|
153
|
+
*/
|
|
154
|
+
private recordFailure(strandId: string): void {
|
|
155
|
+
const failures = (this.failureStates.get(strandId)?.failures ?? 0) + 1;
|
|
156
|
+
const delay = Math.min(this.pollInterval * 2 ** (failures - 1), MAX_RETRY_BACKOFF_MS);
|
|
157
|
+
this.failureStates.set(strandId, { failures, nextAttemptAt: this.now() + delay });
|
|
158
|
+
log('Strand %s launch failed (%d consecutive) - next attempt in %dms', strandId, failures, delay);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Poll for strand changes.
|
|
163
|
+
*/
|
|
164
|
+
private async poll(): Promise<void> {
|
|
165
|
+
if (!this.running) return;
|
|
166
|
+
|
|
167
|
+
try {
|
|
168
|
+
const currentStrands = await this.queryable.queryStrands();
|
|
169
|
+
const currentMap = new Map(currentStrands.map(s => [s.Id, s]));
|
|
170
|
+
|
|
171
|
+
// Find added strands
|
|
172
|
+
for (const strand of currentStrands) {
|
|
173
|
+
if (this.suppressed.has(strand.Id)) continue; // deliberately stopped locally
|
|
174
|
+
if (this.knownStrands.has(strand.Id)) continue;
|
|
175
|
+
const failure = this.failureStates.get(strand.Id);
|
|
176
|
+
// Clock read per candidate, not once per poll: an earlier strand's launch
|
|
177
|
+
// may have taken a long time, and this one's gate must reflect that.
|
|
178
|
+
if (failure && this.now() < failure.nextAttemptAt) continue; // backing off after a failed launch
|
|
179
|
+
const decision = this.evaluateFilter(strand);
|
|
180
|
+
if (decision === 'reject') continue;
|
|
181
|
+
log('Strand added: %s', strand.Id);
|
|
182
|
+
// Set BEFORE the await: this doubles as the in-flight guard that stops the
|
|
183
|
+
// interval timer from starting a second concurrent launch for the same strand.
|
|
184
|
+
this.knownStrands.set(strand.Id, strand);
|
|
185
|
+
if (decision === 'defer') {
|
|
186
|
+
// Provisional admission: sAppId unknown, re-check on later polls.
|
|
187
|
+
this.provisional.add(strand.Id);
|
|
188
|
+
}
|
|
189
|
+
try {
|
|
190
|
+
await this.callbacks.onStrandAdded(strand);
|
|
191
|
+
this.failureStates.delete(strand.Id);
|
|
192
|
+
} catch (error) {
|
|
193
|
+
log('Error handling strand add for %s: %o', strand.Id, error);
|
|
194
|
+
this.forgetStrand(strand.Id);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// Re-evaluate provisional admissions whose sAppId may now be resolvable.
|
|
199
|
+
// Snapshot ids so we can mutate provisional/knownStrands inside the loop.
|
|
200
|
+
for (const strandId of [...this.provisional]) {
|
|
201
|
+
const strand = currentMap.get(strandId);
|
|
202
|
+
if (!strand) continue; // handled by the removed-strand loop below
|
|
203
|
+
const decision = this.evaluateFilter(strand);
|
|
204
|
+
if (decision === 'pass') {
|
|
205
|
+
// Resolved to a match - admission is now final.
|
|
206
|
+
this.provisional.delete(strandId);
|
|
207
|
+
} else if (decision === 'reject') {
|
|
208
|
+
// Resolved to a non-match - stop the provisionally-admitted strand.
|
|
209
|
+
log('Provisional strand rejected on re-evaluation: %s', strandId);
|
|
210
|
+
this.knownStrands.delete(strandId);
|
|
211
|
+
this.provisional.delete(strandId);
|
|
212
|
+
try {
|
|
213
|
+
await this.callbacks.onStrandRemoved(strandId);
|
|
214
|
+
} catch (error) {
|
|
215
|
+
log('Error handling strand remove for %s: %o', strandId, error);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
// decision === 'defer': still unknown, leave provisional for next poll.
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// Find removed strands. A strand whose launch failed is no longer in
|
|
222
|
+
// knownStrands, so it correctly never fires onStrandRemoved — nothing was
|
|
223
|
+
// ever started for it.
|
|
224
|
+
for (const [strandId] of this.knownStrands) {
|
|
225
|
+
if (!currentMap.has(strandId)) {
|
|
226
|
+
log('Strand removed: %s', strandId);
|
|
227
|
+
this.knownStrands.delete(strandId);
|
|
228
|
+
this.provisional.delete(strandId);
|
|
229
|
+
try {
|
|
230
|
+
await this.callbacks.onStrandRemoved(strandId);
|
|
231
|
+
} catch (error) {
|
|
232
|
+
log('Error handling strand remove for %s: %o', strandId, error);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// Drop backoff and suppression state for strands whose control-network row is
|
|
238
|
+
// gone; a row that reappears is a fresh strand and gets a fresh first attempt,
|
|
239
|
+
// un-backed-off and un-suppressed.
|
|
240
|
+
for (const strandId of [...this.failureStates.keys()]) {
|
|
241
|
+
if (!currentMap.has(strandId)) {
|
|
242
|
+
this.failureStates.delete(strandId);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
// NOTE: the clear needs a poll that actually SEES the row absent, so a sibling that
|
|
246
|
+
// unpublishes and re-publishes a locally-stopped id inside one poll interval leaves
|
|
247
|
+
// it suppressed for the session. Contrived today — re-seating a removed id is
|
|
248
|
+
// owner-gated and manual — and there is no row version to distinguish the new row
|
|
249
|
+
// from the old. If re-publishing a removed id ever becomes routine, give the row a
|
|
250
|
+
// generation column and clear suppression on a change rather than on absence.
|
|
251
|
+
for (const strandId of [...this.suppressed]) {
|
|
252
|
+
if (!currentMap.has(strandId)) {
|
|
253
|
+
this.suppressed.delete(strandId);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
} catch (error) {
|
|
257
|
+
// NOTE: a machine cut off from its party before it ever received the `Strand` block
|
|
258
|
+
// fails this read `cohort-unreachable` on every poll and logs here (observed in
|
|
259
|
+
// `control-cohort-edge-carries-data`). Harmless: nothing was ever launched from a block
|
|
260
|
+
// this machine never held, and this catch keeps the known set. Do not answer it as
|
|
261
|
+
// empty the way `ControlDatabase.queryRevokedStamps` does for `Revocation` — an empty
|
|
262
|
+
// answer here runs the removed-strand loop above. Revisit if the log volume matters.
|
|
263
|
+
log('Error polling strands: %o', error);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* Start watching for strand changes
|
|
269
|
+
*/
|
|
270
|
+
async start(): Promise<void> {
|
|
271
|
+
if (this.running) {
|
|
272
|
+
log('StrandWatcher already running');
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
log('Starting StrandWatcher');
|
|
277
|
+
this.running = true;
|
|
278
|
+
|
|
279
|
+
// Set up periodic polling
|
|
280
|
+
this.pollTimer = setInterval(() => {
|
|
281
|
+
void this.poll();
|
|
282
|
+
}, this.pollInterval);
|
|
283
|
+
|
|
284
|
+
// Defer the first poll so start() doesn't block the caller.
|
|
285
|
+
// Strands added via addStrand() typically arrive after start() completes.
|
|
286
|
+
this.initialPollTimer = setTimeout(() => {
|
|
287
|
+
this.initialPollTimer = null;
|
|
288
|
+
void this.poll();
|
|
289
|
+
}, 100);
|
|
290
|
+
|
|
291
|
+
log('StrandWatcher started');
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/**
|
|
295
|
+
* Stop watching for strand changes
|
|
296
|
+
*/
|
|
297
|
+
async stop(): Promise<void> {
|
|
298
|
+
if (!this.running) {
|
|
299
|
+
return;
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
log('Stopping StrandWatcher');
|
|
303
|
+
this.running = false;
|
|
304
|
+
|
|
305
|
+
if (this.initialPollTimer) {
|
|
306
|
+
clearTimeout(this.initialPollTimer);
|
|
307
|
+
this.initialPollTimer = null;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
if (this.pollTimer) {
|
|
311
|
+
clearInterval(this.pollTimer);
|
|
312
|
+
this.pollTimer = null;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
this.knownStrands.clear();
|
|
316
|
+
this.provisional.clear();
|
|
317
|
+
this.failureStates.clear();
|
|
318
|
+
this.suppressed.clear();
|
|
319
|
+
log('StrandWatcher stopped');
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* Forget a strand whose launch failed outside this watcher, so a later poll
|
|
324
|
+
* re-offers it — gated by the same backoff a watcher-driven failure gets.
|
|
325
|
+
*
|
|
326
|
+
* A failed launch normally leaves nothing running (StrandInstanceManager drops the
|
|
327
|
+
* record), which is what makes re-offering it correct. One case does leave something
|
|
328
|
+
* running: a row this machine published lands on an instance something else already
|
|
329
|
+
* attached, and honouring the founder request on it (CadreNode.launchStrand →
|
|
330
|
+
* StrandInstanceManager.foundExistingStrand) throws. The instance stays up as a joiner
|
|
331
|
+
* and the retry re-attempts the bootstrap on it, which is what should happen — but do
|
|
332
|
+
* not read the line above as "nothing is running".
|
|
333
|
+
*/
|
|
334
|
+
forgetStrand(strandId: string): void {
|
|
335
|
+
this.knownStrands.delete(strandId);
|
|
336
|
+
this.provisional.delete(strandId);
|
|
337
|
+
this.recordFailure(strandId);
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* Never offer this strand again this session. Two callers, both meaning "the retry
|
|
342
|
+
* ladder cannot help here": a deliberate local stop, and a launch that can never
|
|
343
|
+
* succeed — a strand whose id is unusable as a storage scope key
|
|
344
|
+
* (`CadreNode.handleStrandAdded`), which every later attempt would reject identically.
|
|
345
|
+
* A launch that merely FAILED is not one of them; that goes to {@link forgetStrand}.
|
|
346
|
+
*
|
|
347
|
+
* Cleared when the strand's control row disappears, because a row that reappears is a
|
|
348
|
+
* strand the party re-published and the stop said nothing about it; also cleared by
|
|
349
|
+
* {@link stop}, since sApp configs do not survive it either, and by
|
|
350
|
+
* {@link unsuppressStrand} when the caller claims the strand again.
|
|
351
|
+
*/
|
|
352
|
+
suppressStrand(strandId: string): void {
|
|
353
|
+
log('Suppressing strand %s — deliberate local stop, will not be re-offered', strandId);
|
|
354
|
+
this.suppressed.add(strandId);
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
/**
|
|
358
|
+
* Revoke a {@link suppressStrand}: a deliberate local claim overrides the deliberate
|
|
359
|
+
* local stop that preceded it.
|
|
360
|
+
*
|
|
361
|
+
* Not cosmetic — the suppression check runs before the `knownStrands` one, so a
|
|
362
|
+
* suppressed id is never re-recorded there, and the removed-strand loop iterates
|
|
363
|
+
* `knownStrands`. A strand re-claimed while still suppressed would therefore run with
|
|
364
|
+
* the watcher blind to it: a party-wide removal would never stop it locally. That is
|
|
365
|
+
* reachable whenever the stop found the id already un-known — after a claim that
|
|
366
|
+
* failed, or for a strand the filter never admitted.
|
|
367
|
+
*/
|
|
368
|
+
unsuppressStrand(strandId: string): void {
|
|
369
|
+
if (this.suppressed.delete(strandId)) {
|
|
370
|
+
log('Strand %s re-claimed — suppression lifted', strandId);
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/**
|
|
375
|
+
* Get currently known strands
|
|
376
|
+
*/
|
|
377
|
+
getKnownStrands(): Map<string, StrandRow> {
|
|
378
|
+
return new Map(this.knownStrands);
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* Force an immediate poll (useful for testing)
|
|
383
|
+
*/
|
|
384
|
+
async forcePoll(): Promise<void> {
|
|
385
|
+
await this.poll();
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
|