@mmmbuto/nexuscrew 0.9.57 → 0.9.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +32 -8
- package/frontend/dist/assets/index-CZFJnks_.js +169 -0
- package/frontend/dist/index.html +1 -1
- package/frontend/dist/version.json +1 -1
- package/lib/fleet/managed.js +57 -0
- package/lib/nodes/access-presets.js +1 -0
- package/lib/nodes/ask-slot-verifier.js +72 -0
- package/lib/nodes/commands.js +7 -3
- package/lib/nodes/reverse-slot-proof.js +19 -7
- package/lib/nodes/store.js +54 -6
- package/lib/notify/ask-answer-service.js +23 -11
- package/lib/notify/ask-receipts.js +1 -0
- package/lib/notify/ask-relay.js +261 -74
- package/lib/notify/asks.js +117 -9
- package/lib/notify/event-feed-asks-routes.js +79 -19
- package/lib/notify/event-feed-client.js +130 -50
- package/lib/notify/event-feed-producers.js +1 -0
- package/lib/notify/imported-dismissals.js +106 -0
- package/lib/notify/routes.js +39 -20
- package/lib/proxy/federation.js +112 -12
- package/lib/proxy/resource-acl.js +1 -0
- package/lib/server.js +161 -23
- package/lib/tmux/actions.js +14 -11
- package/package.json +1 -1
- package/frontend/dist/assets/index-CAgxp3qK.js +0 -169
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
const BACKOFF_MIN_MS = 5000;
|
|
15
15
|
const BACKOFF_MAX_MS = 60000;
|
|
16
16
|
const IDLE_TIMEOUT_MS = 60000;
|
|
17
|
+
const ERROR_BODY_TIMEOUT_MS = 5000;
|
|
17
18
|
const MAX_CONSECUTIVE_RESETS = 3;
|
|
18
19
|
// `resync-exhausted` ferma il loop ma non deve parcheggiare la view
|
|
19
20
|
// per sempre. Al tappo la view prende un cooldown a gradini — 30 s → 2 min →
|
|
@@ -68,6 +69,12 @@ function contentTypeOf(r) {
|
|
|
68
69
|
}
|
|
69
70
|
}
|
|
70
71
|
|
|
72
|
+
function isNamedFeedReason(reason) {
|
|
73
|
+
return ['shape', 'owner-mismatch', 'snapshot-oversize', 'snapshot-element-oversize',
|
|
74
|
+
'snapshot-resync-required', 'snapshot-content-type'].includes(reason)
|
|
75
|
+
|| (typeof reason === 'string' && reason.startsWith('snapshot-schema:'));
|
|
76
|
+
}
|
|
77
|
+
|
|
71
78
|
function createEventFeedClient(opts = {}) {
|
|
72
79
|
// deps: { nodesPath, token(), eventsHub, now, fetchImpl, log }
|
|
73
80
|
const fetchImpl = opts.fetchImpl || fetch;
|
|
@@ -81,14 +88,47 @@ function createEventFeedClient(opts = {}) {
|
|
|
81
88
|
: MIN_SNAPSHOT_INTERVAL_MS;
|
|
82
89
|
// ownerId -> view state {cursor, generation, asks, notifications, fleetState,
|
|
83
90
|
// stale, lastError}; generation guards against obsolete fetches.
|
|
91
|
+
const isAskDismissed = typeof opts.isAskDismissed === 'function' ? opts.isAskDismissed : () => false;
|
|
92
|
+
const visibleAsks = (ownerId, asks) => asks.filter(ask => !isAskDismissed(ownerId, ask));
|
|
84
93
|
const views = new Map();
|
|
85
94
|
const running = new Map(); // nodeId -> {abort}
|
|
86
95
|
let stopped = false;
|
|
87
96
|
let generation = 0;
|
|
88
97
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
98
|
+
const retries = new Map(); // ownerId -> persistent retry state
|
|
99
|
+
const configurations = new Map();
|
|
100
|
+
function retryFor(ownerId) {
|
|
101
|
+
if (!retries.has(ownerId)) retries.set(ownerId, { fails: 0, nextRetryAt: 0, timer: null });
|
|
102
|
+
return retries.get(ownerId);
|
|
103
|
+
}
|
|
104
|
+
function clearRetry(ownerId, reset = false) {
|
|
105
|
+
const retry = retries.get(ownerId);
|
|
106
|
+
if (!retry) return;
|
|
107
|
+
if (retry.timer) clearTimeout(retry.timer);
|
|
108
|
+
retry.timer = null;
|
|
109
|
+
if (reset) { retry.fails = 0; retry.nextRetryAt = 0; }
|
|
110
|
+
}
|
|
111
|
+
function active(ownerId, slot) {
|
|
112
|
+
return !stopped && running.get(ownerId) === slot && !slot.controller.signal.aborted;
|
|
113
|
+
}
|
|
114
|
+
async function withDeadline(slot, milliseconds, operation) {
|
|
115
|
+
const controller = slot ? slot.controller : new AbortController();
|
|
116
|
+
const signal = controller.signal;
|
|
117
|
+
let rejectAbort;
|
|
118
|
+
const aborted = new Promise((_resolve, reject) => { rejectAbort = () => reject(new Error('acquisition aborted')); });
|
|
119
|
+
signal.addEventListener('abort', rejectAbort, { once: true });
|
|
120
|
+
const timer = setTimeout(() => controller.abort(), milliseconds);
|
|
121
|
+
try {
|
|
122
|
+
if (signal.aborted) throw new Error('acquisition aborted');
|
|
123
|
+
return await Promise.race([operation(signal), aborted]);
|
|
124
|
+
} finally {
|
|
125
|
+
clearTimeout(timer);
|
|
126
|
+
signal.removeEventListener('abort', rejectAbort);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
function backoffMs(ownerId) {
|
|
130
|
+
const retry = retryFor(ownerId);
|
|
131
|
+
const fails = ++retry.fails;
|
|
92
132
|
const base = Math.min(BACKOFF_MAX_MS, BACKOFF_MIN_MS * Math.pow(2, fails - 1));
|
|
93
133
|
return Math.floor(base / 2 + Math.random() * (base / 2));
|
|
94
134
|
}
|
|
@@ -187,7 +227,7 @@ function createEventFeedClient(opts = {}) {
|
|
|
187
227
|
// ask: the tombstone is what keeps it from coming back.
|
|
188
228
|
view.asks = Array.isArray(snap.asks)
|
|
189
229
|
? snap.asks
|
|
190
|
-
.filter((a) => a && !askIsClosed(ownerId, a.ownerAskId || a.id, snap.viewEpoch))
|
|
230
|
+
.filter((a) => a && !askIsClosed(ownerId, a.ownerAskId || a.id, snap.viewEpoch) && !isAskDismissed(ownerId, a))
|
|
191
231
|
.map((a) => ({ ...a, ownerId }))
|
|
192
232
|
: [];
|
|
193
233
|
view.notifications = Array.isArray(snap.notifications) ? snap.notifications.map((e) => ({ ...e, ownerId })) : [];
|
|
@@ -226,15 +266,19 @@ function createEventFeedClient(opts = {}) {
|
|
|
226
266
|
eventId: envelope.eventId, ts: Date.now(),
|
|
227
267
|
});
|
|
228
268
|
} else if (f.type === 'ask') {
|
|
269
|
+
const ask = { id: f.askId, question: f.question, options: f.options, session: f.session, ts: f.ts,
|
|
270
|
+
ownerAskTs: Number.isSafeInteger(f.askTs) && f.askTs > 0 ? f.askTs : null, ownerId: envelope.ownerId };
|
|
271
|
+
if (isAskDismissed(envelope.ownerId, ask)) return false;
|
|
229
272
|
opts.eventsHub.broadcast({ type: 'ask', ownerId: envelope.ownerId, eventId: envelope.eventId,
|
|
230
|
-
ask
|
|
273
|
+
ask });
|
|
231
274
|
} else if (f.type === 'ask-closed') {
|
|
275
|
+
if (typeof opts.onAskClosed === 'function') opts.onAskClosed(envelope.ownerId, f.askId, f.outcome);
|
|
232
276
|
// The owner says the ask is closed: the VIEW drops it too, not only the
|
|
233
277
|
// UI, or the next state read hands the card back.
|
|
234
278
|
const closedView = dropAskFromView(envelope.ownerId, f.askId);
|
|
235
279
|
if (f.outcome === 'dismissed') noteAskClosed(envelope.ownerId, f.askId, closedView.viewEpoch);
|
|
236
280
|
opts.eventsHub.broadcast({ type: f.outcome === 'dismissed' ? 'ask-dismissed' : 'ask-answered',
|
|
237
|
-
id: f.askId, ownerId: envelope.ownerId, eventId: envelope.eventId });
|
|
281
|
+
id: f.askId, ownerAskId: f.askId, ownerId: envelope.ownerId, eventId: envelope.eventId });
|
|
238
282
|
} else if (f.type === 'file-notice') {
|
|
239
283
|
// Stessa forma della consegna locale (lib/files/routes.js): il nome del
|
|
240
284
|
// file e' SEMPRE nel corpo, con la caption quando c'e'. Prima l'importato
|
|
@@ -296,7 +340,7 @@ function createEventFeedClient(opts = {}) {
|
|
|
296
340
|
return Math.max(oldest + INGRESS_WINDOW_MS, t) + INGRESS_BLOCK_BACKOFF_MS;
|
|
297
341
|
}
|
|
298
342
|
|
|
299
|
-
async function streamOnce(peer, ownerId, store) {
|
|
343
|
+
async function streamOnce(peer, ownerId, store, slot) {
|
|
300
344
|
const view = viewFor(ownerId);
|
|
301
345
|
// The deadline expired: the peer is admitted again and the block is closed
|
|
302
346
|
// by the transition that reopens the stream, not by a silent timeout.
|
|
@@ -305,15 +349,16 @@ function createEventFeedClient(opts = {}) {
|
|
|
305
349
|
view.ingressBlockReason = null;
|
|
306
350
|
}
|
|
307
351
|
const after = view.cursor ? `?after=${encodeURIComponent(view.cursor)}` : '';
|
|
308
|
-
const ctrl =
|
|
309
|
-
running.set(ownerId, { abort: () => ctrl.abort() });
|
|
310
|
-
let timer = setTimeout(() => ctrl.abort(), IDLE_TIMEOUT_MS); // 60 s without bytes
|
|
352
|
+
const ctrl = slot.controller;
|
|
311
353
|
try {
|
|
312
|
-
const r = await fetchImpl(routeUrl(peer, '/event-feed', after), {
|
|
313
|
-
headers: identityHeaders(peer, store), signal
|
|
314
|
-
});
|
|
315
|
-
if (
|
|
316
|
-
|
|
354
|
+
const r = await withDeadline(slot, IDLE_TIMEOUT_MS, (signal) => fetchImpl(routeUrl(peer, '/event-feed', after), {
|
|
355
|
+
headers: identityHeaders(peer, store), signal,
|
|
356
|
+
}));
|
|
357
|
+
if (!active(ownerId, slot)) return;
|
|
358
|
+
if (r.status === 403) {
|
|
359
|
+
const body = await withDeadline(slot, ERROR_BODY_TIMEOUT_MS, () => r.json().catch(() => ({})));
|
|
360
|
+
if (!active(ownerId, slot)) return;
|
|
361
|
+
if (body.reason === 'events-disabled') { view.stale = true; return; }
|
|
317
362
|
}
|
|
318
363
|
if (r.status === 409) { // reset-required / cursor-future: resync from a fresh snapshot
|
|
319
364
|
view.cursor = null;
|
|
@@ -321,17 +366,15 @@ function createEventFeedClient(opts = {}) {
|
|
|
321
366
|
return;
|
|
322
367
|
}
|
|
323
368
|
if (!r.ok) { view.stale = true; throw new Error(`event-feed HTTP ${r.status}`); }
|
|
324
|
-
|
|
369
|
+
if (!active(ownerId, slot)) return;
|
|
325
370
|
const reader = r.body.getReader();
|
|
326
371
|
const dec = new TextDecoder();
|
|
327
372
|
let buffer = '';
|
|
328
373
|
for (;;) {
|
|
329
|
-
const { done, value } = await reader.read();
|
|
330
|
-
if (
|
|
331
|
-
|
|
332
|
-
//
|
|
333
|
-
clearTimeout(timer);
|
|
334
|
-
timer = setTimeout(() => ctrl.abort(), IDLE_TIMEOUT_MS);
|
|
374
|
+
const { done, value } = await withDeadline(slot, IDLE_TIMEOUT_MS, () => reader.read());
|
|
375
|
+
if (!active(ownerId, slot)) return;
|
|
376
|
+
if (done) throw new Error('event-feed EOF');
|
|
377
|
+
// Each read owns one deadline; no previous timeout survives the chunk.
|
|
335
378
|
const chunk = dec.decode(value, { stream: true });
|
|
336
379
|
buffer += chunk;
|
|
337
380
|
let idx;
|
|
@@ -376,35 +419,42 @@ function createEventFeedClient(opts = {}) {
|
|
|
376
419
|
view.consecutiveResets = 0; // a usable frame: the streak is over
|
|
377
420
|
view.resyncExhaustions = 0; // and a working stream: the cooldown ladder back to the first step
|
|
378
421
|
view.cursor = `${epochStr}:${sStr}`;
|
|
422
|
+
clearRetry(ownerId, true);
|
|
379
423
|
reemit(envelope);
|
|
380
424
|
}
|
|
381
425
|
}
|
|
382
426
|
} finally {
|
|
383
|
-
|
|
384
|
-
|
|
427
|
+
// The acquisition slot owns the connection throughout its lifecycle.
|
|
428
|
+
ctrl.abort();
|
|
385
429
|
}
|
|
386
430
|
}
|
|
387
431
|
|
|
388
432
|
// Capability BEFORE subscribing: an owner without the feed capability makes
|
|
389
433
|
// the view unsupported — no fallback, no infinite backoff.
|
|
390
|
-
async function checkCapability(peer, ownerId) {
|
|
434
|
+
async function checkCapability(peer, ownerId, slot) {
|
|
391
435
|
const view = viewFor(ownerId);
|
|
392
436
|
if (view.capabilityChecked) return true;
|
|
393
437
|
// The health endpoint of the OWNER is probed directly (same pattern as
|
|
394
438
|
// the tunnel health probe): /federation/health with the peer's token.
|
|
395
|
-
const
|
|
396
|
-
|
|
439
|
+
const body = await withDeadline(slot, 5000, async (signal) => {
|
|
440
|
+
const r = await fetchImpl(`http://127.0.0.1:${peer.localPort}/federation/health`, {
|
|
441
|
+
headers: { authorization: `Bearer ${peer.token}` }, signal,
|
|
442
|
+
});
|
|
443
|
+
if (!r.ok) throw new Error(`health HTTP ${r.status}`);
|
|
444
|
+
return r.json();
|
|
397
445
|
});
|
|
398
|
-
if (!
|
|
399
|
-
const body = await r.json();
|
|
446
|
+
if (!active(ownerId, slot)) throw new Error('obsolete acquisition');
|
|
400
447
|
if (body.eventFeedV1 !== true || body.instanceId !== ownerId) {
|
|
401
448
|
view.unsupported = true;
|
|
402
449
|
view.lastError = 'event-feed-unsupported';
|
|
450
|
+
view.capabilityRecheckAt = now() + BACKOFF_MAX_MS;
|
|
403
451
|
// One transition log, not one per tick: the poll gate keeps this
|
|
404
452
|
// owner out from here on.
|
|
405
453
|
log(`event-feed-client: ${ownerId.slice(0, 8)}… no feed capability, subscription refused`);
|
|
406
454
|
return false;
|
|
407
455
|
}
|
|
456
|
+
view.unsupported = false;
|
|
457
|
+
view.capabilityRecheckAt = null;
|
|
408
458
|
view.capabilityChecked = true;
|
|
409
459
|
return true;
|
|
410
460
|
}
|
|
@@ -518,7 +568,7 @@ function createEventFeedClient(opts = {}) {
|
|
|
518
568
|
// vista resta ferma e STALE con la sua causa — ma l'elenco asks torna
|
|
519
569
|
// comunque alla porta, che è autorizzata a decidere: un campo che la
|
|
520
570
|
// chiusura non legge non può tenere aperta una card chiusa dall'owner.
|
|
521
|
-
async function snapshotOnce(peer, ownerId, store, { profile = 'view' } = {}) {
|
|
571
|
+
async function snapshotOnce(peer, ownerId, store, { profile = 'view', slot = null } = {}) {
|
|
522
572
|
const view = viewFor(ownerId);
|
|
523
573
|
// INVARIANTE (punto d'ingresso, per owner): al massimo uno snapshot per
|
|
524
574
|
// finestra, da qualunque causa di resync. Se la finestra non è trascorsa
|
|
@@ -538,14 +588,14 @@ function createEventFeedClient(opts = {}) {
|
|
|
538
588
|
view.lastSnapshotAt = now();
|
|
539
589
|
view.nextSnapshotAllowedAt = null;
|
|
540
590
|
const gen = generation;
|
|
541
|
-
const r = await
|
|
542
|
-
|
|
591
|
+
const { r, text } = await withDeadline(slot, 8000, async (signal) => {
|
|
592
|
+
const r = await fetchImpl(routeUrl(peer, '/event-feed/snapshot'), {
|
|
593
|
+
headers: identityHeaders(peer, store), signal,
|
|
594
|
+
});
|
|
595
|
+
if (!r.ok) throw new Error(`snapshot HTTP ${r.status}`);
|
|
596
|
+
return { r, text: await r.text() };
|
|
543
597
|
});
|
|
544
|
-
if (!
|
|
545
|
-
// Size cap BEFORE JSON.parse: a body over the server budget is refused on
|
|
546
|
-
// sight, never parsed. La regola vive nel validatore unico; qui è la
|
|
547
|
-
// scorciatoia che evita di pagare il parse di un corpo già rifiutato.
|
|
548
|
-
const text = await r.text();
|
|
598
|
+
if ((slot && !active(ownerId, slot)) || gen !== generation) throw new Error('obsolete acquisition');
|
|
549
599
|
const bytes = Buffer.byteLength(text, 'utf8');
|
|
550
600
|
if (bytes > 3 * 1024 * 1024) {
|
|
551
601
|
view.stale = true; view.lastError = 'snapshot-oversize';
|
|
@@ -576,6 +626,7 @@ function createEventFeedClient(opts = {}) {
|
|
|
576
626
|
return { state: 'failed' };
|
|
577
627
|
}
|
|
578
628
|
applySnapshot(ownerId, snap, gen);
|
|
629
|
+
clearRetry(ownerId, true);
|
|
579
630
|
return { state: 'applied', asks: snap.asks, appliedToView: true };
|
|
580
631
|
}
|
|
581
632
|
|
|
@@ -586,15 +637,31 @@ function createEventFeedClient(opts = {}) {
|
|
|
586
637
|
if (!store) return;
|
|
587
638
|
const peers = (store.nodes || []).filter((n) => n && n.direction === 'outbound'
|
|
588
639
|
&& n.eventsReceive === true && n.token && n.localPort && n.nodeId);
|
|
640
|
+
const eligible = new Map(peers.map((peer) => [peer.nodeId, peer]));
|
|
641
|
+
const fingerprint = (peer) => JSON.stringify([peer.token, peer.localPort, peer.nodeId, peer.accessGrants, peer.accessPreset]);
|
|
642
|
+
for (const [ownerId, config] of configurations) {
|
|
643
|
+
const peer = eligible.get(ownerId);
|
|
644
|
+
if (!peer || fingerprint(peer) !== config) {
|
|
645
|
+
const slot = running.get(ownerId);
|
|
646
|
+
if (slot) { running.delete(ownerId); slot.abort(); }
|
|
647
|
+
clearRetry(ownerId, true);
|
|
648
|
+
configurations.delete(ownerId);
|
|
649
|
+
const view = views.get(ownerId);
|
|
650
|
+
if (view) { view.cursor = null; view.stale = true; view.capabilityChecked = false; view.unsupported = false; }
|
|
651
|
+
}
|
|
652
|
+
}
|
|
589
653
|
await Promise.all(peers.map(async (peer) => {
|
|
590
654
|
const ownerId = peer.nodeId;
|
|
655
|
+
configurations.set(ownerId, fingerprint(peer));
|
|
656
|
+
if (now() < retryFor(ownerId).nextRetryAt) return;
|
|
591
657
|
if (running.has(ownerId)) return; // one loop per pair
|
|
592
658
|
// Il guard vale per TUTTO il round, non solo per lo stream: due poll
|
|
593
659
|
// sovrapposti (tick + backoff, o macchina carica) facevano due snapshot
|
|
594
660
|
// nella stessa finestra e il secondo, senza il flag di recupero,
|
|
595
|
-
// saltava il riarma del cooldown.
|
|
596
|
-
//
|
|
597
|
-
const
|
|
661
|
+
// saltava il riarma del cooldown. Lo stesso controller resta attivo
|
|
662
|
+
// dalla prima acquisizione fino alla chiusura dello stream.
|
|
663
|
+
const controller = new AbortController();
|
|
664
|
+
const slot = { controller, abort: () => controller.abort() };
|
|
598
665
|
running.set(ownerId, slot);
|
|
599
666
|
const view = viewFor(ownerId);
|
|
600
667
|
try {
|
|
@@ -619,7 +686,11 @@ function createEventFeedClient(opts = {}) {
|
|
|
619
686
|
recovery = true;
|
|
620
687
|
log(`event-feed-client: ${ownerId.slice(0, 8)}… resync cooldown expired: one fresh snapshot attempt`);
|
|
621
688
|
}
|
|
622
|
-
if (view.unsupported
|
|
689
|
+
if (view.unsupported) {
|
|
690
|
+
if (now() < view.capabilityRecheckAt) return;
|
|
691
|
+
view.capabilityChecked = false;
|
|
692
|
+
}
|
|
693
|
+
if ((view.ingressBlockedUntil || 0) > Date.now()) return;
|
|
623
694
|
// La fase di ACQUISIZIONE del tentativo (capability + snapshot) ha UN
|
|
624
695
|
// SOLO punto di uscita per gli esiti non riusciti: throw, timeout,
|
|
625
696
|
// abort e 'failed' convergono tutti lì — non un ramo per tipo di
|
|
@@ -635,15 +706,16 @@ function createEventFeedClient(opts = {}) {
|
|
|
635
706
|
let snapState = null; // null = tentativo non fatto (cursor già valido)
|
|
636
707
|
try {
|
|
637
708
|
if (!view.capabilityChecked) {
|
|
638
|
-
const capOk = await checkCapability(peer, ownerId);
|
|
709
|
+
const capOk = await checkCapability(peer, ownerId, slot);
|
|
639
710
|
if (!capOk) return; // senza la capability il gate blocca: nessun tentativo fatto
|
|
640
711
|
}
|
|
641
712
|
if (!view.cursor) {
|
|
642
|
-
snapState = await snapshotOnce(peer, ownerId, store); // profilo view
|
|
713
|
+
snapState = await snapshotOnce(peer, ownerId, store, { slot }); // profilo view
|
|
643
714
|
}
|
|
644
715
|
} catch (e) {
|
|
645
716
|
attemptError = e;
|
|
646
717
|
}
|
|
718
|
+
if (stopped || running.get(ownerId) !== slot) return;
|
|
647
719
|
if (attemptError || (snapState && snapState.state === 'failed')) {
|
|
648
720
|
if (recovery) armResyncCooldown(view, attemptError ? 'recovery attempt failed' : 'recovery snapshot failed');
|
|
649
721
|
}
|
|
@@ -651,21 +723,27 @@ function createEventFeedClient(opts = {}) {
|
|
|
651
723
|
// Un tentativo fallito o posticipato ferma il round; NESSUN tentativo
|
|
652
724
|
// (cursor già valido) va dritto allo stream.
|
|
653
725
|
if (snapState && snapState.state !== 'applied') return;
|
|
654
|
-
await streamOnce(peer, ownerId, store);
|
|
726
|
+
await streamOnce(peer, ownerId, store, slot);
|
|
655
727
|
} catch (e) {
|
|
728
|
+
if (stopped || running.get(ownerId) !== slot) return;
|
|
729
|
+
view.cursor = null;
|
|
656
730
|
view.stale = true;
|
|
657
731
|
// A named feed reason (e.g. owner-mismatch) survives a later transport
|
|
658
732
|
// error: it is the reason the operator needs to see.
|
|
659
|
-
if (view.lastError
|
|
733
|
+
if (!isNamedFeedReason(view.lastError)) {
|
|
660
734
|
view.lastError = String((e && e.message) || e);
|
|
661
735
|
}
|
|
662
|
-
const delay = backoffMs(
|
|
736
|
+
const delay = backoffMs(ownerId);
|
|
737
|
+
const retry = retryFor(ownerId);
|
|
738
|
+
clearRetry(ownerId);
|
|
739
|
+
retry.nextRetryAt = now() + delay;
|
|
663
740
|
log(`event-feed-client: ${ownerId.slice(0, 8)}… retry in ${delay} ms (${view.lastError})`);
|
|
664
|
-
setTimeout(() => { void poll(); }, delay);
|
|
741
|
+
retry.timer = setTimeout(() => { retry.timer = null; void poll(); }, delay);
|
|
742
|
+
if (typeof retry.timer.unref === 'function') retry.timer.unref();
|
|
665
743
|
} finally {
|
|
666
|
-
//
|
|
667
|
-
// streamOnce, quello ha già messo (e alla fine tolto) il suo slot vero.
|
|
744
|
+
// Un round obsoleto non libera mai lo slot della nuova generazione.
|
|
668
745
|
if (running.get(ownerId) === slot) running.delete(ownerId);
|
|
746
|
+
slot.abort();
|
|
669
747
|
}
|
|
670
748
|
}));
|
|
671
749
|
}
|
|
@@ -685,7 +763,9 @@ function createEventFeedClient(opts = {}) {
|
|
|
685
763
|
function stop() {
|
|
686
764
|
stopped = true; generation += 1;
|
|
687
765
|
if (pollTimer) { clearInterval(pollTimer); pollTimer = null; }
|
|
766
|
+
for (const ownerId of retries.keys()) clearRetry(ownerId, true);
|
|
688
767
|
for (const [, h] of running) { try { h.abort(); } catch (_) {} }
|
|
768
|
+
running.clear();
|
|
689
769
|
}
|
|
690
770
|
|
|
691
771
|
// Local aggregated read: connected views only, with the watermark the UI
|
|
@@ -706,7 +786,7 @@ function createEventFeedClient(opts = {}) {
|
|
|
706
786
|
lastError: v.lastError || null,
|
|
707
787
|
ingressBlockedUntil: v.ingressBlockedUntil || null,
|
|
708
788
|
ingressBlockReason: v.ingressBlockReason || null,
|
|
709
|
-
asks: v.asks, notifications: v.notifications, fleetState: v.fleetState,
|
|
789
|
+
asks: visibleAsks(ownerId, v.asks), notifications: v.notifications, fleetState: v.fleetState,
|
|
710
790
|
})),
|
|
711
791
|
};
|
|
712
792
|
}
|
|
@@ -74,6 +74,7 @@ const PRODUCERS = {
|
|
|
74
74
|
question: cleanStr(p.question, 3900) || '',
|
|
75
75
|
...(Array.isArray(p.options) ? { options: p.options.slice(0, 8).map((o) => cleanStr(o, 200) || '') } : {}),
|
|
76
76
|
session: cleanStr(p.session, 128) || undefined,
|
|
77
|
+
...(Number.isSafeInteger(p.askTs) && p.askTs > 0 ? { askTs: p.askTs } : {}),
|
|
77
78
|
ts: typeof p.ts === 'number' ? p.ts : undefined,
|
|
78
79
|
}),
|
|
79
80
|
},
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// Durable receiver-to-owner synchronization. No attempt may submit an answer.
|
|
3
|
+
function createImportedDismissalDrainer({ store, relay, now = Date.now, random = Math.random,
|
|
4
|
+
setTimer = setTimeout, clearTimer = clearTimeout, intervalMs = 15000,
|
|
5
|
+
globalBudget = 4, ownerBudget = 1, budgetWindowMs = 60000, bindingForOwner = () => null, readonly = () => false, log = () => {} } = {}) {
|
|
6
|
+
const flights = new Set();
|
|
7
|
+
const globalHits = [];
|
|
8
|
+
const ownerHits = new Map();
|
|
9
|
+
let stopped = false;
|
|
10
|
+
let timer = null;
|
|
11
|
+
let running = null;
|
|
12
|
+
function defer(record, reason, blocked = false, retryAfterMs = 0) {
|
|
13
|
+
const attempts = (record.attempts || 0) + 1;
|
|
14
|
+
const delay = Math.max(retryAfterMs, Math.min(300000, 1000 * 2 ** Math.min(attempts, 9)) * (0.5 + random() * 0.5));
|
|
15
|
+
return store.updateImportedDismissal(record.ownerId, record.ownerAskId, {
|
|
16
|
+
syncState: blocked ? 'blocked' : 'pending', retryBinding: bindingForOwner(record.ownerId), attempts, lastReason: reason || 'owner-unreachable', nextRetryAt: now() + delay,
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
function rejected(record, result = {}) {
|
|
20
|
+
const reason = result.reason || 'owner-unreachable';
|
|
21
|
+
const transient = ['answering', 'delivery-unknown-block', 'reverse-slot-unverified', 'relay-deadline', 'owner-unreachable', 'federation-peer-unreachable'];
|
|
22
|
+
const pending = transient.includes(reason) || result.code === 429 || result.code >= 500;
|
|
23
|
+
return defer(record, reason, !pending, result.retryAfterMs || 0);
|
|
24
|
+
}
|
|
25
|
+
function confirmed(record, outcome) {
|
|
26
|
+
return store.updateImportedDismissal(record.ownerId, record.ownerAskId, {
|
|
27
|
+
syncState: outcome === 'answered' ? 'confirmed-answered' : 'confirmed-dismissed', ownerOutcome: outcome, lastReason: null,
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
async function attempt(record) {
|
|
31
|
+
const key = `${record.ownerId}|${record.ownerAskId}`;
|
|
32
|
+
if (flights.has(key) || stopped || readonly()) return;
|
|
33
|
+
flights.add(key);
|
|
34
|
+
try {
|
|
35
|
+
let current = store.getImportedDismissal(record.ownerId, record.ownerAskId);
|
|
36
|
+
if (!current || current.syncState !== 'pending') return;
|
|
37
|
+
const result = await relay.inspectDismissal({ ownerId: record.ownerId, askId: record.ownerAskId });
|
|
38
|
+
if (stopped || readonly()) return;
|
|
39
|
+
current = store.getImportedDismissal(record.ownerId, record.ownerAskId);
|
|
40
|
+
if (!current || current.syncState !== 'pending' || current.dismissedTs !== record.dismissedTs) return;
|
|
41
|
+
if (!result.ok) { rejected(current, result); return; }
|
|
42
|
+
if (result.generationPrecondition !== true) { defer(current, 'unsupported', true); return; }
|
|
43
|
+
if (!result.ask || (result.ask.ownerAskId || result.ask.id) !== current.ownerAskId
|
|
44
|
+
|| store.askFingerprint(result.ask) !== current.ownerAskFingerprint
|
|
45
|
+
|| !Number.isSafeInteger(result.ask.ts) || result.ask.ts <= 0
|
|
46
|
+
|| (current.generation === 'known' && current.ownerAskTs !== result.ask.ts)) {
|
|
47
|
+
defer(current, 'generation-mismatch', true); return;
|
|
48
|
+
}
|
|
49
|
+
const adopted = store.adoptImportedDismissal(current.ownerId, current.ownerAskId, result.ask);
|
|
50
|
+
if (!adopted.ok) { log(`dismissal adoption refused: ${adopted.reason}`); return; }
|
|
51
|
+
current = adopted.record;
|
|
52
|
+
if (['answered', 'dismissed'].includes(result.status)) { confirmed(current, result.status); return; }
|
|
53
|
+
if (result.status !== 'open') { rejected(current, { code: 409, reason: result.status || 'unknown' }); return; }
|
|
54
|
+
// The timestamp is durable before the request and checked atomically by
|
|
55
|
+
// the owner. A lost response leaves the record pending for reconciliation.
|
|
56
|
+
const out = await relay.relayDismiss({ ownerId: current.ownerId, askId: current.ownerAskId, expectedTs: current.ownerAskTs });
|
|
57
|
+
if (stopped || readonly()) return;
|
|
58
|
+
const latest = store.getImportedDismissal(current.ownerId, current.ownerAskId);
|
|
59
|
+
if (!latest || latest.syncState !== 'pending' || latest.dismissedTs !== current.dismissedTs) return;
|
|
60
|
+
if (out.ok && ['answered', 'dismissed'].includes(out.outcome)) confirmed(latest, out.outcome);
|
|
61
|
+
else rejected(latest, out.ok ? { code: 409, reason: 'unsupported' } : out);
|
|
62
|
+
} catch (error) {
|
|
63
|
+
const current = store.getImportedDismissal(record.ownerId, record.ownerAskId);
|
|
64
|
+
if (!stopped && current && current.syncState === 'pending') defer(current, 'owner-unreachable');
|
|
65
|
+
log(`dismissal synchronization failed: ${String(error.message || error)}`);
|
|
66
|
+
} finally { flights.delete(key); }
|
|
67
|
+
}
|
|
68
|
+
async function run() {
|
|
69
|
+
if (stopped || readonly()) return;
|
|
70
|
+
const t = now();
|
|
71
|
+
const prune = hits => { while (hits.length && hits[0] <= t - budgetWindowMs) hits.shift(); };
|
|
72
|
+
prune(globalHits);
|
|
73
|
+
for (const [ownerId, hits] of ownerHits) { prune(hits); if (!hits.length) ownerHits.delete(ownerId); }
|
|
74
|
+
for (let record of store.listImportedDismissals()) {
|
|
75
|
+
if (globalHits.length >= globalBudget) break;
|
|
76
|
+
if (record.syncState === 'blocked' && record.lastReason !== 'generation-mismatch') {
|
|
77
|
+
const binding = bindingForOwner(record.ownerId);
|
|
78
|
+
if (binding !== null && binding !== record.retryBinding) {
|
|
79
|
+
const resumed = store.updateImportedDismissal(record.ownerId, record.ownerAskId, { syncState: 'pending', nextRetryAt: 0, retryBinding: binding });
|
|
80
|
+
if (resumed.ok) record = resumed.record;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
if (record.syncState !== 'pending' || (record.nextRetryAt || 0) > now()) continue;
|
|
84
|
+
const hits = ownerHits.get(record.ownerId) || [];
|
|
85
|
+
if (hits.length >= ownerBudget) continue;
|
|
86
|
+
hits.push(t); ownerHits.set(record.ownerId, hits); globalHits.push(t);
|
|
87
|
+
await attempt(record);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
function drain() {
|
|
91
|
+
if (running) return running;
|
|
92
|
+
const flight = run().finally(() => { if (running === flight) running = null; });
|
|
93
|
+
running = flight;
|
|
94
|
+
return flight;
|
|
95
|
+
}
|
|
96
|
+
function schedule() {
|
|
97
|
+
if (stopped || timer) return;
|
|
98
|
+
timer = setTimer(() => { timer = null; void drain().catch(error => log(String(error.message || error))).finally(schedule); }, intervalMs);
|
|
99
|
+
if (timer && typeof timer.unref === 'function') timer.unref();
|
|
100
|
+
}
|
|
101
|
+
function start() { schedule(); void drain().catch(error => log(String(error.message || error))); }
|
|
102
|
+
function stop() { stopped = true; if (timer) clearTimer(timer); timer = null; }
|
|
103
|
+
function nudge() { if (!stopped) void drain().catch(error => log(String(error.message || error))); }
|
|
104
|
+
return { start, stop, drain, nudge };
|
|
105
|
+
}
|
|
106
|
+
module.exports = { createImportedDismissalDrainer };
|
package/lib/notify/routes.js
CHANGED
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
// Il paste della risposta riusa ESATTAMENTE pasteToSession (bracketed literal,
|
|
19
19
|
// niente Invio, control char rifiutati): qui si sanifica il testo PRIMA.
|
|
20
20
|
const express = require('express');
|
|
21
|
+
const accessPresets = require('../nodes/access-presets.js');
|
|
21
22
|
const { createAskAnswerService } = require('./ask-answer-service.js');
|
|
22
23
|
// Gli insiemi degli esiti vivono nella coda: il fan-out decide COSA accodare con
|
|
23
24
|
// la stessa definizione con cui la coda decide cosa ritentare. Due copie della
|
|
@@ -43,7 +44,7 @@ const ASK_KEYS = new Set(['question', 'options', 'session', 'target']);
|
|
|
43
44
|
// del nodo di origine, non un chiamante locale (stessa regola di FEDERATED_KEYS).
|
|
44
45
|
// `ownerNode`/`originNode` qualificano l'identita', `askId` e' l'id con cui
|
|
45
46
|
// l'owner conosce la domanda (serve alla risposta per tornare sul bersaglio).
|
|
46
|
-
const FEDERATED_ASK_KEYS = new Set(['askId', 'ownerNode', 'originNode', 'originCell', 'closeOutcome']);
|
|
47
|
+
const FEDERATED_ASK_KEYS = new Set(['askId', 'ownerNode', 'originNode', 'originCell', 'closeOutcome', 'ownerAskTs']);
|
|
47
48
|
const RATE_MAX = 6;
|
|
48
49
|
const RATE_WINDOW_MS = 60 * 1000;
|
|
49
50
|
const RATE_MAX_BUCKETS = 64;
|
|
@@ -116,23 +117,33 @@ function replyLabel(cfg) {
|
|
|
116
117
|
// peer autorizzati di questo nodo. L'enumerazione sta qui e non nel dispatcher
|
|
117
118
|
// perche' il dispatcher conosce solo il target esatto: una wildcard implicita
|
|
118
119
|
// sarebbe un modo per parlare a chi non si e' scelto.
|
|
119
|
-
async function
|
|
120
|
-
if (target
|
|
121
|
-
if (!peerTargets) return [];
|
|
120
|
+
async function resolveQualifiedFanTargets(peerTargets, target, self, log = () => {}) {
|
|
121
|
+
if (target === self) return { targets: [], known: true };
|
|
122
|
+
if (!peerTargets) return { targets: [], known: false };
|
|
122
123
|
let list;
|
|
123
|
-
try { list = await peerTargets(); } catch (_) { return []; }
|
|
124
|
-
if (!Array.isArray(list)) return [];
|
|
124
|
+
try { list = await peerTargets(); } catch (_) { return { targets: [], known: false }; }
|
|
125
|
+
if (!Array.isArray(list)) return { targets: [], known: false };
|
|
125
126
|
const seen = new Set();
|
|
126
127
|
const out = [];
|
|
128
|
+
let skipped = 0;
|
|
127
129
|
for (const raw of list) {
|
|
128
130
|
const id = raw && typeof raw === 'object' ? (raw.nodeId || raw.instanceId) : raw;
|
|
129
131
|
const clean = String(id || '');
|
|
130
|
-
if (!TARGET_RE.test(clean)) continue;
|
|
131
|
-
if (
|
|
132
|
-
|
|
133
|
-
|
|
132
|
+
if (!TARGET_RE.test(clean) || (self && clean === self) || seen.has(clean)) continue;
|
|
133
|
+
if (target !== undefined && clean !== String(target)) continue;
|
|
134
|
+
seen.add(clean);
|
|
135
|
+
// A routed destination is qualified by this node's own direct first hop.
|
|
136
|
+
const view = accessPresets.grantsOf(raw && typeof raw === 'object' ? (raw.firstHop || raw) : null);
|
|
137
|
+
if (!view.configured || view.label !== 'admin') { skipped++; continue; }
|
|
138
|
+
out.push(clean);
|
|
134
139
|
}
|
|
135
|
-
|
|
140
|
+
if (target !== undefined && !seen.has(String(target))) skipped++;
|
|
141
|
+
if (skipped) { try { log(`ask fan-out skipped: not admin (${skipped})`); } catch (_) {} }
|
|
142
|
+
return { targets: out, known: true };
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
async function resolveFanTargets(peerTargets, target, self, log = () => {}) {
|
|
146
|
+
return (await resolveQualifiedFanTargets(peerTargets, target, self, log)).targets;
|
|
136
147
|
}
|
|
137
148
|
|
|
138
149
|
// La CHIUSURA di un ask segue la stessa strada dell'andata: chi ha ricevuto la
|
|
@@ -154,10 +165,13 @@ function createClosureFanout({
|
|
|
154
165
|
const self = localNodeId();
|
|
155
166
|
// `targets` esplicito = ritentativo: si va SOLO verso i pendenti, non di
|
|
156
167
|
// nuovo verso tutti. Senza, si enumerano i peer autorizzati come sempre.
|
|
157
|
-
const
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
const
|
|
168
|
+
const inventory = await resolveQualifiedFanTargets(peerTargets, undefined, self, log);
|
|
169
|
+
const eligible = inventory.targets;
|
|
170
|
+
const allowed = new Set(eligible);
|
|
171
|
+
const targets2 = Array.isArray(targets) && targets.length ? targets.filter(target => allowed.has(target)) : eligible;
|
|
172
|
+
const out = Array.isArray(targets) ? targets.filter(target => !allowed.has(target))
|
|
173
|
+
.map(target => ({ target, status: inventory.known ? 'refused' : 'unknown',
|
|
174
|
+
reason: inventory.known ? 'ask-target-not-admin' : 'ask-targets-unavailable' })) : [];
|
|
161
175
|
for (const target of targets2) {
|
|
162
176
|
try {
|
|
163
177
|
const r = await dispatcher.dispatch({
|
|
@@ -191,7 +205,7 @@ function createClosureFanout({
|
|
|
191
205
|
}
|
|
192
206
|
|
|
193
207
|
function notifyRoutes({
|
|
194
|
-
cfg, notifier, push, asks,
|
|
208
|
+
cfg, notifier, push, asks, submit, sessionExists,
|
|
195
209
|
fleetP = null, instanceId = null, identityMode = 'legacy',
|
|
196
210
|
// Federazione delle notifiche. Assenti (test unitari, montaggi parziali) la
|
|
197
211
|
// route resta esattamente quella locale di prima: nessun percorso nuovo si
|
|
@@ -215,7 +229,8 @@ function notifyRoutes({
|
|
|
215
229
|
// not inject one: the local route and the federated surface must never drift
|
|
216
230
|
// into two different implementations.
|
|
217
231
|
const askService = answerService || createAskAnswerService({
|
|
218
|
-
asks,
|
|
232
|
+
asks,
|
|
233
|
+
submit, receipts,
|
|
219
234
|
labelPrefix: replyLabel(cfg),
|
|
220
235
|
onClosure: (kind, info) => { try { notifier.emitRaw({ type: kind, id: info.askId }); } catch (_) {} },
|
|
221
236
|
});
|
|
@@ -384,7 +399,7 @@ function notifyRoutes({
|
|
|
384
399
|
// perche' il dispatcher non conosce i broadcast: target esatto soltanto, e
|
|
385
400
|
// deve restare cosi' (una wildcard implicita e' un modo per parlare a chi non
|
|
386
401
|
// si e' scelto).
|
|
387
|
-
const askFanTargets = (target, self) => resolveFanTargets(peerTargets, target, self);
|
|
402
|
+
const askFanTargets = (target, self) => resolveFanTargets(peerTargets, target, self, log);
|
|
388
403
|
const closureFanout = closureFanoutDep || createClosureFanout({ dispatcher, peerTargets, localNodeId, log, retry: closureRetry });
|
|
389
404
|
|
|
390
405
|
// da revisione: gated READONLY (mutGate) — crea stato durevole (asks.json) e domande
|
|
@@ -466,12 +481,13 @@ function notifyRoutes({
|
|
|
466
481
|
return res.status(503).json({ status: 'unavailable', reason });
|
|
467
482
|
}
|
|
468
483
|
if (closed.changed && closed.ask) {
|
|
469
|
-
//
|
|
470
|
-
//
|
|
484
|
+
// Preserve the local alias id for older clients and the canonical
|
|
485
|
+
// owner ask id for clients that merge aliases with feed cards.
|
|
471
486
|
notifier.deliverOnlyRaw({
|
|
472
487
|
type: b.closeOutcome === 'dismissed' ? 'ask-dismissed' : 'ask-answered',
|
|
473
488
|
id: closed.ask.id,
|
|
474
489
|
ownerId: owner,
|
|
490
|
+
ownerAskId: closed.ask.ownerAskId,
|
|
475
491
|
});
|
|
476
492
|
}
|
|
477
493
|
return res.json({ status: 'delivered', closed: closed.changed });
|
|
@@ -491,12 +507,14 @@ function notifyRoutes({
|
|
|
491
507
|
// L'id con cui l'OWNER conosce la domanda: e' quello che la risposta
|
|
492
508
|
// deve citare. Il nostro `id` locale resta nostro.
|
|
493
509
|
ownerAskId: b.askId,
|
|
510
|
+
ownerAskTs: b.ownerAskTs,
|
|
494
511
|
originNode: owner,
|
|
495
512
|
originCell: resolved.origin.cell,
|
|
496
513
|
});
|
|
497
514
|
if (!out.ok) {
|
|
498
515
|
return res.status(out.reason === 'cap' ? 429 : 400).json({ status: 'refused', reason: out.reason, error: out.error });
|
|
499
516
|
}
|
|
517
|
+
if (out.suppressed) return res.json({ status: 'delivered', suppressed: true, ownerId: owner });
|
|
500
518
|
const ask = out.ask;
|
|
501
519
|
// deliverOnly: consegna locale alla UI, NIENTE pubblicazione sul feed e
|
|
502
520
|
// niente ri-esportazione. E' l'invariante che uccide il loop A->B->A per
|
|
@@ -550,6 +568,7 @@ function notifyRoutes({
|
|
|
550
568
|
...(ask.options ? { options: ask.options } : {}),
|
|
551
569
|
session: ask.session,
|
|
552
570
|
askId: ask.id,
|
|
571
|
+
ownerAskTs: ask.ts,
|
|
553
572
|
ownerNode: self,
|
|
554
573
|
},
|
|
555
574
|
});
|