@serfab/cadre-core 1.17.0 → 1.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. package/README.md +3 -2
  2. package/dist/cadre-invite-protocol.d.ts +35 -13
  3. package/dist/cadre-invite-protocol.js +90 -25
  4. package/dist/cadre-invite-protocol.js.map +1 -1
  5. package/dist/cadre-node.d.ts +586 -255
  6. package/dist/cadre-node.js +1182 -566
  7. package/dist/cadre-node.js.map +1 -1
  8. package/dist/control-cohort.d.ts +66 -9
  9. package/dist/control-cohort.js +67 -9
  10. package/dist/control-cohort.js.map +1 -1
  11. package/dist/control-database.d.ts +451 -193
  12. package/dist/control-database.js +701 -359
  13. package/dist/control-database.js.map +1 -1
  14. package/dist/control-formation-recorder.d.ts +46 -43
  15. package/dist/control-formation-recorder.js +77 -78
  16. package/dist/control-formation-recorder.js.map +1 -1
  17. package/dist/control-protocol-guard.d.ts +6 -0
  18. package/dist/control-protocol-guard.js +14 -2
  19. package/dist/control-protocol-guard.js.map +1 -1
  20. package/dist/control-schema-version.d.ts +48 -31
  21. package/dist/control-schema-version.js +67 -45
  22. package/dist/control-schema-version.js.map +1 -1
  23. package/dist/control-schema.d.ts +8 -8
  24. package/dist/control-schema.js +16 -16
  25. package/dist/control-schema.js.map +1 -1
  26. package/dist/control-write-retry.d.ts +42 -7
  27. package/dist/control-write-retry.js +87 -19
  28. package/dist/control-write-retry.js.map +1 -1
  29. package/dist/enrolled-machine-store.d.ts +3 -3
  30. package/dist/enrolled-machine-store.js +3 -3
  31. package/dist/formation-token.d.ts +48 -0
  32. package/dist/formation-token.js +26 -0
  33. package/dist/formation-token.js.map +1 -0
  34. package/dist/index.d.ts +15 -14
  35. package/dist/index.js +17 -14
  36. package/dist/index.js.map +1 -1
  37. package/dist/joined-strand-store.d.ts +26 -9
  38. package/dist/joined-strand-store.js +63 -24
  39. package/dist/joined-strand-store.js.map +1 -1
  40. package/dist/link-budget.d.ts +9 -7
  41. package/dist/link-budget.js +9 -7
  42. package/dist/link-budget.js.map +1 -1
  43. package/dist/member-redemption.d.ts +15 -9
  44. package/dist/member-redemption.js +13 -8
  45. package/dist/member-redemption.js.map +1 -1
  46. package/dist/membership-view.d.ts +91 -0
  47. package/dist/membership-view.js +110 -0
  48. package/dist/membership-view.js.map +1 -0
  49. package/dist/owner-grant-protocol.d.ts +11 -7
  50. package/dist/owner-grant-protocol.js +31 -17
  51. package/dist/owner-grant-protocol.js.map +1 -1
  52. package/dist/peer-addr-book.d.ts +10 -1
  53. package/dist/peer-addr-book.js +4 -3
  54. package/dist/peer-addr-book.js.map +1 -1
  55. package/dist/peer-join-backfill.d.ts +96 -30
  56. package/dist/peer-join-backfill.js +272 -128
  57. package/dist/peer-join-backfill.js.map +1 -1
  58. package/dist/pending-join-runner.d.ts +48 -18
  59. package/dist/pending-join-runner.js +74 -56
  60. package/dist/pending-join-runner.js.map +1 -1
  61. package/dist/seed-bootstrap.d.ts +47 -40
  62. package/dist/seed-bootstrap.js +66 -61
  63. package/dist/seed-bootstrap.js.map +1 -1
  64. package/dist/self-address-update.d.ts +13 -0
  65. package/dist/self-address-update.js +19 -0
  66. package/dist/self-address-update.js.map +1 -0
  67. package/dist/strand-database.d.ts +23 -2
  68. package/dist/strand-database.js +26 -6
  69. package/dist/strand-database.js.map +1 -1
  70. package/dist/strand-first-sync-gate.d.ts +176 -89
  71. package/dist/strand-first-sync-gate.js +288 -122
  72. package/dist/strand-first-sync-gate.js.map +1 -1
  73. package/dist/strand-formation-deadlines.d.ts +9 -9
  74. package/dist/strand-formation-deadlines.js +9 -9
  75. package/dist/strand-formation-manager.d.ts +23 -17
  76. package/dist/strand-formation-manager.js +46 -44
  77. package/dist/strand-formation-manager.js.map +1 -1
  78. package/dist/strand-formation-protocol.d.ts +5 -8
  79. package/dist/strand-formation-protocol.js +101 -16
  80. package/dist/strand-formation-protocol.js.map +1 -1
  81. package/dist/strand-instance-manager.d.ts +26 -17
  82. package/dist/strand-instance-manager.js +119 -50
  83. package/dist/strand-instance-manager.js.map +1 -1
  84. package/dist/strand-membership-reconciler.d.ts +16 -13
  85. package/dist/strand-membership-reconciler.js +28 -22
  86. package/dist/strand-membership-reconciler.js.map +1 -1
  87. package/dist/strand-membership-writer.d.ts +40 -28
  88. package/dist/strand-membership-writer.js +212 -153
  89. package/dist/strand-membership-writer.js.map +1 -1
  90. package/dist/strand-move.js +5 -2
  91. package/dist/strand-move.js.map +1 -1
  92. package/dist/strand-revocation-enforcer.d.ts +134 -60
  93. package/dist/strand-revocation-enforcer.js +144 -71
  94. package/dist/strand-revocation-enforcer.js.map +1 -1
  95. package/dist/strand-solicitation.d.ts +50 -39
  96. package/dist/strand-solicitation.js +10 -3
  97. package/dist/strand-solicitation.js.map +1 -1
  98. package/dist/strand-watcher.d.ts +80 -13
  99. package/dist/strand-watcher.js +152 -48
  100. package/dist/strand-watcher.js.map +1 -1
  101. package/dist/table-change-watch.d.ts +63 -0
  102. package/dist/table-change-watch.js +209 -0
  103. package/dist/table-change-watch.js.map +1 -0
  104. package/dist/timeout-scheduler.d.ts +4 -3
  105. package/dist/timeout-scheduler.js +4 -3
  106. package/dist/timeout-scheduler.js.map +1 -1
  107. package/dist/types.d.ts +90 -49
  108. package/dist/types.js +4 -4
  109. package/dist/types.js.map +1 -1
  110. package/package.json +8 -8
  111. package/src/cadre-invite-protocol.ts +119 -31
  112. package/src/cadre-node.ts +1289 -588
  113. package/src/control-cohort.ts +86 -9
  114. package/src/control-database.ts +848 -403
  115. package/src/control-formation-recorder.ts +359 -353
  116. package/src/control-protocol-guard.ts +16 -2
  117. package/src/control-schema-version.ts +101 -51
  118. package/src/control-schema.ts +16 -16
  119. package/src/control-write-retry.ts +91 -19
  120. package/src/delegate-admission.ts +351 -351
  121. package/src/enrolled-machine-store.ts +3 -3
  122. package/src/formation-token.ts +63 -0
  123. package/src/index.ts +30 -18
  124. package/src/joined-strand-store.ts +69 -24
  125. package/src/link-budget.ts +9 -7
  126. package/src/member-redemption.ts +20 -13
  127. package/src/membership-view.ts +144 -0
  128. package/src/owner-grant-protocol.ts +33 -18
  129. package/src/peer-addr-book.ts +4 -3
  130. package/src/peer-join-backfill.ts +961 -761
  131. package/src/pending-join-runner.ts +76 -53
  132. package/src/seed-bootstrap.ts +72 -67
  133. package/src/self-address-update.ts +19 -0
  134. package/src/strand-database.ts +352 -322
  135. package/src/strand-first-sync-gate.ts +334 -134
  136. package/src/strand-formation-deadlines.ts +227 -227
  137. package/src/strand-formation-manager.ts +731 -725
  138. package/src/strand-formation-protocol.ts +122 -24
  139. package/src/strand-instance-manager.ts +2092 -2026
  140. package/src/strand-membership-reconciler.ts +31 -25
  141. package/src/strand-membership-writer.ts +2093 -2019
  142. package/src/strand-move.ts +5 -2
  143. package/src/strand-revocation-enforcer.ts +740 -654
  144. package/src/strand-solicitation.ts +61 -34
  145. package/src/strand-watcher.ts +175 -56
  146. package/src/table-change-watch.ts +285 -0
  147. package/src/timeout-scheduler.ts +29 -28
  148. package/src/types.ts +89 -51
@@ -24,6 +24,7 @@ import {
24
24
  isValidResponderCreatesResult,
25
25
  type FormationResultMessage
26
26
  } from './strand-formation-protocol.js';
27
+ import { classifyTokenRead, type FormationTokenRead } from './formation-token.js';
27
28
 
28
29
  const log = debug('sereus:cadre:solicitation');
29
30
 
@@ -40,6 +41,25 @@ export interface DisclosureValidator {
40
41
  validateDisclosure(token: string, disclosure: StrandFormationDisclosure): Promise<boolean>;
41
42
  }
42
43
 
44
+ /**
45
+ * A bound invite's host strand as {@link FormationUsageRecorder.resolveStrand} read it, once
46
+ * per session: the membership invitation is issued from it and the consent row recorded
47
+ * against it without reading the `Strand` row again.
48
+ */
49
+ export interface BoundHostStrand {
50
+ strandId: string;
51
+ /** `'c'` closed: the joiner gets a membership invitation; `'o'` open: nothing to invite into. */
52
+ strandType: 'o' | 'c';
53
+ /**
54
+ * The row's one-off `StampId`, which the consent row names (`FormationUsage.StrandStampId`).
55
+ * Fixed for the life of the row; a row removed and re-seated in between gets a new one, and
56
+ * the `StrandExists` CHECK then refuses the consent row.
57
+ */
58
+ strandStampId: string;
59
+ /** The closed-strand read-gating secret; null for an open strand. */
60
+ memberPrivateKey: string | null;
61
+ }
62
+
43
63
  /**
44
64
  * Discriminated result of resolving the host strand an invite binds to (responder side):
45
65
  *
@@ -52,16 +72,13 @@ export interface DisclosureValidator {
52
72
  * non-existent strand, which would fail the deferred `FormationUsage.StrandExists` CHECK at
53
73
  * commit and drop the result frame.
54
74
  *
55
- * Replaces the older `{ strandId, memberPrivateKey } | null` shape, which conflated `bound`
56
- * and `missing` (both produced a non-null result whenever the invite had a `StrandId`).
57
- *
58
75
  * The invite's `ValidationUrl` (the outside approval hook) is deliberately NOT surfaced here:
59
- * the recorder that performs the write reads it itself, so the choice of which hook to trust
60
- * never travels through a caller.
76
+ * the recorder that performs the write takes it from the session's {@link FormationTokenRead},
77
+ * so the choice of which hook to trust never travels through a caller.
61
78
  */
62
79
  export type ResolvedHostStrand =
63
80
  | { kind: 'unbound' }
64
- | { kind: 'bound'; strandId: string; memberPrivateKey: string | null }
81
+ | ({ kind: 'bound' } & BoundHostStrand)
65
82
  | { kind: 'missing'; strandId: string };
66
83
 
67
84
  /** One redemption of an invite against an already-existing host strand (the record-only consent write). */
@@ -87,48 +104,52 @@ export interface AuthorizedFormationUsage {
87
104
  }
88
105
 
89
106
  /**
90
- * Interface for recording formation usage
107
+ * The responder's view of the formation-invite tables. A session reads its token once
108
+ * ({@link readToken}) and hands that read to every later step, so no step reads the invite again.
91
109
  */
92
110
  export interface FormationUsageRecorder {
93
111
  /**
94
- * Record that a formation invite was used. Where {@link authorizeUsage} is implemented, this
95
- * is the same as authorizing and then calling `record()` straight away.
112
+ * Record that a formation invite was used, reading the invite for itself. Where
113
+ * {@link authorizeUsage} is implemented, this is the same as reading the token, authorizing,
114
+ * and calling `record()` straight away.
96
115
  */
97
116
  recordUsage(params: FormationUsageParams): Promise<void>;
98
117
 
99
118
  /**
100
119
  * Everything {@link recordUsage} does short of the write: obtain the outside approval (when
101
- * the invite demands one) and pre-check the seat budget, returning a handle whose `record()`
102
- * writes the consent row with exactly the fields that were approved. Lets the manager ask
103
- * before it writes anything into the host strand, and write consent only after.
120
+ * `read` says the invite demands one) and, after an obtained approval, re-check the seat
121
+ * budget, returning a handle whose `record()` writes the consent row with exactly the fields
122
+ * that were approved. Lets the manager ask before it writes anything into the host strand,
123
+ * and write consent only after. `host` is the resolved host strand the consent row names.
104
124
  * Optional: a recorder without it is treated as having nothing to ask up front.
105
125
  */
106
- authorizeUsage?(params: FormationUsageParams): Promise<AuthorizedFormationUsage>;
107
-
108
- /**
109
- * Check if a token has already been used (for single-use invites)
110
- */
111
- isTokenUsed(token: string): Promise<boolean>;
126
+ authorizeUsage?(
127
+ params: FormationUsageParams,
128
+ read: FormationTokenRead,
129
+ host: BoundHostStrand
130
+ ): Promise<AuthorizedFormationUsage>;
112
131
 
113
132
  /**
114
- * Check if a token is valid and not expired. An invalid answer says why: `'unknown'` (no
115
- * invitation row here — it may not have replicated yet) or `'expired'`. An answer without
116
- * a reason is treated as `'unknown'`, the retryable one.
133
+ * Read one token as a formation session acts on it: the invite row (null when there is none
134
+ * here — it may not have replicated yet) and, for a metered unexpired invite, its recorded
135
+ * use count. One read per session; `classifyTokenRead` turns it into the token check.
117
136
  */
118
- isTokenValid(token: string): Promise<{ valid: boolean; reason?: 'unknown' | 'expired'; invitation?: OpenInvitation }>;
137
+ readToken(token: string): Promise<FormationTokenRead>;
119
138
 
120
139
  /**
121
- * Resolve the host strand an invite binds to, classifying it as unbound / bound /
122
- * missing (see {@link ResolvedHostStrand}). Optional: a recorder that does not bind
123
- * strands omits it, and {@link StrandFormationManager} treats every invite as `unbound`
124
- * and falls back to its provisioner. Keeping resolution behind this interface keeps the
125
- * manager DB-agnostic and unit-testable with an in-memory fake.
140
+ * Resolve the host strand the read invite binds to, classifying it as unbound / bound /
141
+ * missing (see {@link ResolvedHostStrand}). Reads the `Strand` row, never the invite.
142
+ * Optional: a recorder that does not bind strands omits it, and
143
+ * {@link StrandFormationManager} treats every invite as `unbound` and falls back to its
144
+ * provisioner. Keeping resolution behind this interface keeps the manager DB-agnostic and
145
+ * unit-testable with an in-memory fake.
126
146
  */
127
- resolveStrand?(token: string): Promise<ResolvedHostStrand>;
147
+ resolveStrand?(read: FormationTokenRead): Promise<ResolvedHostStrand>;
128
148
 
129
149
  /**
130
150
  * Provision a NEW strand for an UNBOUND invite and record consent against it
131
151
  * ATOMICALLY (single `FormationUsage` row → single-use enforced on the next redemption).
152
+ * Takes the invite's approval hook and seat budget from `read`.
132
153
  * Optional: a recorder that only supports provision-then-record omits it, and the
133
154
  * manager falls back to its {@link StrandProvisioner}. Mirrors the create-strand-by-consent
134
155
  * path the schema's `Strand.AuthorizedInsert` `FormationUsage` branch authorizes. Returns the
@@ -147,14 +168,14 @@ export interface FormationUsageRecorder {
147
168
  disclosure: string;
148
169
  /** Aborted when the caller has given up; observed BEFORE any write so the invite stays unspent. */
149
170
  signal?: AbortSignal;
150
- }): Promise<{ strandId: string; memberPrivateKey: string | null }>;
171
+ }, read: FormationTokenRead): Promise<{ strandId: string; memberPrivateKey: string | null }>;
151
172
 
152
173
  /**
153
174
  * Is ANY invitation this recorder knows about unexpired and not fully
154
175
  * consumed? Optional: a recorder that cannot enumerate invitations omits it,
155
176
  * and only locally-minted invitations hold the connection gate's formation
156
- * exemption open. Distinct from the per-token {@link isTokenValid} /
157
- * {@link isTokenUsed} pair — the connection gate has no token to ask about.
177
+ * exemption open. Distinct from the per-token {@link readToken} — the
178
+ * connection gate has no token to ask about.
158
179
  */
159
180
  hasOutstandingInvitation?(): Promise<boolean>;
160
181
  }
@@ -470,8 +491,8 @@ export class StrandSolicitationService {
470
491
  * in-memory and dies with the process. Expired entries are dropped. With
471
492
  * no {@link FormationUsageRecorder} configured there is no consumption
472
493
  * oracle, so an unexpired minted token counts as outstanding by
473
- * construction; with one, a token it reports used is deleted (consumption
474
- * is permanent, so the registry never reconsiders it).
494
+ * construction; with one, a token whose row it reads as expired or used up
495
+ * is deleted (both are permanent, so the registry never reconsiders it).
475
496
  * 2. The recorder's optional {@link FormationUsageRecorder.hasOutstandingInvitation},
476
497
  * which survives restarts and sees invitations replicated in from siblings.
477
498
  *
@@ -488,7 +509,13 @@ export class StrandSolicitationService {
488
509
  if (!this.formationUsageRecorder) {
489
510
  return true;
490
511
  }
491
- if (!(await this.formationUsageRecorder.isTokenUsed(token))) {
512
+ // NOTE: a read of its own, not shared with a formation session's: this runs while a
513
+ // stranger's connection is being admitted, before any stream or token exists, under the
514
+ // gate's fail-open deadline, and the session that follows reads on a different call.
515
+ // An unknown token stays outstanding (its row may not be published yet); an expired or
516
+ // used-up row is dropped, the same `token-spent` the formation handler would answer.
517
+ const check = classifyTokenRead(await this.formationUsageRecorder.readToken(token), now);
518
+ if (check.valid || check.code === 'token-unknown') {
492
519
  return true;
493
520
  }
494
521
  this.mintedInvitations.delete(token);
@@ -1,4 +1,6 @@
1
1
  import debug from 'debug';
2
+ import { scheduleRuns } from './table-change-watch.js';
3
+ import type { TableChangeWatch, TableChangeWatchOptions } from './table-change-watch.js';
2
4
  import type { StrandFilter, StrandRow } from './types.js';
3
5
 
4
6
  const log = debug('sereus:cadre:strand-watcher');
@@ -7,7 +9,7 @@ const log = debug('sereus:cadre:strand-watcher');
7
9
  * Outcome of evaluating a strand against the current filter.
8
10
  * - `pass`: admit (final)
9
11
  * - `reject`: exclude
10
- * - `defer`: admit provisionally; sAppId not yet known, re-evaluate next poll
12
+ * - `defer`: admit provisionally; sAppId not yet known, re-evaluate when it is learned
11
13
  */
12
14
  type FilterDecision = 'pass' | 'reject' | 'defer';
13
15
 
@@ -15,10 +17,34 @@ type FilterDecision = 'pass' | 'reject' | 'defer';
15
17
  * Ceiling on the retry delay for a strand whose launch keeps failing. Attempts
16
18
  * are never abandoned — the failure this backoff exists for (a transient network
17
19
  * or storage fault) can last a long time — but a permanently-unlaunchable strand
18
- * must not re-attempt on every poll and storm the host app with `strand:error`.
20
+ * must not re-attempt on every run and storm the host app with `strand:error`.
19
21
  */
20
22
  const MAX_RETRY_BACKOFF_MS = 5 * 60 * 1000;
21
23
 
24
+ /**
25
+ * Default backstop: the longest the watcher goes without re-reading when no change to
26
+ * `Strand` or `JoinedStrand` is heard (`docs/architecture.md` → "Table change subscriptions"
27
+ * says what the change subscription misses).
28
+ *
29
+ * NOTE: on a cadre larger than `CONTROL_REPLICATION_BREADTH` (16), a machine outside the
30
+ * `Strand` block's cohort hears no change event, so this backstop is its only path and a
31
+ * strand founded elsewhere reaches it up to 5 minutes later. If that matters, shorten the
32
+ * backstop for such machines rather than for everyone.
33
+ */
34
+ export const DEFAULT_STRAND_WATCH_BACKSTOP_MS = 5 * 60 * 1000;
35
+
36
+ /**
37
+ * First step of the launch-retry and read-retry ladders, unless the backstop is shorter: a
38
+ * test or scenario that passes a short interval keeps the short base it always had.
39
+ */
40
+ const LAUNCH_RETRY_BASE_MS = 5000;
41
+
42
+ /** Ceiling on the read-retry ladder (see `StrandWatcher.scheduleReadRetry`), unless the backstop is shorter. */
43
+ const READ_RETRY_CAP_MS = 60_000;
44
+
45
+ /** Delay before the first run, so `start()` never reads synchronously. */
46
+ const INITIAL_RUN_DELAY_MS = 100;
47
+
22
48
  /** Per-strand consecutive-failure state driving the launch retry backoff. */
23
49
  interface StrandFailureState {
24
50
  /** Consecutive `onStrandAdded` failures since the last success. */
@@ -45,10 +71,14 @@ export interface StrandWatcherCallbacks {
45
71
  }
46
72
 
47
73
  /**
48
- * Interface for querying strands from control network
74
+ * What the watcher reads, and how it hears that what it reads changed.
49
75
  */
50
76
  export interface StrandQueryable {
51
77
  queryStrands(): Promise<StrandRow[]>;
78
+ /** Subscribe to changes of the tables {@link queryStrands} reads. Absent: runs come from `trigger` and the backstop only. */
79
+ watchChanges?(onChange: () => Promise<void>, options: TableChangeWatchOptions): TableChangeWatch;
80
+ /** True when the last {@link queryStrands} answered part of its result from a stand-in after a failed read. */
81
+ lastReadPartial?(): boolean;
52
82
  }
53
83
 
54
84
  /**
@@ -63,11 +93,16 @@ export interface SAppIdLookup {
63
93
  * Watches the control network's Strand table for changes and triggers
64
94
  * strand instance start/stop via callbacks.
65
95
  *
66
- * Uses polling until Optimystic supports reactive subscriptions.
96
+ * Every read is one run on a {@link TableChangeWatch}: once at start, on a change to the
97
+ * tables the queryable reads, on the watcher's own retries, and on the backstop.
67
98
  */
68
99
  export class StrandWatcher {
69
100
  private readonly filter: StrandFilter;
70
- private readonly pollInterval: number;
101
+ /** Longest gap between runs when no change is heard. */
102
+ private readonly backstopMs: number;
103
+ /** First step of the launch-retry and read-retry ladders. */
104
+ private readonly retryBaseMs: number;
105
+ private readonly readRetryCapMs: number;
71
106
  private readonly callbacks: StrandWatcherCallbacks;
72
107
  private readonly queryable: StrandQueryable;
73
108
  private readonly sAppIdLookup?: SAppIdLookup;
@@ -81,7 +116,7 @@ export class StrandWatcher {
81
116
  private readonly now: () => number;
82
117
 
83
118
  private knownStrands: Map<string, StrandRow> = new Map();
84
- /** Ids admitted under a `defer` decision; re-evaluated each poll until they resolve. */
119
+ /** Ids admitted under a `defer` decision; re-evaluated each run until they resolve. */
85
120
  private provisional: Set<string> = new Set();
86
121
  /** Ids whose last launch attempt threw, with the backoff gating their retry. */
87
122
  private failureStates: Map<string, StrandFailureState> = new Map();
@@ -92,31 +127,39 @@ export class StrandWatcher {
92
127
  * strand, the permanence has to be recorded explicitly.
93
128
  */
94
129
  private suppressed: Set<string> = new Set();
95
- private pollTimer: ReturnType<typeof setInterval> | null = null;
96
- private initialPollTimer: ReturnType<typeof setTimeout> | null = null;
130
+ /** Consecutive runs whose read failed or was partial; drives {@link scheduleReadRetry}. */
131
+ private readFailures = 0;
132
+ /** Built by {@link start}, disposed by {@link stop}; a restart builds a new one. */
133
+ private watch: TableChangeWatch | null = null;
97
134
  private running = false;
98
135
 
136
+ /**
137
+ * @param backstopMs - `CadreNodeConfig.strandWatchInterval`: the longest the watcher goes
138
+ * without re-reading when no change is heard. Also caps the retry ladders' first step.
139
+ */
99
140
  constructor(
100
141
  queryable: StrandQueryable,
101
142
  callbacks: StrandWatcherCallbacks,
102
143
  filter: StrandFilter = { mode: 'all' },
103
- pollInterval: number = 5000,
144
+ backstopMs: number = DEFAULT_STRAND_WATCH_BACKSTOP_MS,
104
145
  sAppIdLookup?: SAppIdLookup,
105
146
  now: () => number = Date.now
106
147
  ) {
107
148
  this.queryable = queryable;
108
149
  this.callbacks = callbacks;
109
150
  this.filter = filter;
110
- this.pollInterval = pollInterval;
151
+ this.backstopMs = backstopMs;
152
+ this.retryBaseMs = Math.min(backstopMs, LAUNCH_RETRY_BASE_MS);
153
+ this.readRetryCapMs = Math.min(backstopMs, READ_RETRY_CAP_MS);
111
154
  this.sAppIdLookup = sAppIdLookup;
112
155
  this.now = now;
113
- log('StrandWatcher created with filter: %o, interval: %dms', filter, pollInterval);
156
+ log('StrandWatcher created with filter: %o, backstop: %dms', filter, backstopMs);
114
157
  }
115
158
 
116
159
  /**
117
160
  * Evaluate a strand against the current filter, distinguishing a not-yet-known
118
161
  * sAppId (`defer`) from a known non-match (`reject`). A `defer` admission is
119
- * provisional and re-checked on subsequent polls.
162
+ * provisional and re-checked on subsequent runs (see {@link sAppIdLearned}).
120
163
  */
121
164
  private evaluateFilter(strand: StrandRow): FilterDecision {
122
165
  switch (this.filter.mode) {
@@ -133,7 +176,7 @@ export class StrandWatcher {
133
176
  }
134
177
  const sAppId = this.sAppIdLookup.getSAppId(strand.Id);
135
178
  if (sAppId === undefined) {
136
- // sAppId not yet known - admit provisionally and re-evaluate next poll.
179
+ // sAppId not yet known - admit provisionally and re-evaluate once it is learned.
137
180
  log('sAppId unknown for strand %s - deferring filter decision', strand.Id);
138
181
  return 'defer';
139
182
  }
@@ -149,20 +192,73 @@ export class StrandWatcher {
149
192
 
150
193
  /**
151
194
  * Record a failed launch attempt and schedule when the strand may be retried:
152
- * `pollInterval * 2^(failures-1)`, capped at {@link MAX_RETRY_BACKOFF_MS}.
195
+ * `retryBaseMs * 2^(failures-1)`, capped at {@link MAX_RETRY_BACKOFF_MS}.
153
196
  */
154
197
  private recordFailure(strandId: string): void {
155
198
  const failures = (this.failureStates.get(strandId)?.failures ?? 0) + 1;
156
- const delay = Math.min(this.pollInterval * 2 ** (failures - 1), MAX_RETRY_BACKOFF_MS);
199
+ const delay = Math.min(this.retryBaseMs * 2 ** (failures - 1), MAX_RETRY_BACKOFF_MS);
157
200
  this.failureStates.set(strandId, { failures, nextAttemptAt: this.now() + delay });
158
201
  log('Strand %s launch failed (%d consecutive) - next attempt in %dms', strandId, failures, delay);
159
202
  }
160
203
 
161
204
  /**
162
- * Poll for strand changes.
205
+ * One run: read, reconcile, then arm the follow-up runs this run needs. A run the watch
206
+ * starts clears every pending wake, so each run re-arms the retries still owed. A
207
+ * {@link forcePoll} run clears none, so what it arms may sit beside an equivalent wake;
208
+ * whichever fires first runs and clears the other.
163
209
  */
164
- private async poll(): Promise<void> {
165
- if (!this.running) return;
210
+ private async runOnce(): Promise<void> {
211
+ const startedAt = this.now();
212
+ const readComplete = await this.poll();
213
+ if (this.running) {
214
+ this.scheduleLaunchRetry(startedAt);
215
+ this.scheduleReadRetry(readComplete);
216
+ }
217
+ }
218
+
219
+ /**
220
+ * Wake for the earliest launch retry falling due after `since`. A backoff that expired at
221
+ * or before `since` belongs to a strand the run starting then either attempted or skipped
222
+ * for a reason other than backoff (suppressed, filtered out); waking for it would spin.
223
+ * Without this a failed launch would wait for an unrelated change or the backstop.
224
+ */
225
+ private scheduleLaunchRetry(since: number): void {
226
+ let earliest = Infinity;
227
+ for (const { nextAttemptAt } of this.failureStates.values()) {
228
+ if (nextAttemptAt > since) {
229
+ earliest = Math.min(earliest, nextAttemptAt);
230
+ }
231
+ }
232
+ if (earliest !== Infinity) {
233
+ this.watch?.trigger(Math.max(0, earliest - this.now()));
234
+ }
235
+ }
236
+
237
+ /**
238
+ * After a failed or partial read, run again on a ladder from `retryBaseMs`, doubling per
239
+ * consecutive failed run up to `readRetryCapMs`; a complete read resets it. Not left to
240
+ * the backstop because the plugin arms a table's replicated-change bridge on the table's
241
+ * first successful initialization: a machine whose first read failed (cut off from its
242
+ * party, or its sibling unreachable for a never-written `JoinedStrand`) would otherwise
243
+ * hear nothing until then.
244
+ */
245
+ private scheduleReadRetry(readComplete: boolean): void {
246
+ if (readComplete) {
247
+ this.readFailures = 0;
248
+ return;
249
+ }
250
+ this.readFailures++;
251
+ const delay = Math.min(this.retryBaseMs * 2 ** (this.readFailures - 1), this.readRetryCapMs);
252
+ log('Strand read incomplete (%d consecutive) - re-reading in %dms', this.readFailures, delay);
253
+ this.watch?.trigger(delay);
254
+ }
255
+
256
+ /**
257
+ * Read and reconcile. Resolves true when the read answered in full: false when it threw,
258
+ * or when the queryable answered part of it from a stand-in.
259
+ */
260
+ private async poll(): Promise<boolean> {
261
+ if (!this.running) return true;
166
262
 
167
263
  try {
168
264
  const currentStrands = await this.queryable.queryStrands();
@@ -173,17 +269,17 @@ export class StrandWatcher {
173
269
  if (this.suppressed.has(strand.Id)) continue; // deliberately stopped locally
174
270
  if (this.knownStrands.has(strand.Id)) continue;
175
271
  const failure = this.failureStates.get(strand.Id);
176
- // Clock read per candidate, not once per poll: an earlier strand's launch
272
+ // Clock read per candidate, not once per run: an earlier strand's launch
177
273
  // may have taken a long time, and this one's gate must reflect that.
178
274
  if (failure && this.now() < failure.nextAttemptAt) continue; // backing off after a failed launch
179
275
  const decision = this.evaluateFilter(strand);
180
276
  if (decision === 'reject') continue;
181
277
  log('Strand added: %s', strand.Id);
182
- // Set BEFORE the await: this doubles as the in-flight guard that stops the
183
- // interval timer from starting a second concurrent launch for the same strand.
278
+ // Set BEFORE the await: this doubles as the in-flight guard that stops a
279
+ // concurrent run (`forcePoll` beside a scheduled one) from launching the strand twice.
184
280
  this.knownStrands.set(strand.Id, strand);
185
281
  if (decision === 'defer') {
186
- // Provisional admission: sAppId unknown, re-check on later polls.
282
+ // Provisional admission: sAppId unknown, re-check on later runs.
187
283
  this.provisional.add(strand.Id);
188
284
  }
189
285
  try {
@@ -215,7 +311,7 @@ export class StrandWatcher {
215
311
  log('Error handling strand remove for %s: %o', strandId, error);
216
312
  }
217
313
  }
218
- // decision === 'defer': still unknown, leave provisional for next poll.
314
+ // decision === 'defer': still unknown, leave provisional until sAppIdLearned wakes a run.
219
315
  }
220
316
 
221
317
  // Find removed strands. A strand whose launch failed is no longer in
@@ -242,8 +338,8 @@ export class StrandWatcher {
242
338
  this.failureStates.delete(strandId);
243
339
  }
244
340
  }
245
- // NOTE: the clear needs a poll that actually SEES the row absent, so a sibling that
246
- // unpublishes and re-publishes a locally-stopped id inside one poll interval leaves
341
+ // NOTE: the clear needs a run that actually SEES the row absent, so a sibling that
342
+ // unpublishes and re-publishes a locally-stopped id between two runs leaves
247
343
  // it suppressed for the session. Contrived today — re-seating a removed id is
248
344
  // owner-gated and manual — and there is no row version to distinguish the new row
249
345
  // from the old. If re-publishing a removed id ever becomes routine, give the row a
@@ -253,14 +349,17 @@ export class StrandWatcher {
253
349
  this.suppressed.delete(strandId);
254
350
  }
255
351
  }
352
+ return !(this.queryable.lastReadPartial?.() ?? false);
256
353
  } catch (error) {
257
354
  // NOTE: a machine cut off from its party before it ever received the `Strand` block
258
- // fails this read `cohort-unreachable` on every poll and logs here (observed in
259
- // `control-cohort-edge-carries-data`). Harmless: nothing was ever launched from a block
260
- // this machine never held, and this catch keeps the known set. Do not answer it as
261
- // empty the way `ControlDatabase.queryRevokedStamps` does for `Revocation` — an empty
262
- // answer here runs the removed-strand loop above. Revisit if the log volume matters.
355
+ // fails this read `cohort-unreachable` on every step of the read-retry ladder and logs
356
+ // here (observed in `control-cohort-edge-carries-data`). Harmless: nothing was ever
357
+ // launched from a block this machine never held, and this catch keeps the known set. Do
358
+ // not answer it as empty the way `ControlDatabase.queryRevokedStamps` does for
359
+ // `Revocation` — an empty answer here runs the removed-strand loop above. Revisit if
360
+ // the log volume matters.
263
361
  log('Error polling strands: %o', error);
362
+ return false;
264
363
  }
265
364
  }
266
365
 
@@ -274,23 +373,24 @@ export class StrandWatcher {
274
373
  }
275
374
 
276
375
  log('Starting StrandWatcher');
376
+ // Subscribed before the first read, so a commit landing between them is either seen by
377
+ // that read or wakes a follow-up run.
378
+ this.watch = this.subscribe();
277
379
  this.running = true;
278
-
279
- // Set up periodic polling
280
- this.pollTimer = setInterval(() => {
281
- void this.poll();
282
- }, this.pollInterval);
283
-
284
- // Defer the first poll so start() doesn't block the caller.
285
- // Strands added via addStrand() typically arrive after start() completes.
286
- this.initialPollTimer = setTimeout(() => {
287
- this.initialPollTimer = null;
288
- void this.poll();
289
- }, 100);
380
+ // Deferred so start() doesn't block the caller; strands added via addStrand()
381
+ // typically arrive after start() completes.
382
+ this.watch.trigger(INITIAL_RUN_DELAY_MS);
290
383
 
291
384
  log('StrandWatcher started');
292
385
  }
293
386
 
387
+ /** The watch every run goes through: the queryable's change subscription, or a timer-only one. */
388
+ private subscribe(): TableChangeWatch {
389
+ const onChange = (): Promise<void> => this.runOnce();
390
+ const options: TableChangeWatchOptions = { label: 'strand-watcher', backstopMs: this.backstopMs };
391
+ return this.queryable.watchChanges?.(onChange, options) ?? scheduleRuns(onChange, options);
392
+ }
393
+
294
394
  /**
295
395
  * Stop watching for strand changes
296
396
  */
@@ -301,27 +401,22 @@ export class StrandWatcher {
301
401
 
302
402
  log('Stopping StrandWatcher');
303
403
  this.running = false;
304
-
305
- if (this.initialPollTimer) {
306
- clearTimeout(this.initialPollTimer);
307
- this.initialPollTimer = null;
308
- }
309
-
310
- if (this.pollTimer) {
311
- clearInterval(this.pollTimer);
312
- this.pollTimer = null;
313
- }
404
+ // A run already in flight finishes; `running` keeps it from arming anything further.
405
+ this.watch?.dispose();
406
+ this.watch = null;
314
407
 
315
408
  this.knownStrands.clear();
316
409
  this.provisional.clear();
317
410
  this.failureStates.clear();
318
411
  this.suppressed.clear();
412
+ this.readFailures = 0;
319
413
  log('StrandWatcher stopped');
320
414
  }
321
415
 
322
416
  /**
323
- * Forget a strand whose launch failed outside this watcher, so a later poll
324
- * re-offers it — gated by the same backoff a watcher-driven failure gets.
417
+ * Forget a strand whose launch failed outside this watcher, so a later run
418
+ * re-offers it — gated by the same backoff a watcher-driven failure gets, and woken
419
+ * when that backoff expires.
325
420
  *
326
421
  * A failed launch normally leaves nothing running (StrandInstanceManager drops the
327
422
  * record), which is what makes re-offering it correct. One case does leave something
@@ -335,6 +430,27 @@ export class StrandWatcher {
335
430
  this.knownStrands.delete(strandId);
336
431
  this.provisional.delete(strandId);
337
432
  this.recordFailure(strandId);
433
+ if (this.running) {
434
+ this.scheduleLaunchRetry(this.now());
435
+ }
436
+ }
437
+
438
+ /**
439
+ * Re-read soon: something the queryable merges in beside the watched tables changed
440
+ * (`CadreNode`'s machine-local joined-strand records). A no-op once stopped.
441
+ */
442
+ refresh(): void {
443
+ this.watch?.trigger();
444
+ }
445
+
446
+ /**
447
+ * The host just learned `strandId`'s sAppId: if the strand was admitted provisionally
448
+ * under an `sAppId` filter, run so the admission resolves now rather than on the backstop.
449
+ */
450
+ sAppIdLearned(strandId: string): void {
451
+ if (this.provisional.has(strandId)) {
452
+ this.watch?.trigger();
453
+ }
338
454
  }
339
455
 
340
456
  /**
@@ -379,10 +495,13 @@ export class StrandWatcher {
379
495
  }
380
496
 
381
497
  /**
382
- * Force an immediate poll (useful for testing)
498
+ * Run once now, outside the watch's schedule (tests, and callers that just wrote a row
499
+ * and want this node to converge before returning). May overlap a scheduled run:
500
+ * `knownStrands` is updated before each callback's await, so neither launches nor
501
+ * removes a strand twice.
383
502
  */
384
503
  async forcePoll(): Promise<void> {
385
- await this.poll();
504
+ await this.runOnce();
386
505
  }
387
506
  }
388
507