@powerhousedao/reactor 6.2.3-dev.2 → 6.2.3-dev.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +68 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +221 -24
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -4229,9 +4229,35 @@ const DRIVE_AUTH_ERROR_MESSAGES = {
|
|
|
4229
4229
|
forbidden: "Forbidden: insufficient permissions",
|
|
4230
4230
|
authenticationRequired: "Forbidden: authentication required"
|
|
4231
4231
|
};
|
|
4232
|
+
/**
|
|
4233
|
+
* A non-GraphQL HTTP failure against a drive endpoint.
|
|
4234
|
+
*
|
|
4235
|
+
* Drive discovery (`GET <base>/d/:drive`) is REST, not GraphQL, so its
|
|
4236
|
+
* failures cannot be a `GraphQLRequestError` without the name lying about
|
|
4237
|
+
* what was called. It carries the status for the same reason that one does:
|
|
4238
|
+
* `isDriveAuthError` is what decides whether a failure prompts a login, and a
|
|
4239
|
+
* bare `Error` tells it nothing.
|
|
4240
|
+
*/
|
|
4241
|
+
var DriveRequestError = class extends Error {
|
|
4242
|
+
statusCode;
|
|
4243
|
+
constructor(message, statusCode) {
|
|
4244
|
+
super(message);
|
|
4245
|
+
this.name = "DriveRequestError";
|
|
4246
|
+
this.statusCode = statusCode;
|
|
4247
|
+
}
|
|
4248
|
+
};
|
|
4232
4249
|
/** True when the remote rejected the caller as unauthenticated/unauthorized:
|
|
4233
|
-
* an HTTP 401/403, or a Forbidden/Unauthorized GraphQL error.
|
|
4250
|
+
* an HTTP 401/403, or a Forbidden/Unauthorized GraphQL error.
|
|
4251
|
+
*
|
|
4252
|
+
* 403 and 401 only — NOT 404. The drive info endpoint answers a drive the
|
|
4253
|
+
* caller may not read with the same 404 it gives a drive that does not exist,
|
|
4254
|
+
* so that an unauthorized caller cannot enumerate drives by probing slugs.
|
|
4255
|
+
* That is deliberate, and it costs exactly this: a protected drive is
|
|
4256
|
+
* indistinguishable from a typo, and prompting for a login on every 404 would
|
|
4257
|
+
* fire on every mistyped URL. See the `WWW-Authenticate` note on the endpoint
|
|
4258
|
+
* for the signal that would let a client tell the two apart. */
|
|
4234
4259
|
function isDriveAuthError(error) {
|
|
4260
|
+
if (error instanceof DriveRequestError) return error.statusCode === 401 || error.statusCode === 403;
|
|
4235
4261
|
if (!(error instanceof GraphQLRequestError)) return false;
|
|
4236
4262
|
if (error.category === "http") return error.statusCode === 401 || error.statusCode === 403;
|
|
4237
4263
|
if (error.category === "graphql") return Object.values(DRIVE_AUTH_ERROR_MESSAGES).some((m) => error.message.includes(m));
|
|
@@ -5028,6 +5054,10 @@ var GqlRequestChannel = class {
|
|
|
5028
5054
|
if (this.isShutdown) return;
|
|
5029
5055
|
this.pollTimer.triggerNow();
|
|
5030
5056
|
}
|
|
5057
|
+
/** This channel polls a remote itself; it has no holder to hear from. */
|
|
5058
|
+
notePoll() {}
|
|
5059
|
+
/** No holder, so nothing this channel reports may strand one. */
|
|
5060
|
+
lastHolderPollUtcMs() {}
|
|
5031
5061
|
/**
|
|
5032
5062
|
* Initializes the channel by registering it on the remote server and starting polling.
|
|
5033
5063
|
*/
|
|
@@ -5611,6 +5641,7 @@ var GqlResponseChannel = class {
|
|
|
5611
5641
|
lastPersistedOutboxOrdinal = 0;
|
|
5612
5642
|
evictedOutboxFloor = Number.POSITIVE_INFINITY;
|
|
5613
5643
|
appliedOutboxOrdinal = 0;
|
|
5644
|
+
lastPollUtcMs = Date.now();
|
|
5614
5645
|
connectionState = "connecting";
|
|
5615
5646
|
connectionStateCallbacks = /* @__PURE__ */ new Set();
|
|
5616
5647
|
constructor(logger, channelId, remoteName, cursorStorage) {
|
|
@@ -5653,7 +5684,7 @@ var GqlResponseChannel = class {
|
|
|
5653
5684
|
return {
|
|
5654
5685
|
state: this.connectionState,
|
|
5655
5686
|
failureCount: 0,
|
|
5656
|
-
lastSuccessUtcMs:
|
|
5687
|
+
lastSuccessUtcMs: this.lastPollUtcMs,
|
|
5657
5688
|
lastFailureUtcMs: 0,
|
|
5658
5689
|
pushBlocked: false,
|
|
5659
5690
|
pushFailureCount: 0,
|
|
@@ -5669,6 +5700,13 @@ var GqlResponseChannel = class {
|
|
|
5669
5700
|
}
|
|
5670
5701
|
/** Response channels are push-driven; resolvers populate mailboxes directly. */
|
|
5671
5702
|
triggerPull() {}
|
|
5703
|
+
notePoll() {
|
|
5704
|
+
this.lastPollUtcMs = Date.now();
|
|
5705
|
+
}
|
|
5706
|
+
/** This channel is served: its holder's polls are the liveness it reports. */
|
|
5707
|
+
lastHolderPollUtcMs() {
|
|
5708
|
+
return this.lastPollUtcMs;
|
|
5709
|
+
}
|
|
5672
5710
|
async init() {
|
|
5673
5711
|
const cursors = await this.cursorStorage.list(this.remoteName);
|
|
5674
5712
|
const inboxOrdinal = cursors.find((c) => c.cursorType === "inbox")?.cursorOrdinal ?? 0;
|
|
@@ -6348,9 +6386,19 @@ var OutboxMode = /* @__PURE__ */ function(OutboxMode) {
|
|
|
6348
6386
|
const defaultSyncManagerConfig = {
|
|
6349
6387
|
maxDeadLettersPerRemote: 100,
|
|
6350
6388
|
maxInboxBatchSize: 32,
|
|
6351
|
-
maxHeldOperationsPerRemote: 1e4
|
|
6389
|
+
maxHeldOperationsPerRemote: 1e4,
|
|
6390
|
+
staleRemotePollWindowMs: 5 * 6e4
|
|
6352
6391
|
};
|
|
6353
6392
|
const PLAN_KEY_TO_JOB_UUID_CAP = 1e4;
|
|
6393
|
+
/**
|
|
6394
|
+
* Whether a channel failure says the caller could not authenticate or could not
|
|
6395
|
+
* reach the remote, rather than that the remote itself is misconfigured. The
|
|
6396
|
+
* remote record stays on disk for these so a retry after sign-in can re-add it.
|
|
6397
|
+
*/
|
|
6398
|
+
function isCredentialOrNetworkError(error) {
|
|
6399
|
+
if (isDriveAuthError(error)) return true;
|
|
6400
|
+
return error instanceof GraphQLRequestError && error.category === "network";
|
|
6401
|
+
}
|
|
6354
6402
|
/** Where a sync operation's run of ordinals begins. */
|
|
6355
6403
|
function firstOrdinalOf(syncOp) {
|
|
6356
6404
|
return syncOp.operations.length > 0 ? syncOp.operations[0].context.ordinal : 0;
|
|
@@ -6379,6 +6427,11 @@ var SyncManager = class {
|
|
|
6379
6427
|
backfillAbortControllers = /* @__PURE__ */ new Map();
|
|
6380
6428
|
planKeyToJobUuid = /* @__PURE__ */ new Map();
|
|
6381
6429
|
evictedOutboxFloors = /* @__PURE__ */ new Map();
|
|
6430
|
+
prunePending = /* @__PURE__ */ new Set();
|
|
6431
|
+
pruneChain = Promise.resolve();
|
|
6432
|
+
derivingOutboxes = 0;
|
|
6433
|
+
pruneDrainDeferred = false;
|
|
6434
|
+
removing = /* @__PURE__ */ new Set();
|
|
6382
6435
|
lastEnqueuedJobIdByKey = /* @__PURE__ */ new Map();
|
|
6383
6436
|
inboxChunkChain = Promise.resolve();
|
|
6384
6437
|
constructor(logger, remoteStorage, cursorStorage, deadLetterStorage, channelFactory, operationIndex, reactor, eventBus, driveContainerTypes, config = {}) {
|
|
@@ -6430,7 +6483,7 @@ var SyncManager = class {
|
|
|
6430
6483
|
await channel.init();
|
|
6431
6484
|
} catch (error) {
|
|
6432
6485
|
this.logger.error("Error initializing channel for remote (@name, @error)", record.name, error instanceof Error ? error.message : String(error));
|
|
6433
|
-
this.
|
|
6486
|
+
await this.dropRemoteAfterFailedInit(remote, false);
|
|
6434
6487
|
continue;
|
|
6435
6488
|
}
|
|
6436
6489
|
const outboxAckOrdinal = remote.channel.outbox.ackOrdinal;
|
|
@@ -6442,6 +6495,7 @@ var SyncManager = class {
|
|
|
6442
6495
|
this.logger.error("Backfill failed for remote @RemoteName: @Error", remote.meta.name, error instanceof Error ? error : new Error(String(error)));
|
|
6443
6496
|
}).finally(() => {
|
|
6444
6497
|
this.backfillAbortControllers.delete(record.name);
|
|
6498
|
+
this.drainPrunes();
|
|
6445
6499
|
});
|
|
6446
6500
|
}
|
|
6447
6501
|
}
|
|
@@ -6455,6 +6509,8 @@ var SyncManager = class {
|
|
|
6455
6509
|
this.backfillAbortControllers.clear();
|
|
6456
6510
|
this.planKeyToJobUuid.clear();
|
|
6457
6511
|
this.lastEnqueuedJobIdByKey.clear();
|
|
6512
|
+
this.prunePending.clear();
|
|
6513
|
+
this.pruneDrainDeferred = false;
|
|
6458
6514
|
this.batchAggregator.clear();
|
|
6459
6515
|
if (this.eventUnsubscribe) {
|
|
6460
6516
|
this.eventUnsubscribe();
|
|
@@ -6548,8 +6604,7 @@ var SyncManager = class {
|
|
|
6548
6604
|
try {
|
|
6549
6605
|
await channel.init();
|
|
6550
6606
|
} catch (error) {
|
|
6551
|
-
this.
|
|
6552
|
-
await this.remoteStorage.remove(name);
|
|
6607
|
+
await this.dropRemoteAfterFailedInit(remote, !isCredentialOrNetworkError(error));
|
|
6553
6608
|
throw error;
|
|
6554
6609
|
}
|
|
6555
6610
|
const backfillController = new AbortController();
|
|
@@ -6559,6 +6614,7 @@ var SyncManager = class {
|
|
|
6559
6614
|
this.logger.error("Backfill failed for remote @RemoteName: @Error", remote.meta.name, error instanceof Error ? error : new Error(String(error)));
|
|
6560
6615
|
}).finally(() => {
|
|
6561
6616
|
this.backfillAbortControllers.delete(name);
|
|
6617
|
+
this.drainPrunes();
|
|
6562
6618
|
});
|
|
6563
6619
|
return remote;
|
|
6564
6620
|
}
|
|
@@ -6570,21 +6626,62 @@ var SyncManager = class {
|
|
|
6570
6626
|
async remove(name) {
|
|
6571
6627
|
const remote = this.remotes.get(name);
|
|
6572
6628
|
if (!remote) throw new Error(`Remote with name '${name}' does not exist`);
|
|
6629
|
+
this.removing.add(name);
|
|
6630
|
+
try {
|
|
6631
|
+
await this.teardownRemoteResources(remote);
|
|
6632
|
+
await this.remoteStorage.remove(name);
|
|
6633
|
+
await this.cursorStorage.remove(name);
|
|
6634
|
+
} finally {
|
|
6635
|
+
this.remotes.delete(name);
|
|
6636
|
+
this.removing.delete(name);
|
|
6637
|
+
}
|
|
6638
|
+
}
|
|
6639
|
+
/**
|
|
6640
|
+
* Drops a remote whose channel.init() rejected, optionally removing its
|
|
6641
|
+
* stored record. Ordered like remove(): the registry slot is released last,
|
|
6642
|
+
* so a concurrent add of the same name cannot slip in and have its record
|
|
6643
|
+
* deleted by the removal below. A failure to tear down must not replace the
|
|
6644
|
+
* init error the caller has to classify.
|
|
6645
|
+
*/
|
|
6646
|
+
async dropRemoteAfterFailedInit(remote, removeStorageRecord) {
|
|
6647
|
+
const name = remote.meta.name;
|
|
6648
|
+
this.removing.add(name);
|
|
6649
|
+
try {
|
|
6650
|
+
await this.teardownRemoteResources(remote);
|
|
6651
|
+
if (removeStorageRecord) await this.remoteStorage.remove(name);
|
|
6652
|
+
} catch (error) {
|
|
6653
|
+
this.logger.error("Error tearing down remote after failed init (@name, @error)", name, error instanceof Error ? error.message : String(error));
|
|
6654
|
+
} finally {
|
|
6655
|
+
this.remotes.delete(name);
|
|
6656
|
+
this.removing.delete(name);
|
|
6657
|
+
}
|
|
6658
|
+
}
|
|
6659
|
+
/**
|
|
6660
|
+
* Releases everything wiring a remote up holds: the in-flight backfill, the
|
|
6661
|
+
* channel, the status tracker entry and the connection-state subscription.
|
|
6662
|
+
* The registry slot is deliberately NOT released here -- the caller holds it
|
|
6663
|
+
* until any storage removal has finished, so a concurrent add of the same
|
|
6664
|
+
* name is refused for the whole teardown.
|
|
6665
|
+
*/
|
|
6666
|
+
async teardownRemoteResources(remote) {
|
|
6667
|
+
const name = remote.meta.name;
|
|
6573
6668
|
const backfillController = this.backfillAbortControllers.get(name);
|
|
6574
6669
|
if (backfillController) {
|
|
6575
6670
|
backfillController.abort();
|
|
6576
6671
|
this.backfillAbortControllers.delete(name);
|
|
6577
6672
|
}
|
|
6578
|
-
|
|
6579
|
-
|
|
6580
|
-
|
|
6581
|
-
|
|
6582
|
-
|
|
6583
|
-
|
|
6584
|
-
|
|
6585
|
-
|
|
6673
|
+
try {
|
|
6674
|
+
await remote.channel.shutdown();
|
|
6675
|
+
} finally {
|
|
6676
|
+
this.syncStatusTracker.untrackRemote(name);
|
|
6677
|
+
const unsub = this.connectionStateUnsubscribes.get(name);
|
|
6678
|
+
if (unsub) {
|
|
6679
|
+
unsub();
|
|
6680
|
+
this.connectionStateUnsubscribes.delete(name);
|
|
6681
|
+
}
|
|
6682
|
+
this.evictedOutboxFloors.delete(name);
|
|
6683
|
+
this.prunePending.delete(name);
|
|
6586
6684
|
}
|
|
6587
|
-
this.remotes.delete(name);
|
|
6588
6685
|
}
|
|
6589
6686
|
list() {
|
|
6590
6687
|
return Array.from(this.remotes.values());
|
|
@@ -6679,7 +6776,7 @@ var SyncManager = class {
|
|
|
6679
6776
|
this.logger.debug("Loaded @count persisted dead letters for remote @name", records.length, remote.meta.name);
|
|
6680
6777
|
}
|
|
6681
6778
|
getRemotesForCollection(collectionId) {
|
|
6682
|
-
return Array.from(this.remotes.values()).filter((remote) => remote.meta.collectionId.key === collectionId);
|
|
6779
|
+
return Array.from(this.remotes.values()).filter((remote) => remote.meta.collectionId.key === collectionId && !this.removing.has(remote.meta.name));
|
|
6683
6780
|
}
|
|
6684
6781
|
async processCompleteBatch(batch) {
|
|
6685
6782
|
if (this.isShutdown) return;
|
|
@@ -6690,7 +6787,11 @@ var SyncManager = class {
|
|
|
6690
6787
|
for (const remote of remotes) if (!affectedRemotes.includes(remote)) affectedRemotes.push(remote);
|
|
6691
6788
|
}
|
|
6692
6789
|
for (const remote of affectedRemotes) trimMailboxFromBatch(remote.channel.inbox, batch);
|
|
6693
|
-
for (const remote of affectedRemotes)
|
|
6790
|
+
for (const remote of affectedRemotes) {
|
|
6791
|
+
if (!this.remotes.has(remote.meta.name) || this.removing.has(remote.meta.name)) continue;
|
|
6792
|
+
await this.updateOutbox(remote, remote.channel.outbox.latestOrdinal, OutboxMode.BatchTriggered);
|
|
6793
|
+
}
|
|
6794
|
+
await this.drainPrunes();
|
|
6694
6795
|
}
|
|
6695
6796
|
handleInboxAdded(remote, syncOps) {
|
|
6696
6797
|
if (this.isShutdown) return;
|
|
@@ -6891,15 +6992,95 @@ var SyncManager = class {
|
|
|
6891
6992
|
const floor = firstOrdinalOf(evicted[0]);
|
|
6892
6993
|
const known = this.evictedOutboxFloors.get(remote.meta.name);
|
|
6893
6994
|
this.evictedOutboxFloors.set(remote.meta.name, known === void 0 ? floor : Math.min(known, floor));
|
|
6894
|
-
this.
|
|
6995
|
+
const staleMs = this.stalePollAgeMs(remote);
|
|
6996
|
+
if (staleMs !== void 0) {
|
|
6997
|
+
let held = kept;
|
|
6998
|
+
for (const syncOp of evicted) held += syncOp.operations.length;
|
|
6999
|
+
if (!this.prunePending.has(remote.meta.name)) {
|
|
7000
|
+
this.prunePending.add(remote.meta.name);
|
|
7001
|
+
this.logger.warn("Outbox for @RemoteName (@Collection) is past its bound of @Cap operations holding @Held, and it has not been polled for @StaleMs ms; marking the channel for removal once this derivation ends", remote.meta.name, remote.meta.collectionId.key, cap, held, staleMs);
|
|
7002
|
+
}
|
|
7003
|
+
} else this.logger.warn("Outbox for @RemoteName is past its bound of @Cap operations; evicting @Count entries from ordinal @Floor, to be derived again once it drains", remote.meta.name, cap, evicted.length, floor);
|
|
6895
7004
|
remote.channel.outbox.remove(...evicted);
|
|
6896
7005
|
}
|
|
7006
|
+
/**
|
|
7007
|
+
* How long a served remote's holder has been silent, if past the window.
|
|
7008
|
+
*
|
|
7009
|
+
* The channel is asked, rather than its config inspected: SyncManager serves
|
|
7010
|
+
* and subscribes with the same interface, both kinds report lastSuccessUtcMs,
|
|
7011
|
+
* and the caller-supplied channelConfig.type is a free-form string the
|
|
7012
|
+
* factories do not read -- so neither could tell a dead served channel from a
|
|
7013
|
+
* client whose switchboard is merely unreachable. Only a channel that claims
|
|
7014
|
+
* a holder by reporting when it last heard from one can be removed for that
|
|
7015
|
+
* holder's silence; a channel that reports nothing (or 0) is never pruned.
|
|
7016
|
+
*/
|
|
7017
|
+
stalePollAgeMs(remote) {
|
|
7018
|
+
const last = remote.channel.lastHolderPollUtcMs();
|
|
7019
|
+
if (last === void 0 || last <= 0) return;
|
|
7020
|
+
const age = Date.now() - last;
|
|
7021
|
+
return age >= this.config.staleRemotePollWindowMs ? age : void 0;
|
|
7022
|
+
}
|
|
7023
|
+
/**
|
|
7024
|
+
* Removes the remotes marked stale during eviction.
|
|
7025
|
+
*
|
|
7026
|
+
* Removing a remote that a derivation is still iterating would pull its
|
|
7027
|
+
* mailboxes out from under it, so a drain that arrives during one is deferred
|
|
7028
|
+
* rather than run; the last derivation to finish re-arms it.
|
|
7029
|
+
*/
|
|
7030
|
+
drainPrunes() {
|
|
7031
|
+
if (this.derivingOutboxes > 0) {
|
|
7032
|
+
this.pruneDrainDeferred = true;
|
|
7033
|
+
return Promise.resolve();
|
|
7034
|
+
}
|
|
7035
|
+
const next = this.pruneChain.then(async () => {
|
|
7036
|
+
if (this.isShutdown) return;
|
|
7037
|
+
for (const name of [...this.prunePending]) {
|
|
7038
|
+
if (this.derivingOutboxes > 0) {
|
|
7039
|
+
this.pruneDrainDeferred = true;
|
|
7040
|
+
return;
|
|
7041
|
+
}
|
|
7042
|
+
this.prunePending.delete(name);
|
|
7043
|
+
const remote = this.remotes.get(name);
|
|
7044
|
+
if (!remote) continue;
|
|
7045
|
+
if (this.stalePollAgeMs(remote) === void 0) {
|
|
7046
|
+
this.logger.info("Stale removal of @name revoked: its holder polled while the outbox was being derived", name);
|
|
7047
|
+
continue;
|
|
7048
|
+
}
|
|
7049
|
+
try {
|
|
7050
|
+
await this.remove(name);
|
|
7051
|
+
} catch (error) {
|
|
7052
|
+
this.logger.error("Failed to remove stale remote (@name, @error)", name, error instanceof Error ? error.message : String(error));
|
|
7053
|
+
}
|
|
7054
|
+
}
|
|
7055
|
+
});
|
|
7056
|
+
this.pruneChain = next.catch(() => {});
|
|
7057
|
+
return next;
|
|
7058
|
+
}
|
|
6897
7059
|
outboxOperationCount(remote) {
|
|
6898
7060
|
let count = 0;
|
|
6899
7061
|
for (const syncOp of remote.channel.outbox.items) count += syncOp.operations.length;
|
|
6900
7062
|
return count;
|
|
6901
7063
|
}
|
|
7064
|
+
/**
|
|
7065
|
+
* Derives this remote's outbox, holding off prunes for the duration.
|
|
7066
|
+
*
|
|
7067
|
+
* A backfill elsewhere can finish at any await in here and drain the prunes
|
|
7068
|
+
* it marked; the count is what keeps that drain from removing the remote this
|
|
7069
|
+
* call is still adding to.
|
|
7070
|
+
*/
|
|
6902
7071
|
async updateOutbox(remote, ackOrdinal, mode = OutboxMode.Backfill, signal) {
|
|
7072
|
+
this.derivingOutboxes++;
|
|
7073
|
+
try {
|
|
7074
|
+
await this.deriveOutbox(remote, ackOrdinal, mode, signal);
|
|
7075
|
+
} finally {
|
|
7076
|
+
this.derivingOutboxes--;
|
|
7077
|
+
if (this.derivingOutboxes === 0 && this.pruneDrainDeferred) {
|
|
7078
|
+
this.pruneDrainDeferred = false;
|
|
7079
|
+
this.drainPrunes();
|
|
7080
|
+
}
|
|
7081
|
+
}
|
|
7082
|
+
}
|
|
7083
|
+
async deriveOutbox(remote, ackOrdinal, mode, signal) {
|
|
6903
7084
|
const composedSignal = signal ? AbortSignal.any([signal, this.abortController.signal]) : this.abortController.signal;
|
|
6904
7085
|
const startOrdinal = this.refillOrdinal(remote, ackOrdinal);
|
|
6905
7086
|
let maxOrdinal = startOrdinal;
|
|
@@ -7001,6 +7182,10 @@ var SyncBuilder = class {
|
|
|
7001
7182
|
this.config.maxHeldOperationsPerRemote = limit;
|
|
7002
7183
|
return this;
|
|
7003
7184
|
}
|
|
7185
|
+
withStaleRemotePollWindowMs(windowMs) {
|
|
7186
|
+
this.config.staleRemotePollWindowMs = windowMs;
|
|
7187
|
+
return this;
|
|
7188
|
+
}
|
|
7004
7189
|
build(reactor, logger, operationIndex, eventBus, db, driveContainerTypes) {
|
|
7005
7190
|
return this.buildModule(reactor, logger, operationIndex, eventBus, db, driveContainerTypes).syncManager;
|
|
7006
7191
|
}
|
|
@@ -7903,19 +8088,27 @@ var ReactorBuilder = class {
|
|
|
7903
8088
|
} else executorManager = new SimpleJobExecutorManager(() => new SimpleJobExecutor(this.logger, documentModelRegistry, operationStore, eventBus, writeCache, operationIndex, documentMetaCache, collectionMembershipCache, this.driveContainerTypes, this.executorConfig, this.signatureVerifier, executionScope), eventBus, queue, jobTracker, this.logger, resolver, this.executorConfig.jobTimeoutMs, this.executorConfig.deferredJobTtlMs);
|
|
7904
8089
|
await executorManager.start(executorStartCount);
|
|
7905
8090
|
const callerReadModels = Array.from(new Set([...this.readModels]));
|
|
8091
|
+
const degradedComponents = [];
|
|
8092
|
+
const startDegraded = (component, error) => {
|
|
8093
|
+
degradedComponents.push({
|
|
8094
|
+
component,
|
|
8095
|
+
error: error instanceof Error ? error : new Error(String(error))
|
|
8096
|
+
});
|
|
8097
|
+
this.logger?.error("Reactor component started degraded: @component", component, error);
|
|
8098
|
+
};
|
|
7906
8099
|
const documentViewConsistencyTracker = new ConsistencyTracker();
|
|
7907
8100
|
const documentView = new KyselyDocumentView(database, operationStore, operationIndex, writeCache, documentViewConsistencyTracker, featureFlags.documentDecisions);
|
|
7908
8101
|
try {
|
|
7909
8102
|
await documentView.init();
|
|
7910
8103
|
} catch (error) {
|
|
7911
|
-
|
|
8104
|
+
startDegraded("document view", error);
|
|
7912
8105
|
}
|
|
7913
8106
|
const documentIndexerConsistencyTracker = new ConsistencyTracker();
|
|
7914
8107
|
const documentIndexer = new KyselyDocumentIndexer(database, operationIndex, writeCache, documentIndexerConsistencyTracker);
|
|
7915
8108
|
try {
|
|
7916
8109
|
await documentIndexer.init();
|
|
7917
8110
|
} catch (error) {
|
|
7918
|
-
|
|
8111
|
+
startDegraded("document indexer", error);
|
|
7919
8112
|
}
|
|
7920
8113
|
const subscriptionManager = new ReactorSubscriptionManager(new DefaultSubscriptionErrorHandler());
|
|
7921
8114
|
const subscriptionNotificationReadModel = new SubscriptionNotificationReadModel(subscriptionManager, documentView);
|
|
@@ -7924,9 +8117,9 @@ var ReactorBuilder = class {
|
|
|
7924
8117
|
try {
|
|
7925
8118
|
await processorManager.init();
|
|
7926
8119
|
} catch (error) {
|
|
7927
|
-
|
|
8120
|
+
startDegraded("processor manager", error);
|
|
7928
8121
|
}
|
|
7929
|
-
for (const factory of this.readModelFactories) {
|
|
8122
|
+
for (const [index, factory] of this.readModelFactories.entries()) try {
|
|
7930
8123
|
const readModel = await factory({
|
|
7931
8124
|
documentModelRegistry,
|
|
7932
8125
|
operationIndex,
|
|
@@ -7934,6 +8127,8 @@ var ReactorBuilder = class {
|
|
|
7934
8127
|
processorManagerConsistencyTracker
|
|
7935
8128
|
});
|
|
7936
8129
|
callerReadModels.push(readModel);
|
|
8130
|
+
} catch (error) {
|
|
8131
|
+
startDegraded(`read model ${index}${factory.name ? ` (${factory.name})` : ""}`, error);
|
|
7937
8132
|
}
|
|
7938
8133
|
const readModelInstances = [
|
|
7939
8134
|
...callerReadModels,
|
|
@@ -7993,8 +8188,10 @@ var ReactorBuilder = class {
|
|
|
7993
8188
|
syncModule,
|
|
7994
8189
|
reactor,
|
|
7995
8190
|
groupReevaluationTrigger,
|
|
7996
|
-
pools: this.instrumentedPools
|
|
8191
|
+
pools: this.instrumentedPools,
|
|
8192
|
+
degradedComponents
|
|
7997
8193
|
};
|
|
8194
|
+
if (degradedComponents.length > 0) this.logger.warn("Reactor started with @count degraded component(s): @components", degradedComponents.length, degradedComponents.map(({ component }) => component).join(", "));
|
|
7998
8195
|
if (this.signalHandlersEnabled) this.attachSignalHandlers(module);
|
|
7999
8196
|
return module;
|
|
8000
8197
|
}
|
|
@@ -8792,6 +8989,6 @@ var DocumentIntegrityService = class {
|
|
|
8792
8989
|
}
|
|
8793
8990
|
};
|
|
8794
8991
|
//#endregion
|
|
8795
|
-
export { ALWAYS_READABLE_SCOPES, APPEND_CONDITION_FAILED_PREFIX, AppendConditionFailedError, AuthEnforcementDisabledError, BareReadGate, BaseReadModel, ChannelError, ChannelErrorSource, ChannelScheme, ConsistencyTracker, DEFAULT_DRIVE_CONTAINER_TYPES, DOCUMENT_INDEXER_READ_MODEL, DOCUMENT_VIEW_READ_MODEL, DRIVE_AUTH_ERROR_MESSAGES, DefaultSubscriptionErrorHandler, DocumentAlreadyExistsError, DocumentChangeType, DocumentExistence, DocumentIntegrityService, DocumentModelRegistry, DocumentModelResolver, DriveClient, DriveCollectionId, DuplicateManifestError, DuplicateModuleError, DuplicateOperationError, EventBus, EventBusAggregateError, GqlRequestChannel, GqlRequestChannelFactory, GqlResponseChannel, GqlResponseChannelFactory, HybridProjectionCoordinator, SimpleJobExecutor as InMemoryJobExecutor, SimpleJobExecutor, InMemoryJobTracker, InMemoryQueue, IntervalPollTimer, InvalidModuleError, JobAwaiter, JobExecutorEventTypes, JobStatus, KyselyDocumentIndexer, KyselyDocumentView, KyselyKeyframeStore, KyselyOperationStore, KyselySyncCursorStorage, KyselySyncRemoteStorage, KyselyWriteCache, Mailbox, ModelReadGate, ModuleNotFoundError, NullDocumentModelResolver, OptimisticLockError, PollBehavior, PollingChannelError, ProcessorManager, PropagationMode, QueueEventTypes, REACTOR_SCHEMA, RECOVERABLE_GRAPHQL_ERROR_CODES, Reactor, ReactorBuilder, ReactorClient, ReactorClientBuilder, ReactorEventTypes, ReactorSubscriptionManager, ReadModelCoordinator, RelationalDbProcessor, RelationshipChangeType, RetryAccounting, RevisionMismatchError, SeededStateReader, SimpleJobExecutorManager, SyncBuilder, SyncEventTypes, SyncOperation, SyncOperationAggregateError, SyncOperationStatus, SyncScopeGate, SyncStatus, SyncStatusTracker, addRelationshipAction, authDecisionModel, batchOperationsByDocument, buildDecisionModel, classifyJobFailure, consolidateSyncOperations, createDocumentAction, createForwardingLogger, createHybridProjectionCoordinatorFactory, createMutableShutdownStatus, createReactorHostModuleBase, createRelationalDb, decideAtHead, deleteDocumentAction, documentActions, documentDecisionModel, driveIdFromUrl, envelopesToSyncOperations, errorToInfo, firstOutOfOrderPair, getMigrationStatus, instrumentPgPool, isDriveAuthError, isRecoverableGraphQLError, makeConsistencyKey, parseDriveUrl, parsePagingOptions, quarantinesDocument, readDecisionModel, removeRelationshipAction, runMigrations, sanitizeArg, selectDecisionModel, supportsLiveReadModelRegistration, syncOperationErrorType, trimMailboxFromAckOrdinal, updateRelationshipAction, upgradeDocumentAction, workerEntryPath };
|
|
8992
|
+
export { ALWAYS_READABLE_SCOPES, APPEND_CONDITION_FAILED_PREFIX, AppendConditionFailedError, AuthEnforcementDisabledError, BareReadGate, BaseReadModel, ChannelError, ChannelErrorSource, ChannelScheme, ConsistencyTracker, DEFAULT_DRIVE_CONTAINER_TYPES, DOCUMENT_INDEXER_READ_MODEL, DOCUMENT_VIEW_READ_MODEL, DRIVE_AUTH_ERROR_MESSAGES, DefaultSubscriptionErrorHandler, DocumentAlreadyExistsError, DocumentChangeType, DocumentExistence, DocumentIntegrityService, DocumentModelRegistry, DocumentModelResolver, DriveClient, DriveCollectionId, DriveRequestError, DuplicateManifestError, DuplicateModuleError, DuplicateOperationError, EventBus, EventBusAggregateError, GqlRequestChannel, GqlRequestChannelFactory, GqlResponseChannel, GqlResponseChannelFactory, HybridProjectionCoordinator, SimpleJobExecutor as InMemoryJobExecutor, SimpleJobExecutor, InMemoryJobTracker, InMemoryQueue, IntervalPollTimer, InvalidModuleError, JobAwaiter, JobExecutorEventTypes, JobStatus, KyselyDocumentIndexer, KyselyDocumentView, KyselyKeyframeStore, KyselyOperationStore, KyselySyncCursorStorage, KyselySyncRemoteStorage, KyselyWriteCache, Mailbox, ModelReadGate, ModuleNotFoundError, NullDocumentModelResolver, OptimisticLockError, PollBehavior, PollingChannelError, ProcessorManager, PropagationMode, QueueEventTypes, REACTOR_SCHEMA, RECOVERABLE_GRAPHQL_ERROR_CODES, Reactor, ReactorBuilder, ReactorClient, ReactorClientBuilder, ReactorEventTypes, ReactorSubscriptionManager, ReadModelCoordinator, RelationalDbProcessor, RelationshipChangeType, RetryAccounting, RevisionMismatchError, SeededStateReader, SimpleJobExecutorManager, SyncBuilder, SyncEventTypes, SyncOperation, SyncOperationAggregateError, SyncOperationStatus, SyncScopeGate, SyncStatus, SyncStatusTracker, addRelationshipAction, authDecisionModel, batchOperationsByDocument, buildDecisionModel, classifyJobFailure, consolidateSyncOperations, createDocumentAction, createForwardingLogger, createHybridProjectionCoordinatorFactory, createMutableShutdownStatus, createReactorHostModuleBase, createRelationalDb, decideAtHead, deleteDocumentAction, documentActions, documentDecisionModel, driveIdFromUrl, envelopesToSyncOperations, errorToInfo, firstOutOfOrderPair, getMigrationStatus, instrumentPgPool, isDriveAuthError, isRecoverableGraphQLError, makeConsistencyKey, parseDriveUrl, parsePagingOptions, quarantinesDocument, readDecisionModel, removeRelationshipAction, runMigrations, sanitizeArg, selectDecisionModel, supportsLiveReadModelRegistration, syncOperationErrorType, trimMailboxFromAckOrdinal, updateRelationshipAction, upgradeDocumentAction, workerEntryPath };
|
|
8796
8993
|
|
|
8797
8994
|
//# sourceMappingURL=index.js.map
|