@juzi/wechaty-puppet-service 1.0.130 → 1.0.132

package/dist/cjs/src/client/puppet-service.js

@@ -56,7 +56,9 @@ const config_js_1 = require("../config.js");
 const event_type_rev_js_1 = require("../event-type-rev.js");
 const package_json_js_1 = require("../package-json.js");
 const grpc_manager_js_1 = require("./grpc-manager.js");
+const recovery_js_1 = require("./recovery.js");
 const payload_store_js_1 = require("./payload-store.js");
+const room_cache_coordinator_js_1 = require("./room-cache-coordinator.js");
 const pb_payload_helper_js_1 = require("../utils/pb-payload-helper.js");
 const call_media_mapping_js_1 = require("../utils/call-media-mapping.js");
 const gerror_1 = require("gerror");
@@ -76,7 +78,28 @@ class PuppetService extends PUPPET.Puppet {
     options;
     static VERSION = config_js_1.VERSION;
     _payloadStore;
+    roomCache = new room_cache_coordinator_js_1.RoomCacheCoordinator();
+    eventQueue = Promise.resolve();
+    eventEpoch = 0;
+    eventsBlocked = false;
+    failedEventSeq;
+    activeEventSeq;
+    pendingEventSeq;
+    checkpointQueue = Promise.resolve();
+    pendingCheckpointRollback;
     timeoutMilliseconds;
+    recovery;
+    recoveryTimer;
+    inactivityTimer;
+    recoveryStopped = true;
+    recoveryProcessedSeq = 0;
+    get recoveryState() { return this.recovery.state; }
+    get eventMetadataEnabled() {
+        return this.recovery.enabled || !config_js_1.envVars.WECHATY_PUPPET_SERVICE_DISABLE_EVENT_CACHE();
+    }
+    withRecoveryRequestContext(context, operation) {
+        return (0, recovery_js_1.withRecoveryRequestContext)(context, operation);
+    }
     _grpcManager;
     get grpcManager() {
         if (!this._grpcManager) {
@@ -94,9 +117,14 @@ class PuppetService extends PUPPET.Puppet {
     constructor(options = {}) {
         super(options);
         this.options = options;
+        this.recovery = new recovery_js_1.RecoveryGate(options.recovery, () => {
+            this.stopRecovery('Recovery budget exhausted');
+            this.reportStreamError(new Error('Recovery budget exhausted'));
+        });
         this._payloadStore = new payload_store_js_1.PayloadStore({
             token: config_js_1.envVars.WECHATY_PUPPET_SERVICE_TOKEN(this.options.token),
             logger: this.options.logger,
+            recoveryCursor: this.recovery.enabled,
         });
         this.hookPayloadStore();
         this.FileBoxUuid = (0, mod_js_1.uuidifyFileBoxGrpc)(() => this.grpcManager.client);
@@ -129,12 +157,14 @@ class PuppetService extends PUPPET.Puppet {
         this.log.verbose('PuppetService', 'onStart()');
         this.waitingForLogin = false;
         this.waitingForReady = false;
+        this.suspendEventProcessing();
+        this.eventsBlocked = false;
         if (this._grpcManager) {
             this.log.warn('PuppetService', 'onStart() found this.grpc is already existed. dropped.');
             this._grpcManager = undefined;
         }
         this.log.info('PuppetService', 'start() instanciating GrpcManager ...');
-        const grpcManager = new grpc_manager_js_1.GrpcManager(this.options);
+        const grpcManager = new grpc_manager_js_1.GrpcManager(this.options, this.recovery);
         this.log.info('PuppetService', 'start() instanciating GrpcManager ... done');
         /**
          * Huan(202108): when we started the event stream,
@@ -145,8 +175,20 @@ class PuppetService extends PUPPET.Puppet {
         this.bridgeGrpcEventStream(grpcManager);
         this.log.info('PuppetService', 'start() setting up bridge grpc event stream ... done');
         this.log.info('PuppetService', 'start() starting grpc manager...');
-        const { lastEventSeq, accountId } = await this.getMiscellaneousStoreData();
-        await grpcManager.start(lastEventSeq, accountId);
+        const { lastEventSeq, accountId } = await this.eventResumePosition();
+        this.recoveryStopped = false;
+        this.recovery.begin(accountId, true);
+        this.recoveryProcessedSeq = Number(lastEventSeq ?? 0);
+        this.touchRecoveryStream();
+        try {
+            await grpcManager.start(lastEventSeq, accountId);
+        }
+        catch (error) {
+            if (!this.recovery.enabled) {
+                throw error;
+            }
+            this.requestRecovery(grpcManager);
+        }
         this.log.info('PuppetService', 'start() starting grpc manager... done');
         this.log.info('PuppetService', 'start healthCheck');
         this.startHealthCheck();
@@ -154,11 +196,14 @@ class PuppetService extends PUPPET.Puppet {
     }
     async onStop() {
         this.log.info('PuppetService', 'onStop()');
+        this.stopRecovery('Puppet stopped');
+        this.suspendEventProcessing();
+        await this.checkpointQueue;
         if (this._grpcManager) {
             this.log.info('PuppetService', 'onStop() stopping grpc manager ...');
             const grpcManager = this._grpcManager;
             this._grpcManager = undefined;
-            await grpcManager.stop();
+            await this.recovery.runDetached(async () => grpcManager.stop());
             this.log.info('PuppetService', 'onStop() stopping grpc manager ... done');
         }
         this.log.info('PuppetService', 'onStop() ... done');
@@ -189,11 +234,79 @@ class PuppetService extends PUPPET.Puppet {
     bridgeGrpcEventStream(client) {
         this.log.verbose('PuppetService', 'bridgeGrpcEventStream(client)');
         client
-            .on('data', this.onGrpcStreamEvent.bind(this))
+            .on('data', (event) => {
+            if (client !== this._grpcManager || this.eventsBlocked) {
+                return;
+            }
+            this.touchRecoveryStream();
+            if (this.recovery.enabled && event.getType() === wechaty_grpc_1.puppet.EventType.EVENT_TYPE_LOGIN) {
+                try {
+                    const payload = JSON.parse(event.getPayload());
+                    if (!this.recovery.checkAccount(payload.contactId)) {
+                        this.stopRecovery('Account changed or invalid login');
+                        this.reportStreamError(new Error('Account changed or invalid login'));
+                        return;
+                    }
+                }
+                catch (error) {
+                    this.stopRecovery('Invalid login payload');
+                    this.reportStreamError(error);
+                    return;
+                }
+            }
+            if (this.recovery.enabled && event.getType() === wechaty_grpc_1.puppet.EventType.EVENT_TYPE_LOGOUT) {
+                this.stopRecovery('Account logged out', false);
+            }
+            const epoch = this.eventEpoch;
+            this.eventQueue = this.eventQueue.then(async () => {
+                if (!this.ownsEventWorker(client, epoch)) {
+                    return undefined;
+                }
+                this.activeEventSeq = event.getSeq();
+                try {
+                    if (this.failedEventSeq !== undefined && event.getSeq() > this.failedEventSeq) {
+                        throw new Error(`Replay skipped failed event ${this.failedEventSeq}; stream halted until reset`);
+                    }
+                    await this.recovery.runEvent(() => this.ownsEventWorker(client, epoch), async () => this.onGrpcStreamEvent(event, epoch));
+                    if (!this.ownsEventWorker(client, epoch)) {
+                        return undefined;
+                    }
+                    if (event.getSeq()) {
+                        this.recoveryProcessedSeq = event.getSeq();
+                    }
+                    if (event.getSeq() === this.failedEventSeq) {
+                        this.failedEventSeq = undefined;
+                    }
+                }
+                catch (error) {
+                    if (!this.ownsEventWorker(client, epoch)) {
+                        this.log.error('PuppetService', 'superseded event worker failed: %s', error);
+                        return undefined;
+                    }
+                    this.haltEventStream(client, event, error);
+                }
+                if (epoch === this.eventEpoch) {
+                    this.activeEventSeq = undefined;
+                }
+                return undefined;
+            }).catch((error) => { this.reportStreamError(error); });
+        })
             .on('end', () => {
+            if (this.recovery.enabled) {
+                this.requestRecovery(client);
+            }
             this.log.verbose('PuppetService', 'bridgeGrpcEventStream() eventStream.on(end)');
         })
             .on('error', (e) => {
+            if (this.recovery.enabled) {
+                if (this.recoveryStopped || client !== this._grpcManager) {
+                    this.reportStreamError(e);
+                }
+                else {
+                    this.requestRecovery(client);
+                }
+                return;
+            }
             this.emit('error', e);
             // https://github.com/wechaty/wechaty-puppet-service/issues/16
             // this.log.verbose('PuppetService', 'bridgeGrpcEventStream() eventStream.on(error) %s', e)
@@ -211,10 +324,143 @@ class PuppetService extends PUPPET.Puppet {
             // }
         })
             .on('cancel', (...args) => {
+            if (this.recovery.enabled) {
+                this.requestRecovery(client);
+            }
             this.log.verbose('PuppetService', 'bridgeGrpcEventStream() eventStream.on(cancel), %s', JSON.stringify(args));
         });
     }
-    async onGrpcStreamEvent(event) {
+    suspendEventProcessing() {
+        if (this.activeEventSeq) {
+            this.pendingEventSeq ??= this.activeEventSeq;
+        }
+        this.eventEpoch++;
+        this.eventsBlocked = true;
+        this.activeEventSeq = undefined;
+        this.eventQueue = Promise.resolve();
+    }
+    ownsEventWorker(manager, epoch) {
+        return manager === this._grpcManager && this.isEventEpochCurrent(epoch);
+    }
+    isEventEpochCurrent(epoch) {
+        return epoch === this.eventEpoch && !this.eventsBlocked;
+    }
+    async eventResumePosition(epoch = this.eventEpoch) {
+        return this.withEventCheckpoint(async () => {
+            if (this.recovery.enabled && !this._payloadStore.miscellaneous) {
+                throw new Error('Recovery cursor storage is unavailable');
+            }
+            if (this.pendingCheckpointRollback) {
+                await this.restoreEventMetadata(this.pendingCheckpointRollback);
+                this.pendingCheckpointRollback = undefined;
+            }
+            const lastEventSeq = await this._payloadStore.miscellaneous?.get('lastEventSeq');
+            if (epoch !== this.eventEpoch) {
+                return { lastEventSeq };
+            }
+            if (this.pendingEventSeq !== undefined && this.pendingEventSeq > Number(lastEventSeq ?? 0)) {
+                this.failedEventSeq ??= this.pendingEventSeq;
+            }
+            this.pendingEventSeq = undefined;
+            return { lastEventSeq, accountId: await this._payloadStore.miscellaneous?.get('accountId') };
+        });
+    }
+    async withEventCheckpoint(operation) {
+        const result = this.checkpointQueue.then(operation);
+        this.checkpointQueue = result.then(() => undefined, () => undefined);
+        return result;
+    }
+    async restoreEventMetadata(previous) {
+        for (const [key, value] of previous) {
+            if (value === undefined) {
+                await this._payloadStore.miscellaneous?.delete(key);
+            }
+            else {
+                await this._payloadStore.miscellaneous?.set(key, value);
+            }
+        }
+    }
+    async writeEventMetadata(changes, epoch) {
+        const isCurrent = () => epoch === undefined || this.isEventEpochCurrent(epoch);
+        if (!isCurrent()) {
+            return;
+        }
+        const previous = new Map();
+        for (const key of changes.keys()) {
+            previous.set(key, await this._payloadStore.miscellaneous?.get(key));
+        }
+        if (!isCurrent()) {
+            return;
+        }
+        try {
+            for (const [key, value] of changes) {
+                if (value === undefined) {
+                    await this._payloadStore.miscellaneous?.delete(key);
+                }
+                else {
+                    await this._payloadStore.miscellaneous?.set(key, value);
+                }
+                if (!isCurrent()) {
+                    await this.restoreEventMetadata(previous);
+                    return;
+                }
+            }
+        }
+        catch (error) {
+            try {
+                await this.restoreEventMetadata(previous);
+            }
+            catch (rollbackError) {
+                this.pendingCheckpointRollback = previous;
+                this.log.error('PuppetService', 'checkpoint rollback failed: %s', rollbackError);
+            }
+            throw error;
+        }
+    }
+    async commitEventCheckpoint(seq, timestamp, epoch) {
+        await this.withEventCheckpoint(async () => {
+            if (!this.isEventEpochCurrent(epoch)) {
+                return;
+            }
+            const lastEventSeq = await this._payloadStore.miscellaneous?.get('lastEventSeq');
+            if (lastEventSeq && seq <= Number(lastEventSeq) && seq !== 1) {
+                return;
+            }
+            await this.writeEventMetadata(new Map([
+                ['lastEventTimestamp', timestamp], ['lastEventSeq', String(seq)],
+            ]), epoch);
+        });
+    }
+    reportStreamError(error) {
+        this.log.error('PuppetService', 'event processing failed: %s', error);
+        try {
+            this.emit('error', error);
+        }
+        catch (listenerError) {
+            this.log.error('PuppetService', 'error listener failed: %s', listenerError);
+        }
+    }
+    haltEventStream(manager, event, error) {
+        this.failedEventSeq ??= event.getSeq();
+        this.suspendEventProcessing();
+        try {
+            manager.stopStream();
+        }
+        catch (stopError) {
+            this.log.error('PuppetService', 'stopping failed event stream failed: %s', stopError);
+        }
+        if (this.recovery.enabled) {
+            this.log.error('PuppetService', 'recovering after event processing failure: %s', error);
+            this.requestRecovery(manager);
+        }
+        else {
+            this.reportStreamError(error);
+        }
+    }
+    async onGrpcStreamEvent(event, epoch = this.eventEpoch) {
+        if (!this.isEventEpochCurrent(epoch)) {
+            return;
+        }
         const type = event.getType();
         const payload = event.getPayload();
         const seq = event.getSeq();
@@ -228,15 +474,6 @@ class PuppetService extends PUPPET.Puppet {
                 data: `onGrpcStreamEvent(${event_type_rev_js_1.EventTypeRev[type]})`,
             });
         }
-        if (seq && !config_js_1.envVars.WECHATY_PUPPET_SERVICE_DISABLE_EVENT_CACHE()) {
-            const { lastEventSeq } = await this.getMiscellaneousStoreData();
-            if (!lastEventSeq || (seq > Number(lastEventSeq) || seq === 1)) {
-                await this.setMiscellaneousStoreData({
-                    lastEventSeq: seq.toString(),
-                    lastEventTimestamp: timestamp,
-                });
-            }
-        }
         switch (type) {
             case wechaty_grpc_1.puppet.EventType.EVENT_TYPE_DONG:
                 this.emit('dong', JSON.parse(payload));
@@ -252,28 +489,51 @@ class PuppetService extends PUPPET.Puppet {
                 break;
             case wechaty_grpc_1.puppet.EventType.EVENT_TYPE_LOGIN:
                 {
+                    const loginPayload = JSON.parse(payload);
+                    if (this.recovery.enabled) {
+                        this.recovery.login(loginPayload.contactId);
+                        if (this.recovery.state.status === 'offline') {
+                            this.stopRecovery('Account changed during recovery');
+                            this.reportStreamError(new Error('Account changed during recovery'));
+                            return;
+                        }
+                        if (this.isLoggedIn) {
+                            break;
+                        }
+                    }
                     if (this.waitingForLogin && this.isLoggedIn) {
                         this.log.warn('PuppetService', 'this login event is ignored because the it is expected by event stream reconnect and this puppet is already logged in');
                         return;
                     }
-                    const loginPayload = JSON.parse(payload);
-                    if (!config_js_1.envVars.WECHATY_PUPPET_SERVICE_DISABLE_EVENT_CACHE()) {
+                    if (this.eventMetadataEnabled) {
                         const { accountId } = await this.getMiscellaneousStoreData();
+                        if (!this.isEventEpochCurrent(epoch)) {
+                            return;
+                        }
                         if (accountId !== loginPayload.contactId) {
-                            await this.resetMiscellaneousStoreData();
+                            await this.resetMiscellaneousStoreData(epoch);
+                            if (!this.isEventEpochCurrent(epoch)) {
+                                return;
+                            }
                             await this.setMiscellaneousStoreData({
                                 accountId: loginPayload.contactId,
-                            });
+                            }, epoch);
+                            if (!this.isEventEpochCurrent(epoch)) {
+                                return;
+                            }
                         }
                     }
+                    if (!this.isEventEpochCurrent(epoch)) {
+                        return;
+                    }
                     (async () => this.login(loginPayload.contactId))().catch(e => this.log.error('PuppetService', 'onGrpcStreamEvent() this.login() rejection %s', e.message));
                 }
                 break;
             case wechaty_grpc_1.puppet.EventType.EVENT_TYPE_LOGOUT:
                 {
                     const logoutPayload = JSON.parse(payload);
-                    if (!config_js_1.envVars.WECHATY_PUPPET_SERVICE_DISABLE_EVENT_CACHE()) {
-                        await this.resetMiscellaneousStoreData();
+                    if (this.eventMetadataEnabled) {
+                        await this.resetMiscellaneousStoreData(epoch);
                     }
                     ;
                     (async () => this.logout(logoutPayload.data))().catch(e => this.log.error('PuppetService', 'onGrpcStreamEvent() this.logout() rejection %s', e.message));
@@ -281,8 +541,25 @@ class PuppetService extends PUPPET.Puppet {
                 break;
             case wechaty_grpc_1.puppet.EventType.EVENT_TYPE_DIRTY:
                 await this.fastDirty(JSON.parse(payload));
+                if (!this.isEventEpochCurrent(epoch)) {
+                    return;
+                }
                 this.emit('dirty', JSON.parse(payload));
                 break;
+            case wechaty_grpc_1.puppet.EventType.EVENT_TYPE_ROOM_MEMBER_LOAD: {
+                const loadPayload = JSON.parse(payload);
+                (0, pb_payload_helper_js_1.validateRoomMemberLoadState)(loadPayload);
+                this.roomCache.invalidate(loadPayload.roomId);
+                await this.roomCache.run(loadPayload.roomId, async () => {
+                    await this._payloadStore.room?.delete(loadPayload.roomId);
+                    await this._payloadStore.roomMember?.delete(loadPayload.roomId);
+                });
+                if (!this.isEventEpochCurrent(epoch)) {
+                    return;
+                }
+                this.emit('room-member-load', loadPayload);
+                break;
+            }
             case wechaty_grpc_1.puppet.EventType.EVENT_TYPE_MESSAGE:
                 this.emit('message', JSON.parse(payload));
                 break;
@@ -296,12 +573,40 @@ class PuppetService extends PUPPET.Puppet {
                 this.emit('post-tap', JSON.parse(payload));
                 break;
             case wechaty_grpc_1.puppet.EventType.EVENT_TYPE_READY:
+                if (this.recovery.enabled) {
+                    this.recovery.ready();
+                    if (this.readyIndicator.value()) {
+                        break;
+                    }
+                }
                 if (this.waitingForReady && this.readyIndicator.value()) {
                     this.log.warn('PuppetService', 'this ready event is ignored because the it is expected by event stream reconnect and this puppet is already ready');
                     return;
                 }
                 this.emit('ready', JSON.parse(payload));
                 break;
+            case wechaty_grpc_1.puppet.EventType.EVENT_TYPE_REPLAY_COMPLETE: {
+                if (!this.recovery.enabled) {
+                    break;
+                }
+                const barrier = JSON.parse(payload);
+                if (seq !== 0 || typeof barrier !== 'object' || barrier === null
+                    || !('accountId' in barrier) || !('throughSeq' in barrier)) {
+                    throw new Error('Invalid replay barrier');
+                }
+                const value = barrier;
+                if (typeof value.accountId !== 'string' || !value.accountId || typeof value.throughSeq !== 'number'
+                    || !Number.isSafeInteger(value.throughSeq) || value.throughSeq < 0
+                    || value.throughSeq > this.recoveryProcessedSeq || this.failedEventSeq !== undefined) {
+                    throw new Error('Invalid or incomplete replay barrier');
+                }
+                this.recovery.barrier(value.accountId);
+                if (this.recovery.state.status === 'offline') {
+                    this.stopRecovery(this.recovery.state.reason);
+                    this.reportStreamError(new Error(this.recovery.state.reason));
+                }
+                break;
+            }
             case wechaty_grpc_1.puppet.EventType.EVENT_TYPE_ROOM_INVITE:
                 this.emit('room-invite', JSON.parse(payload));
                 break;
@@ -355,8 +660,12 @@ class PuppetService extends PUPPET.Puppet {
                 // Huan(202003): in default, the `type` type should be `never`, please check.
                 this.log.error(`eventType ${type} unsupported! data: ${payload}`);
         }
+        if (seq && this.isEventEpochCurrent(epoch) && this.eventMetadataEnabled) {
+            await this.commitEventCheckpoint(seq, timestamp, epoch);
+        }
     }
     async logout(reason) {
+        this.stopRecovery(reason ?? 'Account logged out', false);
         this.log.verbose('PuppetService', 'logout(%s)', reason ? `"${reason}"` : '');
         await super.logout(reason);
         try {
@@ -410,8 +719,14 @@ class PuppetService extends PUPPET.Puppet {
             [PUPPET.types.Dirty.Friendship]: async (_) => { },
             [PUPPET.types.Dirty.Message]: async (_) => { },
             [PUPPET.types.Dirty.Post]: async (_) => { },
-            [PUPPET.types.Dirty.Room]: async (id) => this._payloadStore.room?.delete(id),
-            [PUPPET.types.Dirty.RoomMember]: async (id) => this._payloadStore.roomMember?.delete(id),
+            [PUPPET.types.Dirty.Room]: async (id) => {
+                this.roomCache.invalidate(id);
+                return this.roomCache.run(id, async () => this._payloadStore.room?.delete(id));
+            },
+            [PUPPET.types.Dirty.RoomMember]: async (id) => {
+                this.roomCache.invalidate(id);
+                return this.roomCache.run(id, async () => this._payloadStore.roomMember?.delete(id));
+            },
             [PUPPET.types.Dirty.Tag]: async (id) => this._payloadStore.tag?.delete(id),
             [PUPPET.types.Dirty.TagGroup]: async (id) => this._payloadStore.tagGroup?.delete(id),
             [PUPPET.types.Dirty.Unspecified]: async (id) => { throw new Error('Unspecified type with id: ' + id); },
@@ -1580,8 +1895,9 @@ class PuppetService extends PUPPET.Puppet {
      */
     async roomRawPayload(id) {
         this.log.verbose('PuppetService', 'roomRawPayload(%s)', id);
-        const cachedPayload = await this._payloadStore.room?.get(id);
-        if (cachedPayload) {
+        const generation = this.roomCache.generation(id);
+        const cachedPayload = await this.roomCache.run(id, async () => this._payloadStore.room?.get(id));
+        if (cachedPayload && generation === this.roomCache.generation(id)) {
             this.log.silly('PuppetService', 'roomRawPayload(%s) cache HIT', id);
             return cachedPayload;
         }
@@ -1589,26 +1905,52 @@ class PuppetService extends PUPPET.Puppet {
         request.setId(id);
         const response = await util_1.default.promisify(this.grpcManager.client.roomPayload
             .bind(this.grpcManager.client))(request);
-        const payload = {
-            adminIdList: response.getAdminIdsList(),
-            avatar: response.getAvatar(),
-            handle: response.getHandle(),
-            id: response.getId(),
-            memberIdList: response.getMemberIdsList(),
-            ownerId: response.getOwnerId(),
-            topic: response.getTopic(),
-            additionalInfo: response.getAdditionalInfo(),
-            remark: response.getRoomRemark(),
-            external: response.getExternal(),
-        };
-        const createTime = response.getCreateTime();
-        if (createTime) {
-            payload.createTime = (0, timestamp_js_1.millisecondsFromTimestamp)(createTime);
-        }
-        await this._payloadStore.room?.set(id, payload);
+        const payload = (0, pb_payload_helper_js_1.roomPbToPayload)(response);
+        await this.roomCache.commit(id, generation, async () => this._payloadStore.room?.set(id, payload));
         this.log.silly('PuppetService', 'roomRawPayload(%s) cache SET', id);
         return payload;
     }
+    async roomMemberSync(roomId, options) {
+        if (!roomId) {
+            throw new Error('roomId is required');
+        }
+        const request = new wechaty_grpc_1.puppet.RoomMemberSyncRequest();
+        request.setId(roomId);
+        if (options?.force !== undefined) {
+            request.setForce(options.force);
+        }
+        const response = await util_1.default.promisify(this.grpcManager.client.roomMemberSync.bind(this.grpcManager.client))(request);
+        return (0, pb_payload_helper_js_1.roomMemberLoadStatePbToPayload)(response, roomId);
+    }
+    async roomMemberLoadState(roomId) {
+        if (!roomId) {
+            throw new Error('roomId is required');
+        }
+        const request = new wechaty_grpc_1.puppet.RoomMemberLoadStateRequest();
+        request.setId(roomId);
+        const response = await util_1.default.promisify(this.grpcManager.client.roomMemberLoadState.bind(this.grpcManager.client))(request);
+        return (0, pb_payload_helper_js_1.roomMemberLoadStatePbToPayload)(response, roomId);
+    }
+    async roomMemberSnapshot(roomId, revision, options) {
+        if (!roomId) {
+            throw new Error('roomId is required');
+        }
+        (0, pb_payload_helper_js_1.validateRoomMemberRevision)(revision);
+        const request = new wechaty_grpc_1.puppet.RoomMemberSnapshotRequest();
+        request.setId(roomId);
+        request.setRevision(revision);
+        if (options?.pageToken !== undefined) {
+            request.setPageToken(options.pageToken);
+        }
+        if (options?.pageSize !== undefined) {
+            if (!Number.isInteger(options.pageSize) || options.pageSize < 0 || options.pageSize > 500) {
+                throw new Error('pageSize must be an integer between 0 and 500');
+            }
+            request.setPageSize(options.pageSize);
+        }
+        const response = await util_1.default.promisify(this.grpcManager.client.roomMemberSnapshot.bind(this.grpcManager.client))(request);
+        return (0, pb_payload_helper_js_1.roomMemberSnapshotPbToPayload)(response, roomId, revision);
+    }
     async roomRawPayloadParser(payload) {
         // this.log.silly('PuppetService', 'roomRawPayloadParser({id:%s})', payload.id)
         // passthrough
@@ -1618,10 +1960,11 @@ class PuppetService extends PUPPET.Puppet {
         this.log.verbose('PuppetService', 'batchRoomRawPayload(%s)', roomIdList);
         const result = new Map();
         const roomIdSet = new Set(roomIdList);
+        const generations = new Map(Array.from(roomIdSet, id => [id, this.roomCache.generation(id)]));
         const needGetSet = new Set();
         for (const roomId of roomIdSet) {
-            const cachedRoomPayload = await this._payloadStore.room?.get(roomId);
-            if (cachedRoomPayload) {
+            const cachedRoomPayload = await this.roomCache.run(roomId, async () => this._payloadStore.room?.get(roomId));
+            if (cachedRoomPayload && generations.get(roomId) === this.roomCache.generation(roomId)) {
                 result.set(roomId, cachedRoomPayload);
             }
             else {
@@ -1637,27 +1980,18 @@ class PuppetService extends PUPPET.Puppet {
                 const payloads = response.getRoomPayloadsList();
                 for (const payload of payloads) {
                     const roomId = payload.getId();
-                    const puppetPayload = {
-                        adminIdList: payload.getAdminIdsList(),
-                        avatar: payload.getAvatar(),
-                        handle: payload.getHandle(),
-                        id: payload.getId(),
-                        memberIdList: payload.getMemberIdsList(),
-                        ownerId: payload.getOwnerId(),
-                        topic: payload.getTopic(),
-                        additionalInfo: payload.getAdditionalInfo(),
-                        remark: payload.getRoomRemark(),
-                        external: payload.getExternal(),
-                    };
-                    const createTime = payload.getCreateTime();
-                    if (createTime) {
-                        puppetPayload.createTime = (0, timestamp_js_1.millisecondsFromTimestamp)(createTime);
-                    }
+                    const puppetPayload = (0, pb_payload_helper_js_1.roomPbToPayload)(payload);
                     result.set(roomId, puppetPayload);
-                    await this._payloadStore.room?.set(roomId, puppetPayload);
+                    const generation = generations.get(roomId);
+                    if (generation !== undefined) {
+                        await this.roomCache.commit(roomId, generation, async () => this._payloadStore.room?.set(roomId, puppetPayload));
+                    }
                 }
             }
             catch (e) {
+                if (e instanceof room_cache_coordinator_js_1.RoomCacheOperationError) {
+                    throw e;
+                }
                 this.log.error('PuppetService', 'batchRoomRawPayload(%s, %s) error: %s, use one by one method', roomIdList, needGetSet, e);
                 let fallbackSuccessCount = 0;
                 for (const roomId of needGetSet) {
@@ -1843,9 +2177,10 @@ class PuppetService extends PUPPET.Puppet {
     }
     async roomMemberRawPayload(roomId, contactId) {
         this.log.verbose('PuppetService', 'roomMemberRawPayload(%s, %s)', roomId, contactId);
-        const cachedPayload = await this._payloadStore.roomMember?.get(roomId);
+        const generation = this.roomCache.generation(roomId);
+        const cachedPayload = await this.roomCache.run(roomId, async () => this._payloadStore.roomMember?.get(roomId));
         const cachedRoomMemberPayload = cachedPayload && cachedPayload[contactId];
-        if (cachedRoomMemberPayload) {
+        if (cachedRoomMemberPayload && generation === this.roomCache.generation(roomId)) {
             this.log.silly('PuppetService', 'roomMemberRawPayload(%s, %s) cache HIT', roomId, contactId);
             return cachedRoomMemberPayload;
         }
@@ -1864,9 +2199,9 @@ class PuppetService extends PUPPET.Puppet {
             joinScene: response.getJoinScene(),
             joinTime: response.getJoinTime(),
         };
-        await this._payloadStore.roomMember?.set(roomId, {
-            ...cachedPayload,
-            [contactId]: payload,
+        await this.roomCache.commit(roomId, generation, async () => {
+            const current = await this._payloadStore.roomMember?.get(roomId);
+            await this._payloadStore.roomMember?.set(roomId, { ...current, [contactId]: payload });
         });
         this.log.silly('PuppetService', 'roomMemberRawPayload(%s, %s) cache SET', roomId, contactId);
         return payload;
@@ -1878,10 +2213,12 @@ class PuppetService extends PUPPET.Puppet {
     }
     async batchRoomMemberRawPayload(roomId, contactIdList) {
         this.log.verbose('PuppetService', 'batchRoomMemberRawPayload(%s, %s)', roomId, contactIdList);
+        const generation = this.roomCache.generation(roomId);
         const result = new Map();
         const contactIdSet = new Set(contactIdList);
         let needGetSet = new Set();
-        const cachedPayload = await this._payloadStore.roomMember?.get(roomId) || {};
+        const cached = await this.roomCache.run(roomId, async () => this._payloadStore.roomMember?.get(roomId));
+        const cachedPayload = generation === this.roomCache.generation(roomId) ? cached || {} : {};
         if (Object.keys(cachedPayload).length > 0) {
             for (const contactId of contactIdSet) {
                 const cachedRoomMemberPayload = cachedPayload[contactId];
@@ -1904,15 +2241,22 @@ class PuppetService extends PUPPET.Puppet {
                 const response = await util_1.default.promisify(this.grpcManager.client.batchRoomMemberPayload
                     .bind(this.grpcManager.client))(request);
                 const payloads = response.getMemberPayloadsList();
+                const fetched = {};
                 for (const payload of payloads) {
                     const contactId = payload.getId();
                     const puppetPayload = (0, pb_payload_helper_js_1.roomMemberPbToPayload)(payload);
                     result.set(contactId, puppetPayload);
-                    cachedPayload[contactId] = puppetPayload;
+                    fetched[contactId] = puppetPayload;
                 }
-                await this._payloadStore.roomMember?.set(roomId, cachedPayload);
+                await this.roomCache.commit(roomId, generation, async () => {
+                    const current = await this._payloadStore.roomMember?.get(roomId);
+                    await this._payloadStore.roomMember?.set(roomId, { ...current, ...fetched });
+                });
             }
             catch (e) {
+                if (e instanceof room_cache_coordinator_js_1.RoomCacheOperationError) {
+                    throw e;
+                }
                 this.log.error('PuppetService', 'batchRoomMemberRawPayload(%s, %s) error: %s, use one by one method', roomId, needGetSet, e);
                 for (const contactId of needGetSet) {
                     const payload = await this.roomMemberRawPayload(roomId, contactId);
@@ -2641,6 +2985,9 @@ class PuppetService extends PUPPET.Puppet {
     }
     healthCheckInterval;
     startHealthCheck() {
+        if (this.recovery.enabled) {
+            return;
+        }
         this.healthCheckInterval = setInterval(() => {
             this.ding('healthCheck');
         }, 60 * 1000);
@@ -2654,23 +3001,106 @@ class PuppetService extends PUPPET.Puppet {
     waitingForReady = false;
     reconnectIndicator;
     async reset() {
-        if (!this._grpcManager) {
-            this.log.warn('PuppetService', 'grpc manager not constructed, perform regular reset');
-            return super.reset();
-        }
-        if (!this.isLoggedIn) {
-            this.log.warn('PuppetService', 'puppet not logged in, perform regular reset');
-            return super.reset();
+        if (this.recovery.enabled) {
+            if (this._grpcManager) {
+                this.requestRecovery(this._grpcManager);
+            }
+            return;
         }
         if (this.reconnectIndicator.value()) {
             this.log.warn('PuppetService', 'already trying to reconnect, pass this one');
             return;
         }
         this.reconnectIndicator.value(true);
-        this.grpcManager.stopStream();
-        const { lastEventSeq, accountId } = await this.getMiscellaneousStoreData();
+        this.suspendEventProcessing();
+        const epoch = this.eventEpoch;
+        const manager = this._grpcManager;
+        try {
+            await this.checkpointQueue;
+            if (epoch !== this.eventEpoch) {
+                return;
+            }
+            if (!manager || !this.isLoggedIn) {
+                this.log.warn('PuppetService', 'grpc manager unavailable or puppet logged out, perform regular reset');
+                return await super.reset();
+            }
+            await this.resetEventStream(manager, epoch);
+        }
+        finally {
+            this.reconnectIndicator.value(false);
+        }
+    }
+    touchRecoveryStream() {
+        if (!this.recovery.enabled || this.recoveryStopped) {
+            return;
+        }
+        clearTimeout(this.inactivityTimer);
+        this.inactivityTimer = setTimeout(() => {
+            if (this._grpcManager) {
+                this.requestRecovery(this._grpcManager);
+            }
+        }, this.recovery.inactivity);
+    }
+    stopRecovery(reason, stopStream = true) {
+        if (!this.recovery.enabled) {
+            return;
+        }
+        this.recoveryStopped = true;
+        clearTimeout(this.recoveryTimer);
+        clearTimeout(this.inactivityTimer);
+        this.recoveryTimer = undefined;
+        this.recovery.fail(reason);
+        if (stopStream) {
+            this.suspendEventProcessing();
+            this._grpcManager?.stopStream();
+        }
+    }
+    requestRecovery(manager) {
+        if (this.recoveryStopped || manager !== this._grpcManager || this.recoveryTimer) {
+            return;
+        }
+        this.recovery.begin();
+        this.suspendEventProcessing();
+        manager.stopStream();
+        this.recovery.replaceControls();
+        clearTimeout(this.inactivityTimer);
+        const epoch = this.eventEpoch;
+        this.recoveryTimer = setTimeout(() => {
+            this.recoveryTimer = undefined;
+            void this.recovery.runDetached(async () => this.connectRecoveryStream(manager, epoch)).catch((error) => {
+                this.log.warn('PuppetService', 'recovery connection failed: %s', error);
+                if (epoch === this.eventEpoch) {
+                    this.requestRecovery(manager);
+                }
+            });
+        }, 1000);
+    }
+    async connectRecoveryStream(manager, epoch) {
+        const { lastEventSeq, accountId } = await this.eventResumePosition(epoch);
+        if (epoch !== this.eventEpoch || this.recoveryStopped || manager !== this._grpcManager) {
+            return;
+        }
+        this.recovery.begin(accountId);
+        this.recoveryProcessedSeq = Number(lastEventSeq ?? 0);
+        this.eventsBlocked = false;
+        this.touchRecoveryStream();
+        await manager.startStream(lastEventSeq, accountId);
+        if (!this.ownsEventWorker(manager, epoch)) {
+            return;
+        }
+        await util_1.default.promisify(manager.client.start.bind(manager.client))(new wechaty_grpc_1.puppet.StartRequest());
+    }
+    async resetEventStream(manager, epoch) {
+        manager.stopStream();
+        const { lastEventSeq, accountId } = await this.eventResumePosition(epoch);
+        if (epoch !== this.eventEpoch || manager !== this._grpcManager) {
+            return;
+        }
         const onLoginResolve = (resolve) => {
             const onLogin = (event) => {
+                if (!this.ownsEventWorker(manager, epoch)) {
+                    return;
+                }
                 const type = event.getType();
                 const payload = event.getPayload();
                 if (this.waitingForLogin && type === wechaty_grpc_1.puppet.EventType.EVENT_TYPE_LOGIN) {
@@ -2686,6 +3116,9 @@ class PuppetService extends PUPPET.Puppet {
         };
         const onReadyResolve = (resolve) => {
             const onReady = (event) => {
+                if (!this.ownsEventWorker(manager, epoch)) {
+                    return;
+                }
                 const type = event.getType();
                 if (this.waitingForReady && type === wechaty_grpc_1.puppet.EventType.EVENT_TYPE_READY) {
                     this.waitingForReady = false;
@@ -2694,64 +3127,89 @@ class PuppetService extends PUPPET.Puppet {
             };
             return onReady;
         };
-        let onLogin;
-        let onReady;
+        const removeListeners = [];
         const loginFuture = new Promise(resolve => {
-            onLogin = onLoginResolve(resolve);
-            this.grpcManager.on('data', onLogin);
+            const onLogin = onLoginResolve(resolve);
+            manager.on('data', onLogin);
+            removeListeners.push(() => { manager.off('data', onLogin); });
         });
         const readyFuture = new Promise(resolve => {
-            onReady = onReadyResolve(resolve);
-            this.grpcManager.on('data', onReady);
+            const onReady = onReadyResolve(resolve);
+            manager.on('data', onReady);
+            removeListeners.push(() => { manager.off('data', onReady); });
         });
         const startTime = Date.now();
         const timeoutMilliseconds = this.timeoutMilliseconds / 10; // 2 min default, 4 min xiaoju-bot
-        while (true) {
+        this.waitingForLogin = true;
+        this.waitingForReady = true;
+        this.eventsBlocked = false;
+        try {
+            while (true) {
+                try {
+                    if (!this.ownsEventWorker(manager, epoch)) {
+                        return;
+                    }
+                    await (0, gerror_1.timeoutPromise)(manager.startStream(lastEventSeq, accountId), 30 * 1000);
+                    if (!this.ownsEventWorker(manager, epoch)) {
+                        return;
+                    }
+                    break;
+                }
+                catch (e) {
+                    if (!this.ownsEventWorker(manager, epoch)) {
+                        return;
+                    }
+                    if (Date.now() - startTime < timeoutMilliseconds) {
+                        this.log.warn('failed to start stream, will try again in 15 seconds');
+                        await new Promise(resolve => {
+                            setTimeout(resolve, 5000);
+                        });
+                    }
+                    else {
+                        this.log.warn('failed to start stream and reaches timeout, will perform regular reset');
+                        return super.reset();
+                    }
+                }
+            }
             try {
-                await (0, gerror_1.timeoutPromise)(this.grpcManager.startStream(lastEventSeq, accountId), 30 * 1000);
-                break;
+                await (0, gerror_1.timeoutPromise)(loginFuture, ResetLoginTimeout)
+                    .finally(() => {
+                    if (epoch === this.eventEpoch) {
+                        this.waitingForLogin = false;
+                    }
+                });
             }
             catch (e) {
-                if (Date.now() - startTime < timeoutMilliseconds) {
-                    this.log.warn('failed to start stream, will try again in 15 seconds');
-                    await new Promise(resolve => {
-                        setTimeout(resolve, 5000);
-                    });
-                }
-                else {
-                    this.log.warn('failed to start stream and reaches timeout, will perform regular reset');
-                    this.reconnectIndicator.value(false);
-                    return super.reset();
+                if (!this.ownsEventWorker(manager, epoch)) {
+                    return;
                 }
+                this.log.warn('PuppetService', 'waiting for event reset login error, will perform regular reset');
+                return super.reset();
+            }
+            try {
+                await (0, gerror_1.timeoutPromise)(readyFuture, ResetReadyTimeout)
+                    .finally(() => {
+                    if (epoch === this.eventEpoch) {
+                        this.waitingForReady = false;
+                    }
+                });
+            }
+            catch (e) {
+                this.log.warn('PuppetService', 'waiting for event reset ready error, will do nothing');
             }
         }
-        this.waitingForLogin = true;
-        this.waitingForReady = true;
-        try {
-            await (0, gerror_1.timeoutPromise)(loginFuture, ResetLoginTimeout)
-                .finally(() => {
+        finally {
+            for (const removeListener of removeListeners) {
+                removeListener();
+            }
+            if (epoch === this.eventEpoch) {
                 this.waitingForLogin = false;
-                this.reconnectIndicator.value(false);
-                this.grpcManager.off('data', onLogin);
-            });
-        }
-        catch (e) {
-            this.log.warn('PuppetService', 'waiting for event reset login error, will perform regular reset');
-            return super.reset();
-        }
-        try {
-            await (0, gerror_1.timeoutPromise)(readyFuture, ResetReadyTimeout)
-                .finally(() => {
                 this.waitingForReady = false;
-                this.grpcManager.off('data', onReady);
-            });
-        }
-        catch (e) {
-            this.log.warn('PuppetService', 'waiting for event reset ready error, will do nothing');
+            }
         }
     }
     async getMiscellaneousStoreData() {
-        if (config_js_1.envVars.WECHATY_PUPPET_SERVICE_DISABLE_EVENT_CACHE()) {
+        if (!this.eventMetadataEnabled) {
             return {
                 lastEventSeq: undefined,
                 lastEventTimestamp: undefined,
@@ -2760,7 +3218,7 @@ class PuppetService extends PUPPET.Puppet {
         }
         const lastEventTimestamp = await this._payloadStore.miscellaneous?.get('lastEventTimestamp');
         let lastEventSeq = await this._payloadStore.miscellaneous?.get('lastEventSeq');
-        if ((Date.now() - Number(lastEventTimestamp || 0)) > this.timeoutMilliseconds) {
+        if (!this.recovery.enabled && (Date.now() - Number(lastEventTimestamp || 0)) > this.timeoutMilliseconds) {
             this.log.warn(`last event was ${(Date.now() - Number(lastEventTimestamp || 0)) / 1000} seconds ago, will not request event cache`);
             lastEventSeq = undefined;
         }
@@ -2771,27 +3229,29 @@ class PuppetService extends PUPPET.Puppet {
             accountId,
         };
     }
-    async setMiscellaneousStoreData(data) {
-        if (config_js_1.envVars.WECHATY_PUPPET_SERVICE_DISABLE_EVENT_CACHE()) {
+    async setMiscellaneousStoreData(data, epoch) {
+        if (!this.eventMetadataEnabled) {
             return;
         }
-        if (typeof data.lastEventSeq !== 'undefined') {
-            await this._payloadStore.miscellaneous?.set('lastEventSeq', data.lastEventSeq);
+        const changes = new Map();
+        if (data.lastEventTimestamp !== undefined) {
+            changes.set('lastEventTimestamp', data.lastEventTimestamp);
         }
-        if (typeof data.lastEventTimestamp !== 'undefined') {
-            await this._payloadStore.miscellaneous?.set('lastEventTimestamp', data.lastEventTimestamp);
+        if (data.accountId !== undefined) {
+            changes.set('accountId', data.accountId);
         }
-        if (typeof data.accountId !== 'undefined') {
-            await this._payloadStore.miscellaneous?.set('accountId', data.accountId);
+        if (data.lastEventSeq !== undefined) {
+            changes.set('lastEventSeq', data.lastEventSeq);
         }
+        await this.withEventCheckpoint(async () => this.writeEventMetadata(changes, epoch));
     }
-    async resetMiscellaneousStoreData() {
-        if (config_js_1.envVars.WECHATY_PUPPET_SERVICE_DISABLE_EVENT_CACHE()) {
+    async resetMiscellaneousStoreData(epoch) {
+        if (!this.eventMetadataEnabled) {
             return;
         }
-        await this._payloadStore.miscellaneous?.delete('lastEventSeq');
-        await this._payloadStore.miscellaneous?.delete('lastEventTimestamp');
-        await this._payloadStore.miscellaneous?.delete('accountId');
+        await this.withEventCheckpoint(async () => this.writeEventMetadata(new Map([
+            ['lastEventSeq', undefined], ['lastEventTimestamp', undefined], ['accountId', undefined],
+        ]), epoch));
     }
 }
 exports.PuppetService = PuppetService;