@juzi/wechaty-puppet-service 1.0.131 → 1.0.132

package/dist/cjs/src/client/puppet-service.js

@@ -56,6 +56,7 @@ const config_js_1 = require("../config.js");
 const event_type_rev_js_1 = require("../event-type-rev.js");
 const package_json_js_1 = require("../package-json.js");
 const grpc_manager_js_1 = require("./grpc-manager.js");
+const recovery_js_1 = require("./recovery.js");
 const payload_store_js_1 = require("./payload-store.js");
 const room_cache_coordinator_js_1 = require("./room-cache-coordinator.js");
 const pb_payload_helper_js_1 = require("../utils/pb-payload-helper.js");
@@ -87,6 +88,18 @@ class PuppetService extends PUPPET.Puppet {
     checkpointQueue = Promise.resolve();
     pendingCheckpointRollback;
     timeoutMilliseconds;
+    recovery;
+    recoveryTimer;
+    inactivityTimer;
+    recoveryStopped = true;
+    recoveryProcessedSeq = 0;
+    get recoveryState() { return this.recovery.state; }
+    get eventMetadataEnabled() {
+        return this.recovery.enabled || !config_js_1.envVars.WECHATY_PUPPET_SERVICE_DISABLE_EVENT_CACHE();
+    }
+    withRecoveryRequestContext(context, operation) {
+        return (0, recovery_js_1.withRecoveryRequestContext)(context, operation);
+    }
     _grpcManager;
     get grpcManager() {
         if (!this._grpcManager) {
@@ -104,9 +117,14 @@ class PuppetService extends PUPPET.Puppet {
     constructor(options = {}) {
         super(options);
         this.options = options;
+        this.recovery = new recovery_js_1.RecoveryGate(options.recovery, () => {
+            this.stopRecovery('Recovery budget exhausted');
+            this.reportStreamError(new Error('Recovery budget exhausted'));
+        });
         this._payloadStore = new payload_store_js_1.PayloadStore({
             token: config_js_1.envVars.WECHATY_PUPPET_SERVICE_TOKEN(this.options.token),
             logger: this.options.logger,
+            recoveryCursor: this.recovery.enabled,
         });
         this.hookPayloadStore();
         this.FileBoxUuid = (0, mod_js_1.uuidifyFileBoxGrpc)(() => this.grpcManager.client);
@@ -146,7 +164,7 @@ class PuppetService extends PUPPET.Puppet {
             this._grpcManager = undefined;
         }
         this.log.info('PuppetService', 'start() instanciating GrpcManager ...');
-        const grpcManager = new grpc_manager_js_1.GrpcManager(this.options);
+        const grpcManager = new grpc_manager_js_1.GrpcManager(this.options, this.recovery);
         this.log.info('PuppetService', 'start() instanciating GrpcManager ... done');
         /**
          * Huan(202108): when we started the event stream,
@@ -158,7 +176,19 @@ class PuppetService extends PUPPET.Puppet {
         this.log.info('PuppetService', 'start() setting up bridge grpc event stream ... done');
         this.log.info('PuppetService', 'start() starting grpc manager...');
         const { lastEventSeq, accountId } = await this.eventResumePosition();
-        await grpcManager.start(lastEventSeq, accountId);
+        this.recoveryStopped = false;
+        this.recovery.begin(accountId, true);
+        this.recoveryProcessedSeq = Number(lastEventSeq ?? 0);
+        this.touchRecoveryStream();
+        try {
+            await grpcManager.start(lastEventSeq, accountId);
+        }
+        catch (error) {
+            if (!this.recovery.enabled) {
+                throw error;
+            }
+            this.requestRecovery(grpcManager);
+        }
         this.log.info('PuppetService', 'start() starting grpc manager... done');
         this.log.info('PuppetService', 'start healthCheck');
         this.startHealthCheck();
@@ -166,13 +196,14 @@ class PuppetService extends PUPPET.Puppet {
     }
     async onStop() {
         this.log.info('PuppetService', 'onStop()');
+        this.stopRecovery('Puppet stopped');
         this.suspendEventProcessing();
         await this.checkpointQueue;
         if (this._grpcManager) {
             this.log.info('PuppetService', 'onStop() stopping grpc manager ...');
             const grpcManager = this._grpcManager;
             this._grpcManager = undefined;
-            await grpcManager.stop();
+            await this.recovery.runDetached(async () => grpcManager.stop());
             this.log.info('PuppetService', 'onStop() stopping grpc manager ... done');
         }
         this.log.info('PuppetService', 'onStop() ... done');
@@ -207,6 +238,25 @@ class PuppetService extends PUPPET.Puppet {
             if (client !== this._grpcManager || this.eventsBlocked) {
                 return;
             }
+            this.touchRecoveryStream();
+            if (this.recovery.enabled && event.getType() === wechaty_grpc_1.puppet.EventType.EVENT_TYPE_LOGIN) {
+                try {
+                    const payload = JSON.parse(event.getPayload());
+                    if (!this.recovery.checkAccount(payload.contactId)) {
+                        this.stopRecovery('Account changed or invalid login');
+                        this.reportStreamError(new Error('Account changed or invalid login'));
+                        return;
+                    }
+                }
+                catch (error) {
+                    this.stopRecovery('Invalid login payload');
+                    this.reportStreamError(error);
+                    return;
+                }
+            }
+            if (this.recovery.enabled && event.getType() === wechaty_grpc_1.puppet.EventType.EVENT_TYPE_LOGOUT) {
+                this.stopRecovery('Account logged out', false);
+            }
             const epoch = this.eventEpoch;
             this.eventQueue = this.eventQueue.then(async () => {
                 if (!this.ownsEventWorker(client, epoch)) {
@@ -217,10 +267,13 @@ class PuppetService extends PUPPET.Puppet {
                     if (this.failedEventSeq !== undefined && event.getSeq() > this.failedEventSeq) {
                         throw new Error(`Replay skipped failed event ${this.failedEventSeq}; stream halted until reset`);
                     }
-                    await this.onGrpcStreamEvent(event, epoch);
+                    await this.recovery.runEvent(() => this.ownsEventWorker(client, epoch), async () => this.onGrpcStreamEvent(event, epoch));
                     if (!this.ownsEventWorker(client, epoch)) {
                         return undefined;
                     }
+                    if (event.getSeq()) {
+                        this.recoveryProcessedSeq = event.getSeq();
+                    }
                     if (event.getSeq() === this.failedEventSeq) {
                         this.failedEventSeq = undefined;
                     }
@@ -239,9 +292,21 @@ class PuppetService extends PUPPET.Puppet {
             }).catch((error) => { this.reportStreamError(error); });
         })
             .on('end', () => {
+            if (this.recovery.enabled) {
+                this.requestRecovery(client);
+            }
             this.log.verbose('PuppetService', 'bridgeGrpcEventStream() eventStream.on(end)');
         })
             .on('error', (e) => {
+            if (this.recovery.enabled) {
+                if (this.recoveryStopped || client !== this._grpcManager) {
+                    this.reportStreamError(e);
+                }
+                else {
+                    this.requestRecovery(client);
+                }
+                return;
+            }
             this.emit('error', e);
             // https://github.com/wechaty/wechaty-puppet-service/issues/16
             // this.log.verbose('PuppetService', 'bridgeGrpcEventStream() eventStream.on(error) %s', e)
@@ -259,6 +324,9 @@ class PuppetService extends PUPPET.Puppet {
             // }
         })
             .on('cancel', (...args) => {
+            if (this.recovery.enabled) {
+                this.requestRecovery(client);
+            }
             this.log.verbose('PuppetService', 'bridgeGrpcEventStream() eventStream.on(cancel), %s', JSON.stringify(args));
         });
     }
@@ -279,6 +347,9 @@ class PuppetService extends PUPPET.Puppet {
     }
     async eventResumePosition(epoch = this.eventEpoch) {
         return this.withEventCheckpoint(async () => {
+            if (this.recovery.enabled && !this._payloadStore.miscellaneous) {
+                throw new Error('Recovery cursor storage is unavailable');
+            }
             if (this.pendingCheckpointRollback) {
                 await this.restoreEventMetadata(this.pendingCheckpointRollback);
                 this.pendingCheckpointRollback = undefined;
@@ -378,7 +449,13 @@ class PuppetService extends PUPPET.Puppet {
         catch (stopError) {
             this.log.error('PuppetService', 'stopping failed event stream failed: %s', stopError);
         }
-        this.reportStreamError(error);
+        if (this.recovery.enabled) {
+            this.log.error('PuppetService', 'recovering after event processing failure: %s', error);
+            this.requestRecovery(manager);
+        }
+        else {
+            this.reportStreamError(error);
+        }
     }
     async onGrpcStreamEvent(event, epoch = this.eventEpoch) {
         if (!this.isEventEpochCurrent(epoch)) {
@@ -412,30 +489,50 @@ class PuppetService extends PUPPET.Puppet {
                 break;
             case wechaty_grpc_1.puppet.EventType.EVENT_TYPE_LOGIN:
                 {
+                    const loginPayload = JSON.parse(payload);
+                    if (this.recovery.enabled) {
+                        this.recovery.login(loginPayload.contactId);
+                        if (this.recovery.state.status === 'offline') {
+                            this.stopRecovery('Account changed during recovery');
+                            this.reportStreamError(new Error('Account changed during recovery'));
+                            return;
+                        }
+                        if (this.isLoggedIn) {
+                            break;
+                        }
+                    }
                     if (this.waitingForLogin && this.isLoggedIn) {
                         this.log.warn('PuppetService', 'this login event is ignored because the it is expected by event stream reconnect and this puppet is already logged in');
                         return;
                     }
-                    const loginPayload = JSON.parse(payload);
-                    if (!config_js_1.envVars.WECHATY_PUPPET_SERVICE_DISABLE_EVENT_CACHE()) {
+                    if (this.eventMetadataEnabled) {
                         const { accountId } = await this.getMiscellaneousStoreData();
                         if (!this.isEventEpochCurrent(epoch)) {
                             return;
                         }
                         if (accountId !== loginPayload.contactId) {
                             await this.resetMiscellaneousStoreData(epoch);
+                            if (!this.isEventEpochCurrent(epoch)) {
+                                return;
+                            }
                             await this.setMiscellaneousStoreData({
                                 accountId: loginPayload.contactId,
                             }, epoch);
+                            if (!this.isEventEpochCurrent(epoch)) {
+                                return;
+                            }
                         }
                     }
+                    if (!this.isEventEpochCurrent(epoch)) {
+                        return;
+                    }
                     (async () => this.login(loginPayload.contactId))().catch(e => this.log.error('PuppetService', 'onGrpcStreamEvent() this.login() rejection %s', e.message));
                 }
                 break;
             case wechaty_grpc_1.puppet.EventType.EVENT_TYPE_LOGOUT:
                 {
                     const logoutPayload = JSON.parse(payload);
-                    if (!config_js_1.envVars.WECHATY_PUPPET_SERVICE_DISABLE_EVENT_CACHE()) {
+                    if (this.eventMetadataEnabled) {
                         await this.resetMiscellaneousStoreData(epoch);
                     }
                     ;
@@ -476,12 +573,40 @@ class PuppetService extends PUPPET.Puppet {
                 this.emit('post-tap', JSON.parse(payload));
                 break;
             case wechaty_grpc_1.puppet.EventType.EVENT_TYPE_READY:
+                if (this.recovery.enabled) {
+                    this.recovery.ready();
+                    if (this.readyIndicator.value()) {
+                        break;
+                    }
+                }
                 if (this.waitingForReady && this.readyIndicator.value()) {
                     this.log.warn('PuppetService', 'this ready event is ignored because the it is expected by event stream reconnect and this puppet is already ready');
                     return;
                 }
                 this.emit('ready', JSON.parse(payload));
                 break;
+            case wechaty_grpc_1.puppet.EventType.EVENT_TYPE_REPLAY_COMPLETE: {
+                if (!this.recovery.enabled) {
+                    break;
+                }
+                const barrier = JSON.parse(payload);
+                if (seq !== 0 || typeof barrier !== 'object' || barrier === null
+                    || !('accountId' in barrier) || !('throughSeq' in barrier)) {
+                    throw new Error('Invalid replay barrier');
+                }
+                const value = barrier;
+                if (typeof value.accountId !== 'string' || !value.accountId || typeof value.throughSeq !== 'number'
+                    || !Number.isSafeInteger(value.throughSeq) || value.throughSeq < 0
+                    || value.throughSeq > this.recoveryProcessedSeq || this.failedEventSeq !== undefined) {
+                    throw new Error('Invalid or incomplete replay barrier');
+                }
+                this.recovery.barrier(value.accountId);
+                if (this.recovery.state.status === 'offline') {
+                    this.stopRecovery(this.recovery.state.reason);
+                    this.reportStreamError(new Error(this.recovery.state.reason));
+                }
+                break;
+            }
             case wechaty_grpc_1.puppet.EventType.EVENT_TYPE_ROOM_INVITE:
                 this.emit('room-invite', JSON.parse(payload));
                 break;
@@ -535,11 +660,12 @@ class PuppetService extends PUPPET.Puppet {
                 // Huan(202003): in default, the `type` type should be `never`, please check.
                 this.log.error(`eventType ${type} unsupported! data: ${payload}`);
         }
-        if (seq && this.isEventEpochCurrent(epoch) && !config_js_1.envVars.WECHATY_PUPPET_SERVICE_DISABLE_EVENT_CACHE()) {
+        if (seq && this.isEventEpochCurrent(epoch) && this.eventMetadataEnabled) {
             await this.commitEventCheckpoint(seq, timestamp, epoch);
         }
     }
     async logout(reason) {
+        this.stopRecovery(reason ?? 'Account logged out', false);
         this.log.verbose('PuppetService', 'logout(%s)', reason ? `"${reason}"` : '');
         await super.logout(reason);
         try {
@@ -2859,6 +2985,9 @@ class PuppetService extends PUPPET.Puppet {
     }
     healthCheckInterval;
     startHealthCheck() {
+        if (this.recovery.enabled) {
+            return;
+        }
         this.healthCheckInterval = setInterval(() => {
             this.ding('healthCheck');
         }, 60 * 1000);
@@ -2872,6 +3001,12 @@ class PuppetService extends PUPPET.Puppet {
     waitingForReady = false;
     reconnectIndicator;
     async reset() {
+        if (this.recovery.enabled) {
+            if (this._grpcManager) {
+                this.requestRecovery(this._grpcManager);
+            }
+            return;
+        }
         if (this.reconnectIndicator.value()) {
             this.log.warn('PuppetService', 'already trying to reconnect, pass this one');
             return;
@@ -2895,6 +3030,66 @@ class PuppetService extends PUPPET.Puppet {
             this.reconnectIndicator.value(false);
         }
     }
+    touchRecoveryStream() {
+        if (!this.recovery.enabled || this.recoveryStopped) {
+            return;
+        }
+        clearTimeout(this.inactivityTimer);
+        this.inactivityTimer = setTimeout(() => {
+            if (this._grpcManager) {
+                this.requestRecovery(this._grpcManager);
+            }
+        }, this.recovery.inactivity);
+    }
+    stopRecovery(reason, stopStream = true) {
+        if (!this.recovery.enabled) {
+            return;
+        }
+        this.recoveryStopped = true;
+        clearTimeout(this.recoveryTimer);
+        clearTimeout(this.inactivityTimer);
+        this.recoveryTimer = undefined;
+        this.recovery.fail(reason);
+        if (stopStream) {
+            this.suspendEventProcessing();
+            this._grpcManager?.stopStream();
+        }
+    }
+    requestRecovery(manager) {
+        if (this.recoveryStopped || manager !== this._grpcManager || this.recoveryTimer) {
+            return;
+        }
+        this.recovery.begin();
+        this.suspendEventProcessing();
+        manager.stopStream();
+        this.recovery.replaceControls();
+        clearTimeout(this.inactivityTimer);
+        const epoch = this.eventEpoch;
+        this.recoveryTimer = setTimeout(() => {
+            this.recoveryTimer = undefined;
+            void this.recovery.runDetached(async () => this.connectRecoveryStream(manager, epoch)).catch((error) => {
+                this.log.warn('PuppetService', 'recovery connection failed: %s', error);
+                if (epoch === this.eventEpoch) {
+                    this.requestRecovery(manager);
+                }
+            });
+        }, 1000);
+    }
+    async connectRecoveryStream(manager, epoch) {
+        const { lastEventSeq, accountId } = await this.eventResumePosition(epoch);
+        if (epoch !== this.eventEpoch || this.recoveryStopped || manager !== this._grpcManager) {
+            return;
+        }
+        this.recovery.begin(accountId);
+        this.recoveryProcessedSeq = Number(lastEventSeq ?? 0);
+        this.eventsBlocked = false;
+        this.touchRecoveryStream();
+        await manager.startStream(lastEventSeq, accountId);
+        if (!this.ownsEventWorker(manager, epoch)) {
+            return;
+        }
+        await util_1.default.promisify(manager.client.start.bind(manager.client))(new wechaty_grpc_1.puppet.StartRequest());
+    }
     async resetEventStream(manager, epoch) {
         manager.stopStream();
         const { lastEventSeq, accountId } = await this.eventResumePosition(epoch);
@@ -3014,7 +3209,7 @@ class PuppetService extends PUPPET.Puppet {
         }
     }
     async getMiscellaneousStoreData() {
-        if (config_js_1.envVars.WECHATY_PUPPET_SERVICE_DISABLE_EVENT_CACHE()) {
+        if (!this.eventMetadataEnabled) {
             return {
                 lastEventSeq: undefined,
                 lastEventTimestamp: undefined,
@@ -3023,7 +3218,7 @@ class PuppetService extends PUPPET.Puppet {
         }
         const lastEventTimestamp = await this._payloadStore.miscellaneous?.get('lastEventTimestamp');
         let lastEventSeq = await this._payloadStore.miscellaneous?.get('lastEventSeq');
-        if ((Date.now() - Number(lastEventTimestamp || 0)) > this.timeoutMilliseconds) {
+        if (!this.recovery.enabled && (Date.now() - Number(lastEventTimestamp || 0)) > this.timeoutMilliseconds) {
             this.log.warn(`last event was ${(Date.now() - Number(lastEventTimestamp || 0)) / 1000} seconds ago, will not request event cache`);
             lastEventSeq = undefined;
         }
@@ -3035,7 +3230,7 @@ class PuppetService extends PUPPET.Puppet {
         };
     }
     async setMiscellaneousStoreData(data, epoch) {
-        if (config_js_1.envVars.WECHATY_PUPPET_SERVICE_DISABLE_EVENT_CACHE()) {
+        if (!this.eventMetadataEnabled) {
             return;
         }
         const changes = new Map();
@@ -3051,7 +3246,7 @@ class PuppetService extends PUPPET.Puppet {
         await this.withEventCheckpoint(async () => this.writeEventMetadata(changes, epoch));
     }
     async resetMiscellaneousStoreData(epoch) {
-        if (config_js_1.envVars.WECHATY_PUPPET_SERVICE_DISABLE_EVENT_CACHE()) {
+        if (!this.eventMetadataEnabled) {
             return;
         }
         await this.withEventCheckpoint(async () => this.writeEventMetadata(new Map([