@dronedeploy/rocos-js-sdk 4.4.7 → 4.4.8

package/cjs/api/StreamRegister.d.ts

@@ -10,8 +10,12 @@ export interface IAcquiredStream<T extends IBaseStream> {
 /**
  * Sole owner of stream lifecycle. Streams are shared per identifier and refcounted: acquireStream
  * gets-or-creates and increments in one synchronous step, and the returned handle's release()
- * tears the stream down when the last holder leaves. Teardown never happens outside
- * release/removeAllStreams, so a held stream can not be stopped or replaced underneath its holders.
+ * tears the stream down when the last holder leaves.
+ *
+ * A held stream can also be stopped and replaced underneath its holders, by reportStreamDead, when
+ * the stream can no longer carry anything. Holders keep the instance they were handed; their
+ * release() finds the entry gone and does nothing, so a replacement never inherits a departing
+ * holder's refcount and a refcount can not go negative.
  */
 export declare class StreamRegister {
     private entries;
@@ -22,6 +26,7 @@ export declare class StreamRegister {
     static getIdentifier($identifier: string, scope?: string): string;
     acquireStream<T extends IBaseStream>(identifier: string, create: () => T, init: (stream: T) => Promise<void>): IAcquiredStream<T>;
     getStream(identifier: string): IBaseStream | undefined;
+    reportStreamDead(identifier: string, stream: IBaseStream): void;
     removeAllStreams(): boolean;
     private release;
     private evict;

package/cjs/api/StreamRegister.js

@@ -6,8 +6,12 @@ const RocosLogger_1 = require("../logger/RocosLogger");
 /**
  * Sole owner of stream lifecycle. Streams are shared per identifier and refcounted: acquireStream
  * gets-or-creates and increments in one synchronous step, and the returned handle's release()
- * tears the stream down when the last holder leaves. Teardown never happens outside
- * release/removeAllStreams, so a held stream can not be stopped or replaced underneath its holders.
+ * tears the stream down when the last holder leaves.
+ *
+ * A held stream can also be stopped and replaced underneath its holders, by reportStreamDead, when
+ * the stream can no longer carry anything. Holders keep the instance they were handed; their
+ * release() finds the entry gone and does nothing, so a replacement never inherits a departing
+ * holder's refcount and a refcount can not go negative.
  */
 class StreamRegister {
     constructor() {
@@ -62,6 +66,17 @@ class StreamRegister {
     getStream(identifier) {
         return this.entries.get(identifier)?.stream;
     }
+    reportStreamDead(identifier, stream) {
+        if (this.entries.get(identifier)?.stream !== stream)
+            return; // already evicted, or already replaced
+        this.logger.warn(`Stream reported dead, evicting so the next acquirer builds a new one. identifier: ${identifier}`);
+        try {
+            this.evict(identifier, stream);
+        }
+        catch (e) {
+            this.logger.error(`Failed to evict dead stream. identifier: ${identifier}`, e);
+        }
+    }
     removeAllStreams() {
         for (const [, entry] of this.entries.entries()) {
             try {

package/cjs/api/streams/telemetry/TelemetryStreamAbstract.d.ts

@@ -29,6 +29,7 @@ export declare abstract class TelemetryStreamAbstract implements ITelemetryStrea
     private sendHeartbeatTime;
     private receivedDataStats;
     private subscriptions;
+    private receiverDead;
     constructor(config: ITelemetryStreamConfig);
     protected abstract stopInternal(): void;
     protected abstract registerReceiver(): () => void;
@@ -41,6 +42,8 @@ export declare abstract class TelemetryStreamAbstract implements ITelemetryStrea
     stopStream(): void;
     addSubscription(params: ITelemetryParams): void;
     removeSubscription(params: ITelemetryParams, terminateReceiverGroup?: boolean): Promise<void>;
+    private onBackgroundActionFailure;
+    private markReceiverDead;
     private isQueryLookup;
     private getSubscribedSources;
     sendAcknowledgment(uid: string, status: TelemetryAckStatus, noRetry: boolean): boolean;
@@ -55,6 +58,7 @@ export declare abstract class TelemetryStreamAbstract implements ITelemetryStrea
     private takeTelemetryAction;
     protected listenMessagesAndRenew(): void;
     private addTerminateToAction;
+    private renewReceiver;
     private messageTimeoutTryAgain;
     /**
      * Auto resubscribe to reduce CPU usage.

package/cjs/api/streams/telemetry/TelemetryStreamAbstract.js

@@ -20,6 +20,14 @@ const resolveTelemetryActions = (actions) => {
 const resolveUnsentTelemetryActions = (queueState) => {
     resolveTelemetryActions(queueState.pendingActions.splice(0));
 };
+const isReceiverGoneError = (error) => {
+    const code = error?.code;
+    if (typeof code === 'number')
+        return code === 9; // gRPC FAILED_PRECONDITION
+    if (typeof code === 'string')
+        return code.toLowerCase() === 'failed_precondition';
+    return false;
+};
 const takeNextTelemetryBatch = (queueState) => {
     const firstAction = queueState.pendingActions.shift();
     if (!firstAction)
@@ -53,6 +61,7 @@ class TelemetryStreamAbstract {
         this.sendHeartbeatTime = 5000;
         this.receivedDataStats = {};
         this.subscriptions = new Map();
+        this.receiverDead = false;
         this.identifier = StreamRegister_1.StreamRegister.getIdentifier(identifier_1.IDENTIFIER_NAME_TELEMETRY, config.scope);
         this.projectId = config.projectId;
         this.callsignsLookup = config.callsignsLookup;
@@ -129,7 +138,7 @@ class TelemetryStreamAbstract {
                 if (toAdd.length) {
                     this.logger.debug('New query subscriptions added, we need to refresh sources', { toAdd });
                     void this.enqueueTelemetryAction('subscribe', this.callsignsLookup, toAdd).catch((error) => {
-                        this.logger.error(`Failed to subscribe: ${error}`);
+                        this.onBackgroundActionFailure('subscribe', error);
                     });
                 }
             }
@@ -142,7 +151,7 @@ class TelemetryStreamAbstract {
                 if (toAdd.callsigns.length && toAdd.sources.length) {
                     this.logger.debug('New subscriptions added, we need to refresh sources', { toAdd });
                     void this.enqueueTelemetryAction('subscribe', new models_1.CallsignsLookup(toAdd.callsigns), toAdd.sources).catch((error) => {
-                        this.logger.error(`Failed to subscribe: ${error}`);
+                        this.onBackgroundActionFailure('subscribe', error);
                     });
                 }
             }
@@ -178,7 +187,7 @@ class TelemetryStreamAbstract {
                     await this.enqueueTelemetryAction('unsubscribe', this.callsignsLookup, toRemove, terminateReceiverGroup);
                 }
                 catch (err) {
-                    this.logger.error(`Failed to unsubscribe: ${err}`);
+                    this.onBackgroundActionFailure('unsubscribe', err);
                 }
             }
             return;
@@ -193,10 +202,24 @@ class TelemetryStreamAbstract {
                 await this.enqueueTelemetryAction('unsubscribe', new models_1.CallsignsLookup(toRemove.callsigns), toRemove.sources, terminateReceiverGroup);
             }
             catch (err) {
-                this.logger.error(`Failed to unsubscribe: ${err}`);
+                this.onBackgroundActionFailure('unsubscribe', err);
             }
         }
     }
+    onBackgroundActionFailure(actionType, error) {
+        this.logger.error(`Failed to ${actionType}: ${error}`);
+        if (isReceiverGoneError(error)) {
+            this.markReceiverDead(error);
+        }
+    }
+    markReceiverDead(error) {
+        if (this.receiverDead)
+            return;
+        this.receiverDead = true;
+        this.logger.error('Receiver is no longer registered with the gateway, retiring the stream', error);
+        StreamRegister_1.StreamRegister.getInstance().reportStreamDead(this.identifier, this);
+        this.messageStream$.error(error);
+    }
     isQueryLookup() {
         return this.callsignsLookup?.isQuery() ?? false;
     }
@@ -263,7 +286,7 @@ class TelemetryStreamAbstract {
                 rocosMessage.subscriberId = this.subscriberId;
                 this.logger.info('onData', `subscriberId has been updated - ${subscriberId}`, { subscriberId });
                 void this.enqueueTelemetryAction('subscribe').catch((error) => {
-                    this.logger.error(`Failed to subscribe: ${error}`);
+                    this.onBackgroundActionFailure('subscribe', error);
                 });
                 this.heartbeat.start(() => this.sendHeartbeat(), this.sendHeartbeatTime);
                 isRobotMessage = false;
@@ -514,14 +537,22 @@ class TelemetryStreamAbstract {
             unsubscribeOperation,
         };
     }
+    renewReceiver() {
+        try {
+            // Remove the received call anyway.
+            this.stopInternal();
+            this.lastMessageReceived = undefined;
+            this.lastRobotMessageReceived = undefined;
+            this.registerReceiver();
+        }
+        catch (e) {
+            this.logger.error('Failed to renew the receiver, retrying on the next check', e);
+        }
+    }
     messageTimeoutTryAgain() {
         // Increase retry times.
         this.retryTimes++;
-        // Remove the received call anyway.
-        this.stopInternal();
-        this.lastMessageReceived = undefined;
-        this.lastRobotMessageReceived = undefined;
-        this.registerReceiver();
+        this.renewReceiver();
     }
     /**
      * Auto resubscribe to reduce CPU usage.
@@ -529,11 +560,7 @@ class TelemetryStreamAbstract {
     autoResubscribe() {
         // Reset
         this.retryTimes = 0;
-        // Remove the received call anyway.
-        this.stopInternal();
-        this.lastMessageReceived = undefined;
-        this.lastRobotMessageReceived = undefined;
-        this.registerReceiver();
+        this.renewReceiver();
     }
     isFoundSource(source) {
         return this.sources.indexOf(source) !== -1;

package/cjs/services/TelemetryService.js

@@ -90,11 +90,13 @@ class TelemetryService extends BaseStreamService_1.BaseStreamService {
             }), (0, operators_1.finalize)(() => {
                 if (params.terminateReceiverGroupOnUnsubscribe) {
                     const removal = stream.removeSubscription(subscriptionParams, true);
-                    void Promise.resolve(removal).finally(release);
+                    void Promise.resolve(removal)
+                        .finally(release)
+                        .catch(() => undefined);
                 }
                 else {
                     release();
-                    void stream.removeSubscription(subscriptionParams);
+                    void Promise.resolve(stream.removeSubscription(subscriptionParams)).catch(() => undefined);
                 }
             }));
         });

package/esm/api/StreamRegister.d.ts

@@ -10,8 +10,12 @@ export interface IAcquiredStream<T extends IBaseStream> {
 /**
  * Sole owner of stream lifecycle. Streams are shared per identifier and refcounted: acquireStream
  * gets-or-creates and increments in one synchronous step, and the returned handle's release()
- * tears the stream down when the last holder leaves. Teardown never happens outside
- * release/removeAllStreams, so a held stream can not be stopped or replaced underneath its holders.
+ * tears the stream down when the last holder leaves.
+ *
+ * A held stream can also be stopped and replaced underneath its holders, by reportStreamDead, when
+ * the stream can no longer carry anything. Holders keep the instance they were handed; their
+ * release() finds the entry gone and does nothing, so a replacement never inherits a departing
+ * holder's refcount and a refcount can not go negative.
  */
 export declare class StreamRegister {
     private entries;
@@ -22,6 +26,7 @@ export declare class StreamRegister {
     static getIdentifier($identifier: string, scope?: string): string;
     acquireStream<T extends IBaseStream>(identifier: string, create: () => T, init: (stream: T) => Promise<void>): IAcquiredStream<T>;
     getStream(identifier: string): IBaseStream | undefined;
+    reportStreamDead(identifier: string, stream: IBaseStream): void;
     removeAllStreams(): boolean;
     private release;
     private evict;

package/esm/api/StreamRegister.js

@@ -3,8 +3,12 @@ import { RocosLogger } from '../logger/RocosLogger';
 /**
  * Sole owner of stream lifecycle. Streams are shared per identifier and refcounted: acquireStream
  * gets-or-creates and increments in one synchronous step, and the returned handle's release()
- * tears the stream down when the last holder leaves. Teardown never happens outside
- * release/removeAllStreams, so a held stream can not be stopped or replaced underneath its holders.
+ * tears the stream down when the last holder leaves.
+ *
+ * A held stream can also be stopped and replaced underneath its holders, by reportStreamDead, when
+ * the stream can no longer carry anything. Holders keep the instance they were handed; their
+ * release() finds the entry gone and does nothing, so a replacement never inherits a departing
+ * holder's refcount and a refcount can not go negative.
  */
 export class StreamRegister {
     constructor() {
@@ -59,6 +63,17 @@ export class StreamRegister {
     getStream(identifier) {
         return this.entries.get(identifier)?.stream;
     }
+    reportStreamDead(identifier, stream) {
+        if (this.entries.get(identifier)?.stream !== stream)
+            return; // already evicted, or already replaced
+        this.logger.warn(`Stream reported dead, evicting so the next acquirer builds a new one. identifier: ${identifier}`);
+        try {
+            this.evict(identifier, stream);
+        }
+        catch (e) {
+            this.logger.error(`Failed to evict dead stream. identifier: ${identifier}`, e);
+        }
+    }
     removeAllStreams() {
         for (const [, entry] of this.entries.entries()) {
             try {

package/esm/api/streams/telemetry/TelemetryStreamAbstract.d.ts

@@ -29,6 +29,7 @@ export declare abstract class TelemetryStreamAbstract implements ITelemetryStrea
     private sendHeartbeatTime;
     private receivedDataStats;
     private subscriptions;
+    private receiverDead;
     constructor(config: ITelemetryStreamConfig);
     protected abstract stopInternal(): void;
     protected abstract registerReceiver(): () => void;
@@ -41,6 +42,8 @@ export declare abstract class TelemetryStreamAbstract implements ITelemetryStrea
     stopStream(): void;
     addSubscription(params: ITelemetryParams): void;
     removeSubscription(params: ITelemetryParams, terminateReceiverGroup?: boolean): Promise<void>;
+    private onBackgroundActionFailure;
+    private markReceiverDead;
     private isQueryLookup;
     private getSubscribedSources;
     sendAcknowledgment(uid: string, status: TelemetryAckStatus, noRetry: boolean): boolean;
@@ -55,6 +58,7 @@ export declare abstract class TelemetryStreamAbstract implements ITelemetryStrea
     private takeTelemetryAction;
     protected listenMessagesAndRenew(): void;
     private addTerminateToAction;
+    private renewReceiver;
     private messageTimeoutTryAgain;
     /**
      * Auto resubscribe to reduce CPU usage.

package/esm/api/streams/telemetry/TelemetryStreamAbstract.js

@@ -17,6 +17,14 @@ const resolveTelemetryActions = (actions) => {
 const resolveUnsentTelemetryActions = (queueState) => {
     resolveTelemetryActions(queueState.pendingActions.splice(0));
 };
+const isReceiverGoneError = (error) => {
+    const code = error?.code;
+    if (typeof code === 'number')
+        return code === 9; // gRPC FAILED_PRECONDITION
+    if (typeof code === 'string')
+        return code.toLowerCase() === 'failed_precondition';
+    return false;
+};
 const takeNextTelemetryBatch = (queueState) => {
     const firstAction = queueState.pendingActions.shift();
     if (!firstAction)
@@ -50,6 +58,7 @@ export class TelemetryStreamAbstract {
         this.sendHeartbeatTime = 5000;
         this.receivedDataStats = {};
         this.subscriptions = new Map();
+        this.receiverDead = false;
         this.identifier = StreamRegister.getIdentifier(IDENTIFIER_NAME_TELEMETRY, config.scope);
         this.projectId = config.projectId;
         this.callsignsLookup = config.callsignsLookup;
@@ -126,7 +135,7 @@ export class TelemetryStreamAbstract {
                 if (toAdd.length) {
                     this.logger.debug('New query subscriptions added, we need to refresh sources', { toAdd });
                     void this.enqueueTelemetryAction('subscribe', this.callsignsLookup, toAdd).catch((error) => {
-                        this.logger.error(`Failed to subscribe: ${error}`);
+                        this.onBackgroundActionFailure('subscribe', error);
                     });
                 }
             }
@@ -139,7 +148,7 @@ export class TelemetryStreamAbstract {
                 if (toAdd.callsigns.length && toAdd.sources.length) {
                     this.logger.debug('New subscriptions added, we need to refresh sources', { toAdd });
                     void this.enqueueTelemetryAction('subscribe', new CallsignsLookup(toAdd.callsigns), toAdd.sources).catch((error) => {
-                        this.logger.error(`Failed to subscribe: ${error}`);
+                        this.onBackgroundActionFailure('subscribe', error);
                     });
                 }
             }
@@ -175,7 +184,7 @@ export class TelemetryStreamAbstract {
                     await this.enqueueTelemetryAction('unsubscribe', this.callsignsLookup, toRemove, terminateReceiverGroup);
                 }
                 catch (err) {
-                    this.logger.error(`Failed to unsubscribe: ${err}`);
+                    this.onBackgroundActionFailure('unsubscribe', err);
                 }
             }
             return;
@@ -190,10 +199,24 @@ export class TelemetryStreamAbstract {
                 await this.enqueueTelemetryAction('unsubscribe', new CallsignsLookup(toRemove.callsigns), toRemove.sources, terminateReceiverGroup);
             }
             catch (err) {
-                this.logger.error(`Failed to unsubscribe: ${err}`);
+                this.onBackgroundActionFailure('unsubscribe', err);
             }
         }
     }
+    onBackgroundActionFailure(actionType, error) {
+        this.logger.error(`Failed to ${actionType}: ${error}`);
+        if (isReceiverGoneError(error)) {
+            this.markReceiverDead(error);
+        }
+    }
+    markReceiverDead(error) {
+        if (this.receiverDead)
+            return;
+        this.receiverDead = true;
+        this.logger.error('Receiver is no longer registered with the gateway, retiring the stream', error);
+        StreamRegister.getInstance().reportStreamDead(this.identifier, this);
+        this.messageStream$.error(error);
+    }
     isQueryLookup() {
         return this.callsignsLookup?.isQuery() ?? false;
     }
@@ -260,7 +283,7 @@ export class TelemetryStreamAbstract {
                 rocosMessage.subscriberId = this.subscriberId;
                 this.logger.info('onData', `subscriberId has been updated - ${subscriberId}`, { subscriberId });
                 void this.enqueueTelemetryAction('subscribe').catch((error) => {
-                    this.logger.error(`Failed to subscribe: ${error}`);
+                    this.onBackgroundActionFailure('subscribe', error);
                 });
                 this.heartbeat.start(() => this.sendHeartbeat(), this.sendHeartbeatTime);
                 isRobotMessage = false;
@@ -511,14 +534,22 @@ export class TelemetryStreamAbstract {
             unsubscribeOperation,
         };
     }
+    renewReceiver() {
+        try {
+            // Remove the received call anyway.
+            this.stopInternal();
+            this.lastMessageReceived = undefined;
+            this.lastRobotMessageReceived = undefined;
+            this.registerReceiver();
+        }
+        catch (e) {
+            this.logger.error('Failed to renew the receiver, retrying on the next check', e);
+        }
+    }
     messageTimeoutTryAgain() {
         // Increase retry times.
         this.retryTimes++;
-        // Remove the received call anyway.
-        this.stopInternal();
-        this.lastMessageReceived = undefined;
-        this.lastRobotMessageReceived = undefined;
-        this.registerReceiver();
+        this.renewReceiver();
     }
     /**
      * Auto resubscribe to reduce CPU usage.
@@ -526,11 +557,7 @@ export class TelemetryStreamAbstract {
     autoResubscribe() {
         // Reset
         this.retryTimes = 0;
-        // Remove the received call anyway.
-        this.stopInternal();
-        this.lastMessageReceived = undefined;
-        this.lastRobotMessageReceived = undefined;
-        this.registerReceiver();
+        this.renewReceiver();
     }
     isFoundSource(source) {
         return this.sources.indexOf(source) !== -1;

package/esm/services/TelemetryService.js

@@ -87,11 +87,13 @@ export class TelemetryService extends BaseStreamService {
             }), finalize(() => {
                 if (params.terminateReceiverGroupOnUnsubscribe) {
                     const removal = stream.removeSubscription(subscriptionParams, true);
-                    void Promise.resolve(removal).finally(release);
+                    void Promise.resolve(removal)
+                        .finally(release)
+                        .catch(() => undefined);
                 }
                 else {
                     release();
-                    void stream.removeSubscription(subscriptionParams);
+                    void Promise.resolve(stream.removeSubscription(subscriptionParams)).catch(() => undefined);
                 }
             }));
         });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dronedeploy/rocos-js-sdk",
-  "version": "4.4.7",
+  "version": "4.4.8",
   "description": "Javascript SDK for rocos",
   "main": "cjs/index.js",
   "module": "esm/index.js",