@dronedeploy/rocos-js-sdk 4.4.6 → 4.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cjs/api/StreamRegister.d.ts +7 -2
- package/cjs/api/StreamRegister.js +21 -6
- package/cjs/api/streams/caller/CallerStreamAbstract.js +2 -1
- package/cjs/api/streams/command/CommandStreamAbstract.js +2 -1
- package/cjs/api/streams/control/ControlStreamAbstract.js +2 -1
- package/cjs/api/streams/fileAccessor/FileAccessorStreamAbstract.js +2 -1
- package/cjs/api/streams/search/SearchStreamAbstract.js +2 -1
- package/cjs/api/streams/telemetry/TelemetryStream.js +1 -0
- package/cjs/api/streams/telemetry/TelemetryStreamAbstract.d.ts +4 -0
- package/cjs/api/streams/telemetry/TelemetryStreamAbstract.js +165 -37
- package/cjs/api/streams/telemetry/TelemetryStreamConnect.js +1 -0
- package/cjs/api/streams/telemetry/TelemetryStreamNode.js +1 -0
- package/cjs/api/streams/webRTCSignalling/WebRTCSignallingStreamAbstract.js +2 -1
- package/cjs/models/maps/Map.d.ts +1 -1
- package/cjs/services/SearchService.js +2 -7
- package/cjs/services/TelemetryService.d.ts +4 -0
- package/cjs/services/TelemetryService.js +14 -2
- package/esm/api/StreamRegister.d.ts +7 -2
- package/esm/api/StreamRegister.js +21 -6
- package/esm/api/streams/caller/CallerStreamAbstract.js +2 -1
- package/esm/api/streams/command/CommandStreamAbstract.js +2 -1
- package/esm/api/streams/control/ControlStreamAbstract.js +2 -1
- package/esm/api/streams/fileAccessor/FileAccessorStreamAbstract.js +2 -1
- package/esm/api/streams/search/SearchStreamAbstract.js +2 -1
- package/esm/api/streams/telemetry/TelemetryStream.js +1 -0
- package/esm/api/streams/telemetry/TelemetryStreamAbstract.d.ts +4 -0
- package/esm/api/streams/telemetry/TelemetryStreamAbstract.js +166 -38
- package/esm/api/streams/telemetry/TelemetryStreamConnect.js +1 -0
- package/esm/api/streams/telemetry/TelemetryStreamNode.js +1 -0
- package/esm/api/streams/webRTCSignalling/WebRTCSignallingStreamAbstract.js +2 -1
- package/esm/models/maps/Map.d.ts +1 -1
- package/esm/services/SearchService.js +2 -7
- package/esm/services/TelemetryService.d.ts +4 -0
- package/esm/services/TelemetryService.js +14 -2
- package/package.json +1 -1
|
@@ -10,8 +10,12 @@ export interface IAcquiredStream<T extends IBaseStream> {
|
|
|
10
10
|
/**
|
|
11
11
|
* Sole owner of stream lifecycle. Streams are shared per identifier and refcounted: acquireStream
|
|
12
12
|
* gets-or-creates and increments in one synchronous step, and the returned handle's release()
|
|
13
|
-
* tears the stream down when the last holder leaves.
|
|
14
|
-
*
|
|
13
|
+
* tears the stream down when the last holder leaves.
|
|
14
|
+
*
|
|
15
|
+
* A held stream can also be stopped and replaced underneath its holders, by reportStreamDead, when
|
|
16
|
+
* the stream can no longer carry anything. Holders keep the instance they were handed; their
|
|
17
|
+
* release() finds the entry gone and does nothing, so a replacement never inherits a departing
|
|
18
|
+
* holder's refcount and a refcount can not go negative.
|
|
15
19
|
*/
|
|
16
20
|
export declare class StreamRegister {
|
|
17
21
|
private entries;
|
|
@@ -22,6 +26,7 @@ export declare class StreamRegister {
|
|
|
22
26
|
static getIdentifier($identifier: string, scope?: string): string;
|
|
23
27
|
acquireStream<T extends IBaseStream>(identifier: string, create: () => T, init: (stream: T) => Promise<void>): IAcquiredStream<T>;
|
|
24
28
|
getStream(identifier: string): IBaseStream | undefined;
|
|
29
|
+
reportStreamDead(identifier: string, stream: IBaseStream): void;
|
|
25
30
|
removeAllStreams(): boolean;
|
|
26
31
|
private release;
|
|
27
32
|
private evict;
|
|
@@ -6,8 +6,12 @@ const RocosLogger_1 = require("../logger/RocosLogger");
|
|
|
6
6
|
/**
|
|
7
7
|
* Sole owner of stream lifecycle. Streams are shared per identifier and refcounted: acquireStream
|
|
8
8
|
* gets-or-creates and increments in one synchronous step, and the returned handle's release()
|
|
9
|
-
* tears the stream down when the last holder leaves.
|
|
10
|
-
*
|
|
9
|
+
* tears the stream down when the last holder leaves.
|
|
10
|
+
*
|
|
11
|
+
* A held stream can also be stopped and replaced underneath its holders, by reportStreamDead, when
|
|
12
|
+
* the stream can no longer carry anything. Holders keep the instance they were handed; their
|
|
13
|
+
* release() finds the entry gone and does nothing, so a replacement never inherits a departing
|
|
14
|
+
* holder's refcount and a refcount can not go negative.
|
|
11
15
|
*/
|
|
12
16
|
class StreamRegister {
|
|
13
17
|
constructor() {
|
|
@@ -62,6 +66,17 @@ class StreamRegister {
|
|
|
62
66
|
getStream(identifier) {
|
|
63
67
|
return this.entries.get(identifier)?.stream;
|
|
64
68
|
}
|
|
69
|
+
reportStreamDead(identifier, stream) {
|
|
70
|
+
if (this.entries.get(identifier)?.stream !== stream)
|
|
71
|
+
return; // already evicted, or already replaced
|
|
72
|
+
this.logger.warn(`Stream reported dead, evicting so the next acquirer builds a new one. identifier: ${identifier}`);
|
|
73
|
+
try {
|
|
74
|
+
this.evict(identifier, stream);
|
|
75
|
+
}
|
|
76
|
+
catch (e) {
|
|
77
|
+
this.logger.error(`Failed to evict dead stream. identifier: ${identifier}`, e);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
65
80
|
removeAllStreams() {
|
|
66
81
|
for (const [, entry] of this.entries.entries()) {
|
|
67
82
|
try {
|
|
@@ -82,13 +97,13 @@ class StreamRegister {
|
|
|
82
97
|
return;
|
|
83
98
|
// Wait for init to settle before tearing down, so init can never open resources on an already
|
|
84
99
|
// stopped stream. A re-acquire in the meantime revives the entry and cancels the teardown.
|
|
85
|
-
|
|
86
|
-
.catch(() => undefined)
|
|
87
|
-
.then(() => {
|
|
100
|
+
const evictIfUnheld = () => {
|
|
88
101
|
if (this.entries.get(identifier) === entry && entry.refCount <= 0) {
|
|
89
102
|
this.evict(identifier, entry.stream);
|
|
90
103
|
}
|
|
91
|
-
}
|
|
104
|
+
};
|
|
105
|
+
void entry.ready
|
|
106
|
+
.then(() => evictIfUnheld(), () => evictIfUnheld())
|
|
92
107
|
.catch((e) => this.logger.error(`Failed to evict stream. identifier: ${identifier}`, e));
|
|
93
108
|
}
|
|
94
109
|
evict(identifier, stream) {
|
|
@@ -29,7 +29,8 @@ class CallerStreamAbstract {
|
|
|
29
29
|
this.token = token;
|
|
30
30
|
}
|
|
31
31
|
stopStream() {
|
|
32
|
-
|
|
32
|
+
this.statusStream$.next(models_1.SubscriberStatusEnum.STOPPED);
|
|
33
|
+
this.statusStream$.complete();
|
|
33
34
|
}
|
|
34
35
|
invokeRequest(params) {
|
|
35
36
|
this.logger.info('invokeRequest', params);
|
|
@@ -31,7 +31,8 @@ class CommandStreamAbstract {
|
|
|
31
31
|
this.token = token;
|
|
32
32
|
}
|
|
33
33
|
stopStream() {
|
|
34
|
-
|
|
34
|
+
this.statusStream$.next(models_1.SubscriberStatusEnum.STOPPED);
|
|
35
|
+
this.statusStream$.complete();
|
|
35
36
|
}
|
|
36
37
|
invokeRequest(params) {
|
|
37
38
|
if (params.timeoutMs === undefined || params.timeoutMs === null) {
|
|
@@ -28,7 +28,8 @@ class ControlStreamAbstract {
|
|
|
28
28
|
this.token = token;
|
|
29
29
|
}
|
|
30
30
|
stopStream() {
|
|
31
|
-
|
|
31
|
+
this.statusStream$.next(models_1.SubscriberStatusEnum.STOPPED);
|
|
32
|
+
this.statusStream$.complete();
|
|
32
33
|
}
|
|
33
34
|
takeActionWithAck(action, params, ackTimeoutInSeconds = 15) {
|
|
34
35
|
return new rxjs_1.Observable((emitter) => {
|
|
@@ -27,7 +27,8 @@ class FileAccessorStreamAbstract {
|
|
|
27
27
|
this.token = token;
|
|
28
28
|
}
|
|
29
29
|
stopStream() {
|
|
30
|
-
|
|
30
|
+
this.statusStream$.next(models_1.SubscriberStatusEnum.STOPPED);
|
|
31
|
+
this.statusStream$.complete();
|
|
31
32
|
}
|
|
32
33
|
async serverReqOp(params) {
|
|
33
34
|
return this.serverReqOpInternal(params);
|
|
@@ -28,7 +28,8 @@ class SearchStreamAbstract {
|
|
|
28
28
|
this.token = token;
|
|
29
29
|
}
|
|
30
30
|
stopStream() {
|
|
31
|
-
|
|
31
|
+
this.statusStream$.next(models_1.SubscriberStatusEnum.STOPPED);
|
|
32
|
+
this.statusStream$.complete();
|
|
32
33
|
}
|
|
33
34
|
searchStream(params) {
|
|
34
35
|
const request = this.convertToQueryMessage(params.query);
|
|
@@ -24,6 +24,7 @@ class TelemetryStream extends TelemetryStreamAbstract_1.TelemetryStreamAbstract
|
|
|
24
24
|
this.client = new teletubby_pb_client_1.TelemetryGatewayClient(transport);
|
|
25
25
|
}
|
|
26
26
|
stopInternal() {
|
|
27
|
+
this.subscriberId = undefined;
|
|
27
28
|
if (this.abortController) {
|
|
28
29
|
this.logger.info('Starting stop stream process');
|
|
29
30
|
this.abortController.abort();
|
|
@@ -29,6 +29,7 @@ export declare abstract class TelemetryStreamAbstract implements ITelemetryStrea
|
|
|
29
29
|
private sendHeartbeatTime;
|
|
30
30
|
private receivedDataStats;
|
|
31
31
|
private subscriptions;
|
|
32
|
+
private receiverDead;
|
|
32
33
|
constructor(config: ITelemetryStreamConfig);
|
|
33
34
|
protected abstract stopInternal(): void;
|
|
34
35
|
protected abstract registerReceiver(): () => void;
|
|
@@ -41,6 +42,8 @@ export declare abstract class TelemetryStreamAbstract implements ITelemetryStrea
|
|
|
41
42
|
stopStream(): void;
|
|
42
43
|
addSubscription(params: ITelemetryParams): void;
|
|
43
44
|
removeSubscription(params: ITelemetryParams, terminateReceiverGroup?: boolean): Promise<void>;
|
|
45
|
+
private onBackgroundActionFailure;
|
|
46
|
+
private markReceiverDead;
|
|
44
47
|
private isQueryLookup;
|
|
45
48
|
private getSubscribedSources;
|
|
46
49
|
sendAcknowledgment(uid: string, status: TelemetryAckStatus, noRetry: boolean): boolean;
|
|
@@ -55,6 +58,7 @@ export declare abstract class TelemetryStreamAbstract implements ITelemetryStrea
|
|
|
55
58
|
private takeTelemetryAction;
|
|
56
59
|
protected listenMessagesAndRenew(): void;
|
|
57
60
|
private addTerminateToAction;
|
|
61
|
+
private renewReceiver;
|
|
58
62
|
private messageTimeoutTryAgain;
|
|
59
63
|
/**
|
|
60
64
|
* Auto resubscribe to reduce CPU usage.
|
|
@@ -13,6 +13,41 @@ const arrayRemove_1 = require("../../../helpers/arrayRemove");
|
|
|
13
13
|
const arrayUnique_1 = require("../../../helpers/arrayUnique");
|
|
14
14
|
const operators_1 = require("rxjs/operators");
|
|
15
15
|
const getSubscriptionsDifference_1 = require("../../../helpers/getSubscriptionsDifference");
|
|
16
|
+
const telemetryActionQueueStates = new WeakMap();
|
|
17
|
+
const resolveTelemetryActions = (actions) => {
|
|
18
|
+
actions.forEach((action) => action.resolve());
|
|
19
|
+
};
|
|
20
|
+
const resolveUnsentTelemetryActions = (queueState) => {
|
|
21
|
+
resolveTelemetryActions(queueState.pendingActions.splice(0));
|
|
22
|
+
};
|
|
23
|
+
const isReceiverGoneError = (error) => {
|
|
24
|
+
const code = error?.code;
|
|
25
|
+
if (typeof code === 'number')
|
|
26
|
+
return code === 9; // gRPC FAILED_PRECONDITION
|
|
27
|
+
if (typeof code === 'string')
|
|
28
|
+
return code.toLowerCase() === 'failed_precondition';
|
|
29
|
+
return false;
|
|
30
|
+
};
|
|
31
|
+
const takeNextTelemetryBatch = (queueState) => {
|
|
32
|
+
const firstAction = queueState.pendingActions.shift();
|
|
33
|
+
if (!firstAction)
|
|
34
|
+
return [];
|
|
35
|
+
const activeBatch = [firstAction];
|
|
36
|
+
if (firstAction.callsignsLookup.isQuery() || firstAction.terminate)
|
|
37
|
+
return activeBatch;
|
|
38
|
+
while (queueState.pendingActions.length > 0) {
|
|
39
|
+
const [nextAction] = queueState.pendingActions;
|
|
40
|
+
if (nextAction.callsignsLookup.isQuery() ||
|
|
41
|
+
nextAction.terminate ||
|
|
42
|
+
nextAction.actionType !== firstAction.actionType ||
|
|
43
|
+
nextAction.subscriberId !== firstAction.subscriberId ||
|
|
44
|
+
nextAction.registration !== firstAction.registration) {
|
|
45
|
+
break;
|
|
46
|
+
}
|
|
47
|
+
activeBatch.push(queueState.pendingActions.shift());
|
|
48
|
+
}
|
|
49
|
+
return activeBatch;
|
|
50
|
+
};
|
|
16
51
|
class TelemetryStreamAbstract {
|
|
17
52
|
constructor(config) {
|
|
18
53
|
this.telemetryActionQueue = Promise.resolve();
|
|
@@ -26,6 +61,7 @@ class TelemetryStreamAbstract {
|
|
|
26
61
|
this.sendHeartbeatTime = 5000;
|
|
27
62
|
this.receivedDataStats = {};
|
|
28
63
|
this.subscriptions = new Map();
|
|
64
|
+
this.receiverDead = false;
|
|
29
65
|
this.identifier = StreamRegister_1.StreamRegister.getIdentifier(identifier_1.IDENTIFIER_NAME_TELEMETRY, config.scope);
|
|
30
66
|
this.projectId = config.projectId;
|
|
31
67
|
this.callsignsLookup = config.callsignsLookup;
|
|
@@ -38,6 +74,14 @@ class TelemetryStreamAbstract {
|
|
|
38
74
|
this.messageStream$ = new rxjs_1.Subject();
|
|
39
75
|
this.statusStream$ = new rxjs_1.BehaviorSubject(models_1.SubscriberStatusEnum.STOPPED);
|
|
40
76
|
this.heartbeat = new StreamHeartbeat_1.StreamHeartbeat(this.identifier);
|
|
77
|
+
telemetryActionQueueStates.set(this, {
|
|
78
|
+
pendingActions: [],
|
|
79
|
+
activeBatch: [],
|
|
80
|
+
registration: 0,
|
|
81
|
+
drainScheduled: false,
|
|
82
|
+
requestInFlight: false,
|
|
83
|
+
stopped: false,
|
|
84
|
+
});
|
|
41
85
|
RocosStore_1.RocosStore.getChangeSubject()
|
|
42
86
|
.pipe((0, operators_1.filter)((message) => {
|
|
43
87
|
return message.type === 'token' && message.url === this.url;
|
|
@@ -55,6 +99,12 @@ class TelemetryStreamAbstract {
|
|
|
55
99
|
this.registerReceiver();
|
|
56
100
|
}
|
|
57
101
|
stopStream() {
|
|
102
|
+
const queueState = telemetryActionQueueStates.get(this);
|
|
103
|
+
if (queueState) {
|
|
104
|
+
queueState.stopped = true;
|
|
105
|
+
resolveUnsentTelemetryActions(queueState);
|
|
106
|
+
}
|
|
107
|
+
this.subscriberId = undefined;
|
|
58
108
|
this.heartbeat?.stop();
|
|
59
109
|
this.stopInternal();
|
|
60
110
|
if (this.messagesTimer) {
|
|
@@ -87,7 +137,9 @@ class TelemetryStreamAbstract {
|
|
|
87
137
|
const toAdd = (0, arrayRemove_1.arrayRemove)(this.sources, sourcesBefore);
|
|
88
138
|
if (toAdd.length) {
|
|
89
139
|
this.logger.debug('New query subscriptions added, we need to refresh sources', { toAdd });
|
|
90
|
-
void this.enqueueTelemetryAction('subscribe', this.callsignsLookup, toAdd)
|
|
140
|
+
void this.enqueueTelemetryAction('subscribe', this.callsignsLookup, toAdd).catch((error) => {
|
|
141
|
+
this.onBackgroundActionFailure('subscribe', error);
|
|
142
|
+
});
|
|
91
143
|
}
|
|
92
144
|
}
|
|
93
145
|
else {
|
|
@@ -98,7 +150,9 @@ class TelemetryStreamAbstract {
|
|
|
98
150
|
const { toAdd } = (0, getSubscriptionsDifference_1.getSubscriptionsDifference)(before, after);
|
|
99
151
|
if (toAdd.callsigns.length && toAdd.sources.length) {
|
|
100
152
|
this.logger.debug('New subscriptions added, we need to refresh sources', { toAdd });
|
|
101
|
-
void this.enqueueTelemetryAction('subscribe', new models_1.CallsignsLookup(toAdd.callsigns), toAdd.sources)
|
|
153
|
+
void this.enqueueTelemetryAction('subscribe', new models_1.CallsignsLookup(toAdd.callsigns), toAdd.sources).catch((error) => {
|
|
154
|
+
this.onBackgroundActionFailure('subscribe', error);
|
|
155
|
+
});
|
|
102
156
|
}
|
|
103
157
|
}
|
|
104
158
|
}
|
|
@@ -133,7 +187,7 @@ class TelemetryStreamAbstract {
|
|
|
133
187
|
await this.enqueueTelemetryAction('unsubscribe', this.callsignsLookup, toRemove, terminateReceiverGroup);
|
|
134
188
|
}
|
|
135
189
|
catch (err) {
|
|
136
|
-
this.
|
|
190
|
+
this.onBackgroundActionFailure('unsubscribe', err);
|
|
137
191
|
}
|
|
138
192
|
}
|
|
139
193
|
return;
|
|
@@ -148,10 +202,24 @@ class TelemetryStreamAbstract {
|
|
|
148
202
|
await this.enqueueTelemetryAction('unsubscribe', new models_1.CallsignsLookup(toRemove.callsigns), toRemove.sources, terminateReceiverGroup);
|
|
149
203
|
}
|
|
150
204
|
catch (err) {
|
|
151
|
-
this.
|
|
205
|
+
this.onBackgroundActionFailure('unsubscribe', err);
|
|
152
206
|
}
|
|
153
207
|
}
|
|
154
208
|
}
|
|
209
|
+
onBackgroundActionFailure(actionType, error) {
|
|
210
|
+
this.logger.error(`Failed to ${actionType}: ${error}`);
|
|
211
|
+
if (isReceiverGoneError(error)) {
|
|
212
|
+
this.markReceiverDead(error);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
markReceiverDead(error) {
|
|
216
|
+
if (this.receiverDead)
|
|
217
|
+
return;
|
|
218
|
+
this.receiverDead = true;
|
|
219
|
+
this.logger.error('Receiver is no longer registered with the gateway, retiring the stream', error);
|
|
220
|
+
StreamRegister_1.StreamRegister.getInstance().reportStreamDead(this.identifier, this);
|
|
221
|
+
this.messageStream$.error(error);
|
|
222
|
+
}
|
|
155
223
|
isQueryLookup() {
|
|
156
224
|
return this.callsignsLookup?.isQuery() ?? false;
|
|
157
225
|
}
|
|
@@ -204,14 +272,22 @@ class TelemetryStreamAbstract {
|
|
|
204
272
|
// Subscribe message
|
|
205
273
|
case grpc_1.GRPC_SOURCE_SUBSCRIBED: {
|
|
206
274
|
const { subscriberId } = rocosMessage.payload;
|
|
275
|
+
const queueState = telemetryActionQueueStates.get(this);
|
|
276
|
+
if (queueState?.stopped)
|
|
277
|
+
return;
|
|
207
278
|
if (this.subscriberId && subscriberId) {
|
|
208
279
|
this.logger.warn(`Replacing old subscriberId (${this.subscriberId}) with a new one (${subscriberId})`);
|
|
209
280
|
}
|
|
210
|
-
|
|
281
|
+
if (queueState) {
|
|
282
|
+
queueState.registration++;
|
|
283
|
+
resolveUnsentTelemetryActions(queueState);
|
|
284
|
+
}
|
|
211
285
|
this.subscriberId = subscriberId;
|
|
212
286
|
rocosMessage.subscriberId = this.subscriberId;
|
|
213
287
|
this.logger.info('onData', `subscriberId has been updated - ${subscriberId}`, { subscriberId });
|
|
214
|
-
void this.enqueueTelemetryAction('subscribe')
|
|
288
|
+
void this.enqueueTelemetryAction('subscribe').catch((error) => {
|
|
289
|
+
this.onBackgroundActionFailure('subscribe', error);
|
|
290
|
+
});
|
|
215
291
|
this.heartbeat.start(() => this.sendHeartbeat(), this.sendHeartbeatTime);
|
|
216
292
|
isRobotMessage = false;
|
|
217
293
|
break;
|
|
@@ -253,21 +329,69 @@ class TelemetryStreamAbstract {
|
|
|
253
329
|
* unsubscribe landing last strands a subscribe and nothing re-adds it.
|
|
254
330
|
*/
|
|
255
331
|
enqueueTelemetryAction(actionType, callsignsLookup, sources, terminate) {
|
|
256
|
-
const
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
332
|
+
const queueState = telemetryActionQueueStates.get(this);
|
|
333
|
+
if (!queueState || queueState.stopped)
|
|
334
|
+
return Promise.resolve();
|
|
335
|
+
const completion = new Promise((resolve, reject) => {
|
|
336
|
+
queueState.pendingActions.push({
|
|
337
|
+
actionType,
|
|
338
|
+
callsignsLookup: callsignsLookup ?? this.callsignsLookup,
|
|
339
|
+
sources: [...(sources ?? this.sources)],
|
|
340
|
+
terminate: terminate ?? false,
|
|
341
|
+
subscriberId: this.subscriberId,
|
|
342
|
+
registration: queueState.registration,
|
|
343
|
+
resolve,
|
|
344
|
+
reject,
|
|
345
|
+
});
|
|
346
|
+
});
|
|
347
|
+
const scheduleDrain = () => {
|
|
348
|
+
if (queueState.stopped ||
|
|
349
|
+
queueState.requestInFlight ||
|
|
350
|
+
queueState.drainScheduled ||
|
|
351
|
+
queueState.pendingActions.length === 0) {
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
queueState.drainScheduled = true;
|
|
355
|
+
this.telemetryActionQueue = this.telemetryActionQueue.then(async () => {
|
|
356
|
+
queueState.drainScheduled = false;
|
|
357
|
+
if (queueState.stopped || queueState.requestInFlight)
|
|
358
|
+
return;
|
|
359
|
+
const activeBatch = takeNextTelemetryBatch(queueState);
|
|
360
|
+
const [firstAction] = activeBatch;
|
|
361
|
+
if (!firstAction)
|
|
362
|
+
return;
|
|
363
|
+
queueState.activeBatch = activeBatch;
|
|
364
|
+
queueState.requestInFlight = true;
|
|
365
|
+
try {
|
|
366
|
+
await this.takeTelemetryAction(firstAction.actionType, firstAction.callsignsLookup, firstAction.sources);
|
|
367
|
+
resolveTelemetryActions(activeBatch);
|
|
368
|
+
}
|
|
369
|
+
catch (error) {
|
|
370
|
+
activeBatch.forEach((action) => action.reject(error));
|
|
371
|
+
}
|
|
372
|
+
finally {
|
|
373
|
+
queueState.activeBatch = [];
|
|
374
|
+
queueState.requestInFlight = false;
|
|
375
|
+
scheduleDrain();
|
|
376
|
+
}
|
|
377
|
+
});
|
|
378
|
+
};
|
|
379
|
+
scheduleDrain();
|
|
380
|
+
return completion;
|
|
262
381
|
}
|
|
263
|
-
async takeTelemetryAction(actionType, callsignsLookup, sources
|
|
382
|
+
async takeTelemetryAction(actionType, callsignsLookup, sources) {
|
|
264
383
|
this.logger.debug('takeTelemetryAction', actionType);
|
|
384
|
+
const queueState = telemetryActionQueueStates.get(this);
|
|
385
|
+
const activeBatch = queueState?.activeBatch ?? [];
|
|
386
|
+
const [firstAction] = activeBatch;
|
|
265
387
|
// A config fault, not a race: this stream can never address a request.
|
|
266
388
|
if (!this.projectId) {
|
|
267
389
|
this.logger.warn('takeTelemetryAction', actionType, `${actionType} without a projectId - dropped`);
|
|
268
390
|
return Promise.resolve();
|
|
269
391
|
}
|
|
270
|
-
if (!
|
|
392
|
+
if (!firstAction?.subscriberId ||
|
|
393
|
+
firstAction.subscriberId !== this.subscriberId ||
|
|
394
|
+
firstAction.registration !== queueState?.registration) {
|
|
271
395
|
// Deferred, not lost: `onData` re-asserts `this.sources` when the subscriberId arrives.
|
|
272
396
|
this.logger.debug('takeTelemetryAction', actionType, `${actionType} before the receiver was registered - deferred to the post-registration re-assert`);
|
|
273
397
|
return Promise.resolve();
|
|
@@ -275,20 +399,20 @@ class TelemetryStreamAbstract {
|
|
|
275
399
|
const actionSources = sources ?? this.sources;
|
|
276
400
|
const actionCallsigns = callsignsLookup ?? this.callsignsLookup;
|
|
277
401
|
if (!actionCallsigns.isQuery()) {
|
|
278
|
-
// Compose the request message
|
|
279
402
|
const req = teletubby_pb_1.TelemetryRequest.create({
|
|
280
|
-
subscriberId:
|
|
281
|
-
requestedActions:
|
|
282
|
-
{
|
|
283
|
-
operation: actionType,
|
|
284
|
-
callsigns:
|
|
285
|
-
sources:
|
|
286
|
-
}
|
|
287
|
-
|
|
403
|
+
subscriberId: firstAction.subscriberId,
|
|
404
|
+
requestedActions: activeBatch.map((action) => {
|
|
405
|
+
const telemetryAction = teletubby_pb_1.TelemetryAction.create({
|
|
406
|
+
operation: action.actionType,
|
|
407
|
+
callsigns: action.callsignsLookup.getCallsigns(),
|
|
408
|
+
sources: action.sources,
|
|
409
|
+
});
|
|
410
|
+
if (action.actionType === 'unsubscribe' && action.terminate) {
|
|
411
|
+
this.addTerminateToAction(telemetryAction);
|
|
412
|
+
}
|
|
413
|
+
return telemetryAction;
|
|
414
|
+
}),
|
|
288
415
|
});
|
|
289
|
-
if (actionType === 'unsubscribe' && terminate) {
|
|
290
|
-
this.addTerminateToAction(req.requestedActions[0]);
|
|
291
|
-
}
|
|
292
416
|
this.logger.debug('takeTelemetryAction', actionType, 'client.requestTelemetry', this.subscriberId);
|
|
293
417
|
// Send the message to back-end.
|
|
294
418
|
return this.requestTelemetry(req);
|
|
@@ -313,7 +437,7 @@ class TelemetryStreamAbstract {
|
|
|
313
437
|
});
|
|
314
438
|
// Compose the request message
|
|
315
439
|
const req = teletubby_pb_1.TelemetryQueryRequest.create({
|
|
316
|
-
subscriberId:
|
|
440
|
+
subscriberId: firstAction.subscriberId,
|
|
317
441
|
sources: actionSources,
|
|
318
442
|
operation: actionType,
|
|
319
443
|
callsignQuery: {
|
|
@@ -413,14 +537,22 @@ class TelemetryStreamAbstract {
|
|
|
413
537
|
unsubscribeOperation,
|
|
414
538
|
};
|
|
415
539
|
}
|
|
540
|
+
renewReceiver() {
|
|
541
|
+
try {
|
|
542
|
+
// Remove the received call anyway.
|
|
543
|
+
this.stopInternal();
|
|
544
|
+
this.lastMessageReceived = undefined;
|
|
545
|
+
this.lastRobotMessageReceived = undefined;
|
|
546
|
+
this.registerReceiver();
|
|
547
|
+
}
|
|
548
|
+
catch (e) {
|
|
549
|
+
this.logger.error('Failed to renew the receiver, retrying on the next check', e);
|
|
550
|
+
}
|
|
551
|
+
}
|
|
416
552
|
messageTimeoutTryAgain() {
|
|
417
553
|
// Increase retry times.
|
|
418
554
|
this.retryTimes++;
|
|
419
|
-
|
|
420
|
-
this.stopInternal();
|
|
421
|
-
this.lastMessageReceived = undefined;
|
|
422
|
-
this.lastRobotMessageReceived = undefined;
|
|
423
|
-
this.registerReceiver();
|
|
555
|
+
this.renewReceiver();
|
|
424
556
|
}
|
|
425
557
|
/**
|
|
426
558
|
* Auto resubscribe to reduce CPU usage.
|
|
@@ -428,11 +560,7 @@ class TelemetryStreamAbstract {
|
|
|
428
560
|
autoResubscribe() {
|
|
429
561
|
// Reset
|
|
430
562
|
this.retryTimes = 0;
|
|
431
|
-
|
|
432
|
-
this.stopInternal();
|
|
433
|
-
this.lastMessageReceived = undefined;
|
|
434
|
-
this.lastRobotMessageReceived = undefined;
|
|
435
|
-
this.registerReceiver();
|
|
563
|
+
this.renewReceiver();
|
|
436
564
|
}
|
|
437
565
|
isFoundSource(source) {
|
|
438
566
|
return this.sources.indexOf(source) !== -1;
|
|
@@ -40,6 +40,7 @@ class TelemetryStreamConnect extends TelemetryStreamAbstract_1.TelemetryStreamAb
|
|
|
40
40
|
this.client = (0, connect_1.createClient)(teletubby_pb_1.TelemetryGateway, transport);
|
|
41
41
|
}
|
|
42
42
|
stopInternal() {
|
|
43
|
+
this.subscriberId = undefined;
|
|
43
44
|
if (this.abortController) {
|
|
44
45
|
this.logger.info('Starting stop stream process');
|
|
45
46
|
this.abortController.abort();
|
|
@@ -48,6 +48,7 @@ class TelemetryStreamNode extends TelemetryStreamAbstract_1.TelemetryStreamAbstr
|
|
|
48
48
|
this.credentials = config.insecure ? grpc.credentials.createInsecure() : grpc.credentials.createSsl();
|
|
49
49
|
}
|
|
50
50
|
stopInternal() {
|
|
51
|
+
this.subscriberId = undefined;
|
|
51
52
|
try {
|
|
52
53
|
this.logger.info('Starting stop stream process');
|
|
53
54
|
this.logger.info('Stopping stream');
|
package/cjs/models/maps/Map.d.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.SearchService = void 0;
|
|
4
|
-
const models_1 = require("../models");
|
|
5
4
|
const rxjs_1 = require("rxjs");
|
|
6
5
|
const BaseStreamService_1 = require("./BaseStreamService");
|
|
7
6
|
const identifier_1 = require("../constants/identifier");
|
|
@@ -47,12 +46,8 @@ class SearchService extends BaseStreamService_1.BaseStreamService {
|
|
|
47
46
|
port: this.config.port,
|
|
48
47
|
insecure: this.config.insecure,
|
|
49
48
|
});
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
this.status = msg === models_1.SubscriberStatusEnum.STOPPED || msg === models_1.SubscriberStatusEnum.ALIVE;
|
|
53
|
-
this.status$.next(msg);
|
|
54
|
-
});
|
|
55
|
-
}
|
|
49
|
+
// The status forwarder is wired once per stream by BaseStreamService.acquireStream's factory
|
|
50
|
+
// (and released now that the stream completes statusStream$ on stop), so no per-service one here.
|
|
56
51
|
return newStream.stream;
|
|
57
52
|
}
|
|
58
53
|
getStream(config) {
|
|
@@ -28,6 +28,10 @@ export declare class TelemetryService extends BaseStreamService<ITelemetryStream
|
|
|
28
28
|
getStatusChanges(): Observable<IStreamStatusMessage>;
|
|
29
29
|
/**
|
|
30
30
|
* Subscribes to the heartbeat telemetry and checks we receive one at least every 5s
|
|
31
|
+
*
|
|
32
|
+
* @deprecated Superseded by {@link RobotStatusService.getRobotStatusChanges}, which reports the
|
|
33
|
+
* normalised robot status (reported health/readiness plus platform connectivity) rather than only
|
|
34
|
+
* heartbeat-derived online/offline liveness.
|
|
31
35
|
* @param projectId
|
|
32
36
|
* @param callsign
|
|
33
37
|
* @param heartbeatTimeoutMs optionally override the deadline for heartbeats (in ms). Default 5000ms
|
|
@@ -88,8 +88,16 @@ class TelemetryService extends BaseStreamService_1.BaseStreamService {
|
|
|
88
88
|
return (0, rxjs_1.from)(messages$).pipe((0, rxjs_1.mergeAll)(), (0, operators_1.filter)((message) => {
|
|
89
89
|
return callsignsLookup.includesCallsign(message.callsign) && sources.includes(message.source);
|
|
90
90
|
}), (0, operators_1.finalize)(() => {
|
|
91
|
-
|
|
92
|
-
|
|
91
|
+
if (params.terminateReceiverGroupOnUnsubscribe) {
|
|
92
|
+
const removal = stream.removeSubscription(subscriptionParams, true);
|
|
93
|
+
void Promise.resolve(removal)
|
|
94
|
+
.finally(release)
|
|
95
|
+
.catch(() => undefined);
|
|
96
|
+
}
|
|
97
|
+
else {
|
|
98
|
+
release();
|
|
99
|
+
void Promise.resolve(stream.removeSubscription(subscriptionParams)).catch(() => undefined);
|
|
100
|
+
}
|
|
93
101
|
}));
|
|
94
102
|
});
|
|
95
103
|
}
|
|
@@ -111,6 +119,10 @@ class TelemetryService extends BaseStreamService_1.BaseStreamService {
|
|
|
111
119
|
}
|
|
112
120
|
/**
|
|
113
121
|
* Subscribes to the heartbeat telemetry and checks we receive one at least every 5s
|
|
122
|
+
*
|
|
123
|
+
* @deprecated Superseded by {@link RobotStatusService.getRobotStatusChanges}, which reports the
|
|
124
|
+
* normalised robot status (reported health/readiness plus platform connectivity) rather than only
|
|
125
|
+
* heartbeat-derived online/offline liveness.
|
|
114
126
|
* @param projectId
|
|
115
127
|
* @param callsign
|
|
116
128
|
* @param heartbeatTimeoutMs optionally override the deadline for heartbeats (in ms). Default 5000ms
|
|
@@ -10,8 +10,12 @@ export interface IAcquiredStream<T extends IBaseStream> {
|
|
|
10
10
|
/**
|
|
11
11
|
* Sole owner of stream lifecycle. Streams are shared per identifier and refcounted: acquireStream
|
|
12
12
|
* gets-or-creates and increments in one synchronous step, and the returned handle's release()
|
|
13
|
-
* tears the stream down when the last holder leaves.
|
|
14
|
-
*
|
|
13
|
+
* tears the stream down when the last holder leaves.
|
|
14
|
+
*
|
|
15
|
+
* A held stream can also be stopped and replaced underneath its holders, by reportStreamDead, when
|
|
16
|
+
* the stream can no longer carry anything. Holders keep the instance they were handed; their
|
|
17
|
+
* release() finds the entry gone and does nothing, so a replacement never inherits a departing
|
|
18
|
+
* holder's refcount and a refcount can not go negative.
|
|
15
19
|
*/
|
|
16
20
|
export declare class StreamRegister {
|
|
17
21
|
private entries;
|
|
@@ -22,6 +26,7 @@ export declare class StreamRegister {
|
|
|
22
26
|
static getIdentifier($identifier: string, scope?: string): string;
|
|
23
27
|
acquireStream<T extends IBaseStream>(identifier: string, create: () => T, init: (stream: T) => Promise<void>): IAcquiredStream<T>;
|
|
24
28
|
getStream(identifier: string): IBaseStream | undefined;
|
|
29
|
+
reportStreamDead(identifier: string, stream: IBaseStream): void;
|
|
25
30
|
removeAllStreams(): boolean;
|
|
26
31
|
private release;
|
|
27
32
|
private evict;
|
|
@@ -3,8 +3,12 @@ import { RocosLogger } from '../logger/RocosLogger';
|
|
|
3
3
|
/**
|
|
4
4
|
* Sole owner of stream lifecycle. Streams are shared per identifier and refcounted: acquireStream
|
|
5
5
|
* gets-or-creates and increments in one synchronous step, and the returned handle's release()
|
|
6
|
-
* tears the stream down when the last holder leaves.
|
|
7
|
-
*
|
|
6
|
+
* tears the stream down when the last holder leaves.
|
|
7
|
+
*
|
|
8
|
+
* A held stream can also be stopped and replaced underneath its holders, by reportStreamDead, when
|
|
9
|
+
* the stream can no longer carry anything. Holders keep the instance they were handed; their
|
|
10
|
+
* release() finds the entry gone and does nothing, so a replacement never inherits a departing
|
|
11
|
+
* holder's refcount and a refcount can not go negative.
|
|
8
12
|
*/
|
|
9
13
|
export class StreamRegister {
|
|
10
14
|
constructor() {
|
|
@@ -59,6 +63,17 @@ export class StreamRegister {
|
|
|
59
63
|
getStream(identifier) {
|
|
60
64
|
return this.entries.get(identifier)?.stream;
|
|
61
65
|
}
|
|
66
|
+
reportStreamDead(identifier, stream) {
|
|
67
|
+
if (this.entries.get(identifier)?.stream !== stream)
|
|
68
|
+
return; // already evicted, or already replaced
|
|
69
|
+
this.logger.warn(`Stream reported dead, evicting so the next acquirer builds a new one. identifier: ${identifier}`);
|
|
70
|
+
try {
|
|
71
|
+
this.evict(identifier, stream);
|
|
72
|
+
}
|
|
73
|
+
catch (e) {
|
|
74
|
+
this.logger.error(`Failed to evict dead stream. identifier: ${identifier}`, e);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
62
77
|
removeAllStreams() {
|
|
63
78
|
for (const [, entry] of this.entries.entries()) {
|
|
64
79
|
try {
|
|
@@ -79,13 +94,13 @@ export class StreamRegister {
|
|
|
79
94
|
return;
|
|
80
95
|
// Wait for init to settle before tearing down, so init can never open resources on an already
|
|
81
96
|
// stopped stream. A re-acquire in the meantime revives the entry and cancels the teardown.
|
|
82
|
-
|
|
83
|
-
.catch(() => undefined)
|
|
84
|
-
.then(() => {
|
|
97
|
+
const evictIfUnheld = () => {
|
|
85
98
|
if (this.entries.get(identifier) === entry && entry.refCount <= 0) {
|
|
86
99
|
this.evict(identifier, entry.stream);
|
|
87
100
|
}
|
|
88
|
-
}
|
|
101
|
+
};
|
|
102
|
+
void entry.ready
|
|
103
|
+
.then(() => evictIfUnheld(), () => evictIfUnheld())
|
|
89
104
|
.catch((e) => this.logger.error(`Failed to evict stream. identifier: ${identifier}`, e));
|
|
90
105
|
}
|
|
91
106
|
evict(identifier, stream) {
|
|
@@ -26,7 +26,8 @@ export class CallerStreamAbstract {
|
|
|
26
26
|
this.token = token;
|
|
27
27
|
}
|
|
28
28
|
stopStream() {
|
|
29
|
-
|
|
29
|
+
this.statusStream$.next(SubscriberStatusEnum.STOPPED);
|
|
30
|
+
this.statusStream$.complete();
|
|
30
31
|
}
|
|
31
32
|
invokeRequest(params) {
|
|
32
33
|
this.logger.info('invokeRequest', params);
|
|
@@ -28,7 +28,8 @@ export class CommandStreamAbstract {
|
|
|
28
28
|
this.token = token;
|
|
29
29
|
}
|
|
30
30
|
stopStream() {
|
|
31
|
-
|
|
31
|
+
this.statusStream$.next(SubscriberStatusEnum.STOPPED);
|
|
32
|
+
this.statusStream$.complete();
|
|
32
33
|
}
|
|
33
34
|
invokeRequest(params) {
|
|
34
35
|
if (params.timeoutMs === undefined || params.timeoutMs === null) {
|
|
@@ -25,7 +25,8 @@ export class ControlStreamAbstract {
|
|
|
25
25
|
this.token = token;
|
|
26
26
|
}
|
|
27
27
|
stopStream() {
|
|
28
|
-
|
|
28
|
+
this.statusStream$.next(SubscriberStatusEnum.STOPPED);
|
|
29
|
+
this.statusStream$.complete();
|
|
29
30
|
}
|
|
30
31
|
takeActionWithAck(action, params, ackTimeoutInSeconds = 15) {
|
|
31
32
|
return new Observable((emitter) => {
|
|
@@ -24,7 +24,8 @@ export class FileAccessorStreamAbstract {
|
|
|
24
24
|
this.token = token;
|
|
25
25
|
}
|
|
26
26
|
stopStream() {
|
|
27
|
-
|
|
27
|
+
this.statusStream$.next(SubscriberStatusEnum.STOPPED);
|
|
28
|
+
this.statusStream$.complete();
|
|
28
29
|
}
|
|
29
30
|
async serverReqOp(params) {
|
|
30
31
|
return this.serverReqOpInternal(params);
|
|
@@ -25,7 +25,8 @@ export class SearchStreamAbstract {
|
|
|
25
25
|
this.token = token;
|
|
26
26
|
}
|
|
27
27
|
stopStream() {
|
|
28
|
-
|
|
28
|
+
this.statusStream$.next(SubscriberStatusEnum.STOPPED);
|
|
29
|
+
this.statusStream$.complete();
|
|
29
30
|
}
|
|
30
31
|
searchStream(params) {
|
|
31
32
|
const request = this.convertToQueryMessage(params.query);
|
|
@@ -21,6 +21,7 @@ export class TelemetryStream extends TelemetryStreamAbstract {
|
|
|
21
21
|
this.client = new TelemetryGatewayClient(transport);
|
|
22
22
|
}
|
|
23
23
|
stopInternal() {
|
|
24
|
+
this.subscriberId = undefined;
|
|
24
25
|
if (this.abortController) {
|
|
25
26
|
this.logger.info('Starting stop stream process');
|
|
26
27
|
this.abortController.abort();
|
|
@@ -29,6 +29,7 @@ export declare abstract class TelemetryStreamAbstract implements ITelemetryStrea
|
|
|
29
29
|
private sendHeartbeatTime;
|
|
30
30
|
private receivedDataStats;
|
|
31
31
|
private subscriptions;
|
|
32
|
+
private receiverDead;
|
|
32
33
|
constructor(config: ITelemetryStreamConfig);
|
|
33
34
|
protected abstract stopInternal(): void;
|
|
34
35
|
protected abstract registerReceiver(): () => void;
|
|
@@ -41,6 +42,8 @@ export declare abstract class TelemetryStreamAbstract implements ITelemetryStrea
|
|
|
41
42
|
stopStream(): void;
|
|
42
43
|
addSubscription(params: ITelemetryParams): void;
|
|
43
44
|
removeSubscription(params: ITelemetryParams, terminateReceiverGroup?: boolean): Promise<void>;
|
|
45
|
+
private onBackgroundActionFailure;
|
|
46
|
+
private markReceiverDead;
|
|
44
47
|
private isQueryLookup;
|
|
45
48
|
private getSubscribedSources;
|
|
46
49
|
sendAcknowledgment(uid: string, status: TelemetryAckStatus, noRetry: boolean): boolean;
|
|
@@ -55,6 +58,7 @@ export declare abstract class TelemetryStreamAbstract implements ITelemetryStrea
|
|
|
55
58
|
private takeTelemetryAction;
|
|
56
59
|
protected listenMessagesAndRenew(): void;
|
|
57
60
|
private addTerminateToAction;
|
|
61
|
+
private renewReceiver;
|
|
58
62
|
private messageTimeoutTryAgain;
|
|
59
63
|
/**
|
|
60
64
|
* Auto resubscribe to reduce CPU usage.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { BehaviorSubject, Subject } from 'rxjs';
|
|
2
2
|
import { CallsignsLookup, RocosTelemetryMessage, StreamOptions, SubscriberStatusEnum, } from '../../../models';
|
|
3
3
|
import { GRPC_SOURCE_NOOP, GRPC_SOURCE_SUBSCRIBED } from '../../../constants/grpc';
|
|
4
|
-
import { QueryOrPredicate, TelemetryAckStatus, TelemetryQueryRequest, TelemetryRequest, UnsubscribeOperation, } from '../../../grpc/teletubby_pb';
|
|
4
|
+
import { QueryOrPredicate, TelemetryAckStatus, TelemetryAction, TelemetryQueryRequest, TelemetryRequest, UnsubscribeOperation, } from '../../../grpc/teletubby_pb';
|
|
5
5
|
import { IDENTIFIER_NAME_TELEMETRY } from '../../../constants/identifier';
|
|
6
6
|
import { RocosStore } from '../../../store/RocosStore';
|
|
7
7
|
import { StreamHeartbeat } from '../../atoms/StreamHeartbeat';
|
|
@@ -10,6 +10,41 @@ import { arrayRemove } from '../../../helpers/arrayRemove';
|
|
|
10
10
|
import { arrayUnique } from '../../../helpers/arrayUnique';
|
|
11
11
|
import { filter } from 'rxjs/operators';
|
|
12
12
|
import { getSubscriptionsDifference } from '../../../helpers/getSubscriptionsDifference';
|
|
13
|
+
const telemetryActionQueueStates = new WeakMap();
|
|
14
|
+
const resolveTelemetryActions = (actions) => {
|
|
15
|
+
actions.forEach((action) => action.resolve());
|
|
16
|
+
};
|
|
17
|
+
const resolveUnsentTelemetryActions = (queueState) => {
|
|
18
|
+
resolveTelemetryActions(queueState.pendingActions.splice(0));
|
|
19
|
+
};
|
|
20
|
+
const isReceiverGoneError = (error) => {
|
|
21
|
+
const code = error?.code;
|
|
22
|
+
if (typeof code === 'number')
|
|
23
|
+
return code === 9; // gRPC FAILED_PRECONDITION
|
|
24
|
+
if (typeof code === 'string')
|
|
25
|
+
return code.toLowerCase() === 'failed_precondition';
|
|
26
|
+
return false;
|
|
27
|
+
};
|
|
28
|
+
const takeNextTelemetryBatch = (queueState) => {
|
|
29
|
+
const firstAction = queueState.pendingActions.shift();
|
|
30
|
+
if (!firstAction)
|
|
31
|
+
return [];
|
|
32
|
+
const activeBatch = [firstAction];
|
|
33
|
+
if (firstAction.callsignsLookup.isQuery() || firstAction.terminate)
|
|
34
|
+
return activeBatch;
|
|
35
|
+
while (queueState.pendingActions.length > 0) {
|
|
36
|
+
const [nextAction] = queueState.pendingActions;
|
|
37
|
+
if (nextAction.callsignsLookup.isQuery() ||
|
|
38
|
+
nextAction.terminate ||
|
|
39
|
+
nextAction.actionType !== firstAction.actionType ||
|
|
40
|
+
nextAction.subscriberId !== firstAction.subscriberId ||
|
|
41
|
+
nextAction.registration !== firstAction.registration) {
|
|
42
|
+
break;
|
|
43
|
+
}
|
|
44
|
+
activeBatch.push(queueState.pendingActions.shift());
|
|
45
|
+
}
|
|
46
|
+
return activeBatch;
|
|
47
|
+
};
|
|
13
48
|
export class TelemetryStreamAbstract {
|
|
14
49
|
constructor(config) {
|
|
15
50
|
this.telemetryActionQueue = Promise.resolve();
|
|
@@ -23,6 +58,7 @@ export class TelemetryStreamAbstract {
|
|
|
23
58
|
this.sendHeartbeatTime = 5000;
|
|
24
59
|
this.receivedDataStats = {};
|
|
25
60
|
this.subscriptions = new Map();
|
|
61
|
+
this.receiverDead = false;
|
|
26
62
|
this.identifier = StreamRegister.getIdentifier(IDENTIFIER_NAME_TELEMETRY, config.scope);
|
|
27
63
|
this.projectId = config.projectId;
|
|
28
64
|
this.callsignsLookup = config.callsignsLookup;
|
|
@@ -35,6 +71,14 @@ export class TelemetryStreamAbstract {
|
|
|
35
71
|
this.messageStream$ = new Subject();
|
|
36
72
|
this.statusStream$ = new BehaviorSubject(SubscriberStatusEnum.STOPPED);
|
|
37
73
|
this.heartbeat = new StreamHeartbeat(this.identifier);
|
|
74
|
+
telemetryActionQueueStates.set(this, {
|
|
75
|
+
pendingActions: [],
|
|
76
|
+
activeBatch: [],
|
|
77
|
+
registration: 0,
|
|
78
|
+
drainScheduled: false,
|
|
79
|
+
requestInFlight: false,
|
|
80
|
+
stopped: false,
|
|
81
|
+
});
|
|
38
82
|
RocosStore.getChangeSubject()
|
|
39
83
|
.pipe(filter((message) => {
|
|
40
84
|
return message.type === 'token' && message.url === this.url;
|
|
@@ -52,6 +96,12 @@ export class TelemetryStreamAbstract {
|
|
|
52
96
|
this.registerReceiver();
|
|
53
97
|
}
|
|
54
98
|
stopStream() {
|
|
99
|
+
const queueState = telemetryActionQueueStates.get(this);
|
|
100
|
+
if (queueState) {
|
|
101
|
+
queueState.stopped = true;
|
|
102
|
+
resolveUnsentTelemetryActions(queueState);
|
|
103
|
+
}
|
|
104
|
+
this.subscriberId = undefined;
|
|
55
105
|
this.heartbeat?.stop();
|
|
56
106
|
this.stopInternal();
|
|
57
107
|
if (this.messagesTimer) {
|
|
@@ -84,7 +134,9 @@ export class TelemetryStreamAbstract {
|
|
|
84
134
|
const toAdd = arrayRemove(this.sources, sourcesBefore);
|
|
85
135
|
if (toAdd.length) {
|
|
86
136
|
this.logger.debug('New query subscriptions added, we need to refresh sources', { toAdd });
|
|
87
|
-
void this.enqueueTelemetryAction('subscribe', this.callsignsLookup, toAdd)
|
|
137
|
+
void this.enqueueTelemetryAction('subscribe', this.callsignsLookup, toAdd).catch((error) => {
|
|
138
|
+
this.onBackgroundActionFailure('subscribe', error);
|
|
139
|
+
});
|
|
88
140
|
}
|
|
89
141
|
}
|
|
90
142
|
else {
|
|
@@ -95,7 +147,9 @@ export class TelemetryStreamAbstract {
|
|
|
95
147
|
const { toAdd } = getSubscriptionsDifference(before, after);
|
|
96
148
|
if (toAdd.callsigns.length && toAdd.sources.length) {
|
|
97
149
|
this.logger.debug('New subscriptions added, we need to refresh sources', { toAdd });
|
|
98
|
-
void this.enqueueTelemetryAction('subscribe', new CallsignsLookup(toAdd.callsigns), toAdd.sources)
|
|
150
|
+
void this.enqueueTelemetryAction('subscribe', new CallsignsLookup(toAdd.callsigns), toAdd.sources).catch((error) => {
|
|
151
|
+
this.onBackgroundActionFailure('subscribe', error);
|
|
152
|
+
});
|
|
99
153
|
}
|
|
100
154
|
}
|
|
101
155
|
}
|
|
@@ -130,7 +184,7 @@ export class TelemetryStreamAbstract {
|
|
|
130
184
|
await this.enqueueTelemetryAction('unsubscribe', this.callsignsLookup, toRemove, terminateReceiverGroup);
|
|
131
185
|
}
|
|
132
186
|
catch (err) {
|
|
133
|
-
this.
|
|
187
|
+
this.onBackgroundActionFailure('unsubscribe', err);
|
|
134
188
|
}
|
|
135
189
|
}
|
|
136
190
|
return;
|
|
@@ -145,10 +199,24 @@ export class TelemetryStreamAbstract {
|
|
|
145
199
|
await this.enqueueTelemetryAction('unsubscribe', new CallsignsLookup(toRemove.callsigns), toRemove.sources, terminateReceiverGroup);
|
|
146
200
|
}
|
|
147
201
|
catch (err) {
|
|
148
|
-
this.
|
|
202
|
+
this.onBackgroundActionFailure('unsubscribe', err);
|
|
149
203
|
}
|
|
150
204
|
}
|
|
151
205
|
}
|
|
206
|
+
onBackgroundActionFailure(actionType, error) {
|
|
207
|
+
this.logger.error(`Failed to ${actionType}: ${error}`);
|
|
208
|
+
if (isReceiverGoneError(error)) {
|
|
209
|
+
this.markReceiverDead(error);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
markReceiverDead(error) {
|
|
213
|
+
if (this.receiverDead)
|
|
214
|
+
return;
|
|
215
|
+
this.receiverDead = true;
|
|
216
|
+
this.logger.error('Receiver is no longer registered with the gateway, retiring the stream', error);
|
|
217
|
+
StreamRegister.getInstance().reportStreamDead(this.identifier, this);
|
|
218
|
+
this.messageStream$.error(error);
|
|
219
|
+
}
|
|
152
220
|
isQueryLookup() {
|
|
153
221
|
return this.callsignsLookup?.isQuery() ?? false;
|
|
154
222
|
}
|
|
@@ -201,14 +269,22 @@ export class TelemetryStreamAbstract {
|
|
|
201
269
|
// Subscribe message
|
|
202
270
|
case GRPC_SOURCE_SUBSCRIBED: {
|
|
203
271
|
const { subscriberId } = rocosMessage.payload;
|
|
272
|
+
const queueState = telemetryActionQueueStates.get(this);
|
|
273
|
+
if (queueState?.stopped)
|
|
274
|
+
return;
|
|
204
275
|
if (this.subscriberId && subscriberId) {
|
|
205
276
|
this.logger.warn(`Replacing old subscriberId (${this.subscriberId}) with a new one (${subscriberId})`);
|
|
206
277
|
}
|
|
207
|
-
|
|
278
|
+
if (queueState) {
|
|
279
|
+
queueState.registration++;
|
|
280
|
+
resolveUnsentTelemetryActions(queueState);
|
|
281
|
+
}
|
|
208
282
|
this.subscriberId = subscriberId;
|
|
209
283
|
rocosMessage.subscriberId = this.subscriberId;
|
|
210
284
|
this.logger.info('onData', `subscriberId has been updated - ${subscriberId}`, { subscriberId });
|
|
211
|
-
void this.enqueueTelemetryAction('subscribe')
|
|
285
|
+
void this.enqueueTelemetryAction('subscribe').catch((error) => {
|
|
286
|
+
this.onBackgroundActionFailure('subscribe', error);
|
|
287
|
+
});
|
|
212
288
|
this.heartbeat.start(() => this.sendHeartbeat(), this.sendHeartbeatTime);
|
|
213
289
|
isRobotMessage = false;
|
|
214
290
|
break;
|
|
@@ -250,21 +326,69 @@ export class TelemetryStreamAbstract {
|
|
|
250
326
|
* unsubscribe landing last strands a subscribe and nothing re-adds it.
|
|
251
327
|
*/
|
|
252
328
|
enqueueTelemetryAction(actionType, callsignsLookup, sources, terminate) {
|
|
253
|
-
const
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
329
|
+
const queueState = telemetryActionQueueStates.get(this);
|
|
330
|
+
if (!queueState || queueState.stopped)
|
|
331
|
+
return Promise.resolve();
|
|
332
|
+
const completion = new Promise((resolve, reject) => {
|
|
333
|
+
queueState.pendingActions.push({
|
|
334
|
+
actionType,
|
|
335
|
+
callsignsLookup: callsignsLookup ?? this.callsignsLookup,
|
|
336
|
+
sources: [...(sources ?? this.sources)],
|
|
337
|
+
terminate: terminate ?? false,
|
|
338
|
+
subscriberId: this.subscriberId,
|
|
339
|
+
registration: queueState.registration,
|
|
340
|
+
resolve,
|
|
341
|
+
reject,
|
|
342
|
+
});
|
|
343
|
+
});
|
|
344
|
+
const scheduleDrain = () => {
|
|
345
|
+
if (queueState.stopped ||
|
|
346
|
+
queueState.requestInFlight ||
|
|
347
|
+
queueState.drainScheduled ||
|
|
348
|
+
queueState.pendingActions.length === 0) {
|
|
349
|
+
return;
|
|
350
|
+
}
|
|
351
|
+
queueState.drainScheduled = true;
|
|
352
|
+
this.telemetryActionQueue = this.telemetryActionQueue.then(async () => {
|
|
353
|
+
queueState.drainScheduled = false;
|
|
354
|
+
if (queueState.stopped || queueState.requestInFlight)
|
|
355
|
+
return;
|
|
356
|
+
const activeBatch = takeNextTelemetryBatch(queueState);
|
|
357
|
+
const [firstAction] = activeBatch;
|
|
358
|
+
if (!firstAction)
|
|
359
|
+
return;
|
|
360
|
+
queueState.activeBatch = activeBatch;
|
|
361
|
+
queueState.requestInFlight = true;
|
|
362
|
+
try {
|
|
363
|
+
await this.takeTelemetryAction(firstAction.actionType, firstAction.callsignsLookup, firstAction.sources);
|
|
364
|
+
resolveTelemetryActions(activeBatch);
|
|
365
|
+
}
|
|
366
|
+
catch (error) {
|
|
367
|
+
activeBatch.forEach((action) => action.reject(error));
|
|
368
|
+
}
|
|
369
|
+
finally {
|
|
370
|
+
queueState.activeBatch = [];
|
|
371
|
+
queueState.requestInFlight = false;
|
|
372
|
+
scheduleDrain();
|
|
373
|
+
}
|
|
374
|
+
});
|
|
375
|
+
};
|
|
376
|
+
scheduleDrain();
|
|
377
|
+
return completion;
|
|
259
378
|
}
|
|
260
|
-
async takeTelemetryAction(actionType, callsignsLookup, sources
|
|
379
|
+
async takeTelemetryAction(actionType, callsignsLookup, sources) {
|
|
261
380
|
this.logger.debug('takeTelemetryAction', actionType);
|
|
381
|
+
const queueState = telemetryActionQueueStates.get(this);
|
|
382
|
+
const activeBatch = queueState?.activeBatch ?? [];
|
|
383
|
+
const [firstAction] = activeBatch;
|
|
262
384
|
// A config fault, not a race: this stream can never address a request.
|
|
263
385
|
if (!this.projectId) {
|
|
264
386
|
this.logger.warn('takeTelemetryAction', actionType, `${actionType} without a projectId - dropped`);
|
|
265
387
|
return Promise.resolve();
|
|
266
388
|
}
|
|
267
|
-
if (!
|
|
389
|
+
if (!firstAction?.subscriberId ||
|
|
390
|
+
firstAction.subscriberId !== this.subscriberId ||
|
|
391
|
+
firstAction.registration !== queueState?.registration) {
|
|
268
392
|
// Deferred, not lost: `onData` re-asserts `this.sources` when the subscriberId arrives.
|
|
269
393
|
this.logger.debug('takeTelemetryAction', actionType, `${actionType} before the receiver was registered - deferred to the post-registration re-assert`);
|
|
270
394
|
return Promise.resolve();
|
|
@@ -272,20 +396,20 @@ export class TelemetryStreamAbstract {
|
|
|
272
396
|
const actionSources = sources ?? this.sources;
|
|
273
397
|
const actionCallsigns = callsignsLookup ?? this.callsignsLookup;
|
|
274
398
|
if (!actionCallsigns.isQuery()) {
|
|
275
|
-
// Compose the request message
|
|
276
399
|
const req = TelemetryRequest.create({
|
|
277
|
-
subscriberId:
|
|
278
|
-
requestedActions:
|
|
279
|
-
{
|
|
280
|
-
operation: actionType,
|
|
281
|
-
callsigns:
|
|
282
|
-
sources:
|
|
283
|
-
}
|
|
284
|
-
|
|
400
|
+
subscriberId: firstAction.subscriberId,
|
|
401
|
+
requestedActions: activeBatch.map((action) => {
|
|
402
|
+
const telemetryAction = TelemetryAction.create({
|
|
403
|
+
operation: action.actionType,
|
|
404
|
+
callsigns: action.callsignsLookup.getCallsigns(),
|
|
405
|
+
sources: action.sources,
|
|
406
|
+
});
|
|
407
|
+
if (action.actionType === 'unsubscribe' && action.terminate) {
|
|
408
|
+
this.addTerminateToAction(telemetryAction);
|
|
409
|
+
}
|
|
410
|
+
return telemetryAction;
|
|
411
|
+
}),
|
|
285
412
|
});
|
|
286
|
-
if (actionType === 'unsubscribe' && terminate) {
|
|
287
|
-
this.addTerminateToAction(req.requestedActions[0]);
|
|
288
|
-
}
|
|
289
413
|
this.logger.debug('takeTelemetryAction', actionType, 'client.requestTelemetry', this.subscriberId);
|
|
290
414
|
// Send the message to back-end.
|
|
291
415
|
return this.requestTelemetry(req);
|
|
@@ -310,7 +434,7 @@ export class TelemetryStreamAbstract {
|
|
|
310
434
|
});
|
|
311
435
|
// Compose the request message
|
|
312
436
|
const req = TelemetryQueryRequest.create({
|
|
313
|
-
subscriberId:
|
|
437
|
+
subscriberId: firstAction.subscriberId,
|
|
314
438
|
sources: actionSources,
|
|
315
439
|
operation: actionType,
|
|
316
440
|
callsignQuery: {
|
|
@@ -410,14 +534,22 @@ export class TelemetryStreamAbstract {
|
|
|
410
534
|
unsubscribeOperation,
|
|
411
535
|
};
|
|
412
536
|
}
|
|
537
|
+
renewReceiver() {
|
|
538
|
+
try {
|
|
539
|
+
// Remove the received call anyway.
|
|
540
|
+
this.stopInternal();
|
|
541
|
+
this.lastMessageReceived = undefined;
|
|
542
|
+
this.lastRobotMessageReceived = undefined;
|
|
543
|
+
this.registerReceiver();
|
|
544
|
+
}
|
|
545
|
+
catch (e) {
|
|
546
|
+
this.logger.error('Failed to renew the receiver, retrying on the next check', e);
|
|
547
|
+
}
|
|
548
|
+
}
|
|
413
549
|
messageTimeoutTryAgain() {
|
|
414
550
|
// Increase retry times.
|
|
415
551
|
this.retryTimes++;
|
|
416
|
-
|
|
417
|
-
this.stopInternal();
|
|
418
|
-
this.lastMessageReceived = undefined;
|
|
419
|
-
this.lastRobotMessageReceived = undefined;
|
|
420
|
-
this.registerReceiver();
|
|
552
|
+
this.renewReceiver();
|
|
421
553
|
}
|
|
422
554
|
/**
|
|
423
555
|
* Auto resubscribe to reduce CPU usage.
|
|
@@ -425,11 +557,7 @@ export class TelemetryStreamAbstract {
|
|
|
425
557
|
autoResubscribe() {
|
|
426
558
|
// Reset
|
|
427
559
|
this.retryTimes = 0;
|
|
428
|
-
|
|
429
|
-
this.stopInternal();
|
|
430
|
-
this.lastMessageReceived = undefined;
|
|
431
|
-
this.lastRobotMessageReceived = undefined;
|
|
432
|
-
this.registerReceiver();
|
|
560
|
+
this.renewReceiver();
|
|
433
561
|
}
|
|
434
562
|
isFoundSource(source) {
|
|
435
563
|
return this.sources.indexOf(source) !== -1;
|
|
@@ -37,6 +37,7 @@ export class TelemetryStreamConnect extends TelemetryStreamAbstract {
|
|
|
37
37
|
this.client = createClient(TelemetryGateway, transport);
|
|
38
38
|
}
|
|
39
39
|
stopInternal() {
|
|
40
|
+
this.subscriberId = undefined;
|
|
40
41
|
if (this.abortController) {
|
|
41
42
|
this.logger.info('Starting stop stream process');
|
|
42
43
|
this.abortController.abort();
|
|
@@ -12,6 +12,7 @@ export class TelemetryStreamNode extends TelemetryStreamAbstract {
|
|
|
12
12
|
this.credentials = config.insecure ? grpc.credentials.createInsecure() : grpc.credentials.createSsl();
|
|
13
13
|
}
|
|
14
14
|
stopInternal() {
|
|
15
|
+
this.subscriberId = undefined;
|
|
15
16
|
try {
|
|
16
17
|
this.logger.info('Starting stop stream process');
|
|
17
18
|
this.logger.info('Stopping stream');
|
package/esm/models/maps/Map.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { SubscriberStatusEnum, } from '../models';
|
|
2
1
|
import { Subject } from 'rxjs';
|
|
3
2
|
import { BaseStreamService } from './BaseStreamService';
|
|
4
3
|
import { IDENTIFIER_NAME_SEARCH } from '../constants/identifier';
|
|
@@ -44,12 +43,8 @@ export class SearchService extends BaseStreamService {
|
|
|
44
43
|
port: this.config.port,
|
|
45
44
|
insecure: this.config.insecure,
|
|
46
45
|
});
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
this.status = msg === SubscriberStatusEnum.STOPPED || msg === SubscriberStatusEnum.ALIVE;
|
|
50
|
-
this.status$.next(msg);
|
|
51
|
-
});
|
|
52
|
-
}
|
|
46
|
+
// The status forwarder is wired once per stream by BaseStreamService.acquireStream's factory
|
|
47
|
+
// (and released now that the stream completes statusStream$ on stop), so no per-service one here.
|
|
53
48
|
return newStream.stream;
|
|
54
49
|
}
|
|
55
50
|
getStream(config) {
|
|
@@ -28,6 +28,10 @@ export declare class TelemetryService extends BaseStreamService<ITelemetryStream
|
|
|
28
28
|
getStatusChanges(): Observable<IStreamStatusMessage>;
|
|
29
29
|
/**
|
|
30
30
|
* Subscribes to the heartbeat telemetry and checks we receive one at least every 5s
|
|
31
|
+
*
|
|
32
|
+
* @deprecated Superseded by {@link RobotStatusService.getRobotStatusChanges}, which reports the
|
|
33
|
+
* normalised robot status (reported health/readiness plus platform connectivity) rather than only
|
|
34
|
+
* heartbeat-derived online/offline liveness.
|
|
31
35
|
* @param projectId
|
|
32
36
|
* @param callsign
|
|
33
37
|
* @param heartbeatTimeoutMs optionally override the deadline for heartbeats (in ms). Default 5000ms
|
|
@@ -85,8 +85,16 @@ export class TelemetryService extends BaseStreamService {
|
|
|
85
85
|
return from(messages$).pipe(mergeAll(), filter((message) => {
|
|
86
86
|
return callsignsLookup.includesCallsign(message.callsign) && sources.includes(message.source);
|
|
87
87
|
}), finalize(() => {
|
|
88
|
-
|
|
89
|
-
|
|
88
|
+
if (params.terminateReceiverGroupOnUnsubscribe) {
|
|
89
|
+
const removal = stream.removeSubscription(subscriptionParams, true);
|
|
90
|
+
void Promise.resolve(removal)
|
|
91
|
+
.finally(release)
|
|
92
|
+
.catch(() => undefined);
|
|
93
|
+
}
|
|
94
|
+
else {
|
|
95
|
+
release();
|
|
96
|
+
void Promise.resolve(stream.removeSubscription(subscriptionParams)).catch(() => undefined);
|
|
97
|
+
}
|
|
90
98
|
}));
|
|
91
99
|
});
|
|
92
100
|
}
|
|
@@ -108,6 +116,10 @@ export class TelemetryService extends BaseStreamService {
|
|
|
108
116
|
}
|
|
109
117
|
/**
|
|
110
118
|
* Subscribes to the heartbeat telemetry and checks we receive one at least every 5s
|
|
119
|
+
*
|
|
120
|
+
* @deprecated Superseded by {@link RobotStatusService.getRobotStatusChanges}, which reports the
|
|
121
|
+
* normalised robot status (reported health/readiness plus platform connectivity) rather than only
|
|
122
|
+
* heartbeat-derived online/offline liveness.
|
|
111
123
|
* @param projectId
|
|
112
124
|
* @param callsign
|
|
113
125
|
* @param heartbeatTimeoutMs optionally override the deadline for heartbeats (in ms). Default 5000ms
|