@dronedeploy/rocos-js-sdk 4.4.6 → 4.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/cjs/api/StreamRegister.d.ts +7 -2
  2. package/cjs/api/StreamRegister.js +21 -6
  3. package/cjs/api/streams/caller/CallerStreamAbstract.js +2 -1
  4. package/cjs/api/streams/command/CommandStreamAbstract.js +2 -1
  5. package/cjs/api/streams/control/ControlStreamAbstract.js +2 -1
  6. package/cjs/api/streams/fileAccessor/FileAccessorStreamAbstract.js +2 -1
  7. package/cjs/api/streams/search/SearchStreamAbstract.js +2 -1
  8. package/cjs/api/streams/telemetry/TelemetryStream.js +1 -0
  9. package/cjs/api/streams/telemetry/TelemetryStreamAbstract.d.ts +4 -0
  10. package/cjs/api/streams/telemetry/TelemetryStreamAbstract.js +165 -37
  11. package/cjs/api/streams/telemetry/TelemetryStreamConnect.js +1 -0
  12. package/cjs/api/streams/telemetry/TelemetryStreamNode.js +1 -0
  13. package/cjs/api/streams/webRTCSignalling/WebRTCSignallingStreamAbstract.js +2 -1
  14. package/cjs/models/maps/Map.d.ts +1 -1
  15. package/cjs/services/SearchService.js +2 -7
  16. package/cjs/services/TelemetryService.d.ts +4 -0
  17. package/cjs/services/TelemetryService.js +14 -2
  18. package/esm/api/StreamRegister.d.ts +7 -2
  19. package/esm/api/StreamRegister.js +21 -6
  20. package/esm/api/streams/caller/CallerStreamAbstract.js +2 -1
  21. package/esm/api/streams/command/CommandStreamAbstract.js +2 -1
  22. package/esm/api/streams/control/ControlStreamAbstract.js +2 -1
  23. package/esm/api/streams/fileAccessor/FileAccessorStreamAbstract.js +2 -1
  24. package/esm/api/streams/search/SearchStreamAbstract.js +2 -1
  25. package/esm/api/streams/telemetry/TelemetryStream.js +1 -0
  26. package/esm/api/streams/telemetry/TelemetryStreamAbstract.d.ts +4 -0
  27. package/esm/api/streams/telemetry/TelemetryStreamAbstract.js +166 -38
  28. package/esm/api/streams/telemetry/TelemetryStreamConnect.js +1 -0
  29. package/esm/api/streams/telemetry/TelemetryStreamNode.js +1 -0
  30. package/esm/api/streams/webRTCSignalling/WebRTCSignallingStreamAbstract.js +2 -1
  31. package/esm/models/maps/Map.d.ts +1 -1
  32. package/esm/services/SearchService.js +2 -7
  33. package/esm/services/TelemetryService.d.ts +4 -0
  34. package/esm/services/TelemetryService.js +14 -2
  35. package/package.json +1 -1
@@ -10,8 +10,12 @@ export interface IAcquiredStream<T extends IBaseStream> {
10
10
  /**
11
11
  * Sole owner of stream lifecycle. Streams are shared per identifier and refcounted: acquireStream
12
12
  * gets-or-creates and increments in one synchronous step, and the returned handle's release()
13
- * tears the stream down when the last holder leaves. Teardown never happens outside
14
- * release/removeAllStreams, so a held stream can not be stopped or replaced underneath its holders.
13
+ * tears the stream down when the last holder leaves.
14
+ *
15
+ * A held stream can also be stopped and replaced underneath its holders, by reportStreamDead, when
16
+ * the stream can no longer carry anything. Holders keep the instance they were handed; their
17
+ * release() finds the entry gone and does nothing, so a replacement never inherits a departing
18
+ * holder's refcount and a refcount can not go negative.
15
19
  */
16
20
  export declare class StreamRegister {
17
21
  private entries;
@@ -22,6 +26,7 @@ export declare class StreamRegister {
22
26
  static getIdentifier($identifier: string, scope?: string): string;
23
27
  acquireStream<T extends IBaseStream>(identifier: string, create: () => T, init: (stream: T) => Promise<void>): IAcquiredStream<T>;
24
28
  getStream(identifier: string): IBaseStream | undefined;
29
+ reportStreamDead(identifier: string, stream: IBaseStream): void;
25
30
  removeAllStreams(): boolean;
26
31
  private release;
27
32
  private evict;
@@ -6,8 +6,12 @@ const RocosLogger_1 = require("../logger/RocosLogger");
6
6
  /**
7
7
  * Sole owner of stream lifecycle. Streams are shared per identifier and refcounted: acquireStream
8
8
  * gets-or-creates and increments in one synchronous step, and the returned handle's release()
9
- * tears the stream down when the last holder leaves. Teardown never happens outside
10
- * release/removeAllStreams, so a held stream can not be stopped or replaced underneath its holders.
9
+ * tears the stream down when the last holder leaves.
10
+ *
11
+ * A held stream can also be stopped and replaced underneath its holders, by reportStreamDead, when
12
+ * the stream can no longer carry anything. Holders keep the instance they were handed; their
13
+ * release() finds the entry gone and does nothing, so a replacement never inherits a departing
14
+ * holder's refcount and a refcount can not go negative.
11
15
  */
12
16
  class StreamRegister {
13
17
  constructor() {
@@ -62,6 +66,17 @@ class StreamRegister {
62
66
  getStream(identifier) {
63
67
  return this.entries.get(identifier)?.stream;
64
68
  }
69
+ reportStreamDead(identifier, stream) {
70
+ if (this.entries.get(identifier)?.stream !== stream)
71
+ return; // already evicted, or already replaced
72
+ this.logger.warn(`Stream reported dead, evicting so the next acquirer builds a new one. identifier: ${identifier}`);
73
+ try {
74
+ this.evict(identifier, stream);
75
+ }
76
+ catch (e) {
77
+ this.logger.error(`Failed to evict dead stream. identifier: ${identifier}`, e);
78
+ }
79
+ }
65
80
  removeAllStreams() {
66
81
  for (const [, entry] of this.entries.entries()) {
67
82
  try {
@@ -82,13 +97,13 @@ class StreamRegister {
82
97
  return;
83
98
  // Wait for init to settle before tearing down, so init can never open resources on an already
84
99
  // stopped stream. A re-acquire in the meantime revives the entry and cancels the teardown.
85
- void entry.ready
86
- .catch(() => undefined)
87
- .then(() => {
100
+ const evictIfUnheld = () => {
88
101
  if (this.entries.get(identifier) === entry && entry.refCount <= 0) {
89
102
  this.evict(identifier, entry.stream);
90
103
  }
91
- })
104
+ };
105
+ void entry.ready
106
+ .then(() => evictIfUnheld(), () => evictIfUnheld())
92
107
  .catch((e) => this.logger.error(`Failed to evict stream. identifier: ${identifier}`, e));
93
108
  }
94
109
  evict(identifier, stream) {
@@ -29,7 +29,8 @@ class CallerStreamAbstract {
29
29
  this.token = token;
30
30
  }
31
31
  stopStream() {
32
- // do nothing for now
32
+ this.statusStream$.next(models_1.SubscriberStatusEnum.STOPPED);
33
+ this.statusStream$.complete();
33
34
  }
34
35
  invokeRequest(params) {
35
36
  this.logger.info('invokeRequest', params);
@@ -31,7 +31,8 @@ class CommandStreamAbstract {
31
31
  this.token = token;
32
32
  }
33
33
  stopStream() {
34
- // do nothing for now
34
+ this.statusStream$.next(models_1.SubscriberStatusEnum.STOPPED);
35
+ this.statusStream$.complete();
35
36
  }
36
37
  invokeRequest(params) {
37
38
  if (params.timeoutMs === undefined || params.timeoutMs === null) {
@@ -28,7 +28,8 @@ class ControlStreamAbstract {
28
28
  this.token = token;
29
29
  }
30
30
  stopStream() {
31
- // do nothing for now
31
+ this.statusStream$.next(models_1.SubscriberStatusEnum.STOPPED);
32
+ this.statusStream$.complete();
32
33
  }
33
34
  takeActionWithAck(action, params, ackTimeoutInSeconds = 15) {
34
35
  return new rxjs_1.Observable((emitter) => {
@@ -27,7 +27,8 @@ class FileAccessorStreamAbstract {
27
27
  this.token = token;
28
28
  }
29
29
  stopStream() {
30
- // do nothing for now
30
+ this.statusStream$.next(models_1.SubscriberStatusEnum.STOPPED);
31
+ this.statusStream$.complete();
31
32
  }
32
33
  async serverReqOp(params) {
33
34
  return this.serverReqOpInternal(params);
@@ -28,7 +28,8 @@ class SearchStreamAbstract {
28
28
  this.token = token;
29
29
  }
30
30
  stopStream() {
31
- // do nothing for now
31
+ this.statusStream$.next(models_1.SubscriberStatusEnum.STOPPED);
32
+ this.statusStream$.complete();
32
33
  }
33
34
  searchStream(params) {
34
35
  const request = this.convertToQueryMessage(params.query);
@@ -24,6 +24,7 @@ class TelemetryStream extends TelemetryStreamAbstract_1.TelemetryStreamAbstract
24
24
  this.client = new teletubby_pb_client_1.TelemetryGatewayClient(transport);
25
25
  }
26
26
  stopInternal() {
27
+ this.subscriberId = undefined;
27
28
  if (this.abortController) {
28
29
  this.logger.info('Starting stop stream process');
29
30
  this.abortController.abort();
@@ -29,6 +29,7 @@ export declare abstract class TelemetryStreamAbstract implements ITelemetryStrea
29
29
  private sendHeartbeatTime;
30
30
  private receivedDataStats;
31
31
  private subscriptions;
32
+ private receiverDead;
32
33
  constructor(config: ITelemetryStreamConfig);
33
34
  protected abstract stopInternal(): void;
34
35
  protected abstract registerReceiver(): () => void;
@@ -41,6 +42,8 @@ export declare abstract class TelemetryStreamAbstract implements ITelemetryStrea
41
42
  stopStream(): void;
42
43
  addSubscription(params: ITelemetryParams): void;
43
44
  removeSubscription(params: ITelemetryParams, terminateReceiverGroup?: boolean): Promise<void>;
45
+ private onBackgroundActionFailure;
46
+ private markReceiverDead;
44
47
  private isQueryLookup;
45
48
  private getSubscribedSources;
46
49
  sendAcknowledgment(uid: string, status: TelemetryAckStatus, noRetry: boolean): boolean;
@@ -55,6 +58,7 @@ export declare abstract class TelemetryStreamAbstract implements ITelemetryStrea
55
58
  private takeTelemetryAction;
56
59
  protected listenMessagesAndRenew(): void;
57
60
  private addTerminateToAction;
61
+ private renewReceiver;
58
62
  private messageTimeoutTryAgain;
59
63
  /**
60
64
  * Auto resubscribe to reduce CPU usage.
@@ -13,6 +13,41 @@ const arrayRemove_1 = require("../../../helpers/arrayRemove");
13
13
  const arrayUnique_1 = require("../../../helpers/arrayUnique");
14
14
  const operators_1 = require("rxjs/operators");
15
15
  const getSubscriptionsDifference_1 = require("../../../helpers/getSubscriptionsDifference");
16
+ const telemetryActionQueueStates = new WeakMap();
17
+ const resolveTelemetryActions = (actions) => {
18
+ actions.forEach((action) => action.resolve());
19
+ };
20
+ const resolveUnsentTelemetryActions = (queueState) => {
21
+ resolveTelemetryActions(queueState.pendingActions.splice(0));
22
+ };
23
+ const isReceiverGoneError = (error) => {
24
+ const code = error?.code;
25
+ if (typeof code === 'number')
26
+ return code === 9; // gRPC FAILED_PRECONDITION
27
+ if (typeof code === 'string')
28
+ return code.toLowerCase() === 'failed_precondition';
29
+ return false;
30
+ };
31
+ const takeNextTelemetryBatch = (queueState) => {
32
+ const firstAction = queueState.pendingActions.shift();
33
+ if (!firstAction)
34
+ return [];
35
+ const activeBatch = [firstAction];
36
+ if (firstAction.callsignsLookup.isQuery() || firstAction.terminate)
37
+ return activeBatch;
38
+ while (queueState.pendingActions.length > 0) {
39
+ const [nextAction] = queueState.pendingActions;
40
+ if (nextAction.callsignsLookup.isQuery() ||
41
+ nextAction.terminate ||
42
+ nextAction.actionType !== firstAction.actionType ||
43
+ nextAction.subscriberId !== firstAction.subscriberId ||
44
+ nextAction.registration !== firstAction.registration) {
45
+ break;
46
+ }
47
+ activeBatch.push(queueState.pendingActions.shift());
48
+ }
49
+ return activeBatch;
50
+ };
16
51
  class TelemetryStreamAbstract {
17
52
  constructor(config) {
18
53
  this.telemetryActionQueue = Promise.resolve();
@@ -26,6 +61,7 @@ class TelemetryStreamAbstract {
26
61
  this.sendHeartbeatTime = 5000;
27
62
  this.receivedDataStats = {};
28
63
  this.subscriptions = new Map();
64
+ this.receiverDead = false;
29
65
  this.identifier = StreamRegister_1.StreamRegister.getIdentifier(identifier_1.IDENTIFIER_NAME_TELEMETRY, config.scope);
30
66
  this.projectId = config.projectId;
31
67
  this.callsignsLookup = config.callsignsLookup;
@@ -38,6 +74,14 @@ class TelemetryStreamAbstract {
38
74
  this.messageStream$ = new rxjs_1.Subject();
39
75
  this.statusStream$ = new rxjs_1.BehaviorSubject(models_1.SubscriberStatusEnum.STOPPED);
40
76
  this.heartbeat = new StreamHeartbeat_1.StreamHeartbeat(this.identifier);
77
+ telemetryActionQueueStates.set(this, {
78
+ pendingActions: [],
79
+ activeBatch: [],
80
+ registration: 0,
81
+ drainScheduled: false,
82
+ requestInFlight: false,
83
+ stopped: false,
84
+ });
41
85
  RocosStore_1.RocosStore.getChangeSubject()
42
86
  .pipe((0, operators_1.filter)((message) => {
43
87
  return message.type === 'token' && message.url === this.url;
@@ -55,6 +99,12 @@ class TelemetryStreamAbstract {
55
99
  this.registerReceiver();
56
100
  }
57
101
  stopStream() {
102
+ const queueState = telemetryActionQueueStates.get(this);
103
+ if (queueState) {
104
+ queueState.stopped = true;
105
+ resolveUnsentTelemetryActions(queueState);
106
+ }
107
+ this.subscriberId = undefined;
58
108
  this.heartbeat?.stop();
59
109
  this.stopInternal();
60
110
  if (this.messagesTimer) {
@@ -87,7 +137,9 @@ class TelemetryStreamAbstract {
87
137
  const toAdd = (0, arrayRemove_1.arrayRemove)(this.sources, sourcesBefore);
88
138
  if (toAdd.length) {
89
139
  this.logger.debug('New query subscriptions added, we need to refresh sources', { toAdd });
90
- void this.enqueueTelemetryAction('subscribe', this.callsignsLookup, toAdd);
140
+ void this.enqueueTelemetryAction('subscribe', this.callsignsLookup, toAdd).catch((error) => {
141
+ this.onBackgroundActionFailure('subscribe', error);
142
+ });
91
143
  }
92
144
  }
93
145
  else {
@@ -98,7 +150,9 @@ class TelemetryStreamAbstract {
98
150
  const { toAdd } = (0, getSubscriptionsDifference_1.getSubscriptionsDifference)(before, after);
99
151
  if (toAdd.callsigns.length && toAdd.sources.length) {
100
152
  this.logger.debug('New subscriptions added, we need to refresh sources', { toAdd });
101
- void this.enqueueTelemetryAction('subscribe', new models_1.CallsignsLookup(toAdd.callsigns), toAdd.sources);
153
+ void this.enqueueTelemetryAction('subscribe', new models_1.CallsignsLookup(toAdd.callsigns), toAdd.sources).catch((error) => {
154
+ this.onBackgroundActionFailure('subscribe', error);
155
+ });
102
156
  }
103
157
  }
104
158
  }
@@ -133,7 +187,7 @@ class TelemetryStreamAbstract {
133
187
  await this.enqueueTelemetryAction('unsubscribe', this.callsignsLookup, toRemove, terminateReceiverGroup);
134
188
  }
135
189
  catch (err) {
136
- this.logger.error(`Failed to unsubscribe: ${err}`);
190
+ this.onBackgroundActionFailure('unsubscribe', err);
137
191
  }
138
192
  }
139
193
  return;
@@ -148,10 +202,24 @@ class TelemetryStreamAbstract {
148
202
  await this.enqueueTelemetryAction('unsubscribe', new models_1.CallsignsLookup(toRemove.callsigns), toRemove.sources, terminateReceiverGroup);
149
203
  }
150
204
  catch (err) {
151
- this.logger.error(`Failed to unsubscribe: ${err}`);
205
+ this.onBackgroundActionFailure('unsubscribe', err);
152
206
  }
153
207
  }
154
208
  }
209
+ onBackgroundActionFailure(actionType, error) {
210
+ this.logger.error(`Failed to ${actionType}: ${error}`);
211
+ if (isReceiverGoneError(error)) {
212
+ this.markReceiverDead(error);
213
+ }
214
+ }
215
+ markReceiverDead(error) {
216
+ if (this.receiverDead)
217
+ return;
218
+ this.receiverDead = true;
219
+ this.logger.error('Receiver is no longer registered with the gateway, retiring the stream', error);
220
+ StreamRegister_1.StreamRegister.getInstance().reportStreamDead(this.identifier, this);
221
+ this.messageStream$.error(error);
222
+ }
155
223
  isQueryLookup() {
156
224
  return this.callsignsLookup?.isQuery() ?? false;
157
225
  }
@@ -204,14 +272,22 @@ class TelemetryStreamAbstract {
204
272
  // Subscribe message
205
273
  case grpc_1.GRPC_SOURCE_SUBSCRIBED: {
206
274
  const { subscriberId } = rocosMessage.payload;
275
+ const queueState = telemetryActionQueueStates.get(this);
276
+ if (queueState?.stopped)
277
+ return;
207
278
  if (this.subscriberId && subscriberId) {
208
279
  this.logger.warn(`Replacing old subscriberId (${this.subscriberId}) with a new one (${subscriberId})`);
209
280
  }
210
- // Update the subscriberId
281
+ if (queueState) {
282
+ queueState.registration++;
283
+ resolveUnsentTelemetryActions(queueState);
284
+ }
211
285
  this.subscriberId = subscriberId;
212
286
  rocosMessage.subscriberId = this.subscriberId;
213
287
  this.logger.info('onData', `subscriberId has been updated - ${subscriberId}`, { subscriberId });
214
- void this.enqueueTelemetryAction('subscribe');
288
+ void this.enqueueTelemetryAction('subscribe').catch((error) => {
289
+ this.onBackgroundActionFailure('subscribe', error);
290
+ });
215
291
  this.heartbeat.start(() => this.sendHeartbeat(), this.sendHeartbeatTime);
216
292
  isRobotMessage = false;
217
293
  break;
@@ -253,21 +329,69 @@ class TelemetryStreamAbstract {
253
329
  * unsubscribe landing last strands a subscribe and nothing re-adds it.
254
330
  */
255
331
  enqueueTelemetryAction(actionType, callsignsLookup, sources, terminate) {
256
- const takeAction = () => this.takeTelemetryAction(actionType, callsignsLookup, sources, terminate);
257
- const run = this.telemetryActionQueue.then(takeAction);
258
- // The tail only signals whose turn it is, so it must never reject, otherwise one failed op
259
- // would wedge every op queued behind it. Callers still see the failure through `run`.
260
- this.telemetryActionQueue = run.then(() => undefined, () => undefined);
261
- return run;
332
+ const queueState = telemetryActionQueueStates.get(this);
333
+ if (!queueState || queueState.stopped)
334
+ return Promise.resolve();
335
+ const completion = new Promise((resolve, reject) => {
336
+ queueState.pendingActions.push({
337
+ actionType,
338
+ callsignsLookup: callsignsLookup ?? this.callsignsLookup,
339
+ sources: [...(sources ?? this.sources)],
340
+ terminate: terminate ?? false,
341
+ subscriberId: this.subscriberId,
342
+ registration: queueState.registration,
343
+ resolve,
344
+ reject,
345
+ });
346
+ });
347
+ const scheduleDrain = () => {
348
+ if (queueState.stopped ||
349
+ queueState.requestInFlight ||
350
+ queueState.drainScheduled ||
351
+ queueState.pendingActions.length === 0) {
352
+ return;
353
+ }
354
+ queueState.drainScheduled = true;
355
+ this.telemetryActionQueue = this.telemetryActionQueue.then(async () => {
356
+ queueState.drainScheduled = false;
357
+ if (queueState.stopped || queueState.requestInFlight)
358
+ return;
359
+ const activeBatch = takeNextTelemetryBatch(queueState);
360
+ const [firstAction] = activeBatch;
361
+ if (!firstAction)
362
+ return;
363
+ queueState.activeBatch = activeBatch;
364
+ queueState.requestInFlight = true;
365
+ try {
366
+ await this.takeTelemetryAction(firstAction.actionType, firstAction.callsignsLookup, firstAction.sources);
367
+ resolveTelemetryActions(activeBatch);
368
+ }
369
+ catch (error) {
370
+ activeBatch.forEach((action) => action.reject(error));
371
+ }
372
+ finally {
373
+ queueState.activeBatch = [];
374
+ queueState.requestInFlight = false;
375
+ scheduleDrain();
376
+ }
377
+ });
378
+ };
379
+ scheduleDrain();
380
+ return completion;
262
381
  }
263
- async takeTelemetryAction(actionType, callsignsLookup, sources, terminate) {
382
+ async takeTelemetryAction(actionType, callsignsLookup, sources) {
264
383
  this.logger.debug('takeTelemetryAction', actionType);
384
+ const queueState = telemetryActionQueueStates.get(this);
385
+ const activeBatch = queueState?.activeBatch ?? [];
386
+ const [firstAction] = activeBatch;
265
387
  // A config fault, not a race: this stream can never address a request.
266
388
  if (!this.projectId) {
267
389
  this.logger.warn('takeTelemetryAction', actionType, `${actionType} without a projectId - dropped`);
268
390
  return Promise.resolve();
269
391
  }
270
- if (!this.subscriberId) {
392
+ if (!firstAction?.subscriberId ||
393
+ firstAction.subscriberId !== this.subscriberId ||
394
+ firstAction.registration !== queueState?.registration) {
271
395
  // Deferred, not lost: `onData` re-asserts `this.sources` when the subscriberId arrives.
272
396
  this.logger.debug('takeTelemetryAction', actionType, `${actionType} before the receiver was registered - deferred to the post-registration re-assert`);
273
397
  return Promise.resolve();
@@ -275,20 +399,20 @@ class TelemetryStreamAbstract {
275
399
  const actionSources = sources ?? this.sources;
276
400
  const actionCallsigns = callsignsLookup ?? this.callsignsLookup;
277
401
  if (!actionCallsigns.isQuery()) {
278
- // Compose the request message
279
402
  const req = teletubby_pb_1.TelemetryRequest.create({
280
- subscriberId: this.subscriberId,
281
- requestedActions: [
282
- {
283
- operation: actionType,
284
- callsigns: actionCallsigns.getCallsigns(),
285
- sources: actionSources,
286
- },
287
- ],
403
+ subscriberId: firstAction.subscriberId,
404
+ requestedActions: activeBatch.map((action) => {
405
+ const telemetryAction = teletubby_pb_1.TelemetryAction.create({
406
+ operation: action.actionType,
407
+ callsigns: action.callsignsLookup.getCallsigns(),
408
+ sources: action.sources,
409
+ });
410
+ if (action.actionType === 'unsubscribe' && action.terminate) {
411
+ this.addTerminateToAction(telemetryAction);
412
+ }
413
+ return telemetryAction;
414
+ }),
288
415
  });
289
- if (actionType === 'unsubscribe' && terminate) {
290
- this.addTerminateToAction(req.requestedActions[0]);
291
- }
292
416
  this.logger.debug('takeTelemetryAction', actionType, 'client.requestTelemetry', this.subscriberId);
293
417
  // Send the message to back-end.
294
418
  return this.requestTelemetry(req);
@@ -313,7 +437,7 @@ class TelemetryStreamAbstract {
313
437
  });
314
438
  // Compose the request message
315
439
  const req = teletubby_pb_1.TelemetryQueryRequest.create({
316
- subscriberId: this.subscriberId,
440
+ subscriberId: firstAction.subscriberId,
317
441
  sources: actionSources,
318
442
  operation: actionType,
319
443
  callsignQuery: {
@@ -413,14 +537,22 @@ class TelemetryStreamAbstract {
413
537
  unsubscribeOperation,
414
538
  };
415
539
  }
540
+ renewReceiver() {
541
+ try {
542
+ // Remove the received call anyway.
543
+ this.stopInternal();
544
+ this.lastMessageReceived = undefined;
545
+ this.lastRobotMessageReceived = undefined;
546
+ this.registerReceiver();
547
+ }
548
+ catch (e) {
549
+ this.logger.error('Failed to renew the receiver, retrying on the next check', e);
550
+ }
551
+ }
416
552
  messageTimeoutTryAgain() {
417
553
  // Increase retry times.
418
554
  this.retryTimes++;
419
- // Remove the received call anyway.
420
- this.stopInternal();
421
- this.lastMessageReceived = undefined;
422
- this.lastRobotMessageReceived = undefined;
423
- this.registerReceiver();
555
+ this.renewReceiver();
424
556
  }
425
557
  /**
426
558
  * Auto resubscribe to reduce CPU usage.
@@ -428,11 +560,7 @@ class TelemetryStreamAbstract {
428
560
  autoResubscribe() {
429
561
  // Reset
430
562
  this.retryTimes = 0;
431
- // Remove the received call anyway.
432
- this.stopInternal();
433
- this.lastMessageReceived = undefined;
434
- this.lastRobotMessageReceived = undefined;
435
- this.registerReceiver();
563
+ this.renewReceiver();
436
564
  }
437
565
  isFoundSource(source) {
438
566
  return this.sources.indexOf(source) !== -1;
@@ -40,6 +40,7 @@ class TelemetryStreamConnect extends TelemetryStreamAbstract_1.TelemetryStreamAb
40
40
  this.client = (0, connect_1.createClient)(teletubby_pb_1.TelemetryGateway, transport);
41
41
  }
42
42
  stopInternal() {
43
+ this.subscriberId = undefined;
43
44
  if (this.abortController) {
44
45
  this.logger.info('Starting stop stream process');
45
46
  this.abortController.abort();
@@ -48,6 +48,7 @@ class TelemetryStreamNode extends TelemetryStreamAbstract_1.TelemetryStreamAbstr
48
48
  this.credentials = config.insecure ? grpc.credentials.createInsecure() : grpc.credentials.createSsl();
49
49
  }
50
50
  stopInternal() {
51
+ this.subscriberId = undefined;
51
52
  try {
52
53
  this.logger.info('Starting stop stream process');
53
54
  this.logger.info('Stopping stream');
@@ -28,7 +28,8 @@ class WebRTCSignallingStreamAbstract {
28
28
  this.token = token;
29
29
  }
30
30
  stopStream() {
31
- // do nothing for now
31
+ this.statusStream$.next(models_1.SubscriberStatusEnum.STOPPED);
32
+ this.statusStream$.complete();
32
33
  }
33
34
  /**
34
35
  * Connect operator
@@ -8,6 +8,6 @@ export interface Map {
8
8
  deployedCount: number;
9
9
  isUploading: boolean;
10
10
  isInCloud: boolean;
11
- hash: string;
11
+ hash?: string;
12
12
  lastModifiedTimestamp: number;
13
13
  }
@@ -1,7 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.SearchService = void 0;
4
- const models_1 = require("../models");
5
4
  const rxjs_1 = require("rxjs");
6
5
  const BaseStreamService_1 = require("./BaseStreamService");
7
6
  const identifier_1 = require("../constants/identifier");
@@ -47,12 +46,8 @@ class SearchService extends BaseStreamService_1.BaseStreamService {
47
46
  port: this.config.port,
48
47
  insecure: this.config.insecure,
49
48
  });
50
- if (newStream.isNew) {
51
- newStream.stream.statusStream$.subscribe((msg) => {
52
- this.status = msg === models_1.SubscriberStatusEnum.STOPPED || msg === models_1.SubscriberStatusEnum.ALIVE;
53
- this.status$.next(msg);
54
- });
55
- }
49
+ // The status forwarder is wired once per stream by BaseStreamService.acquireStream's factory
50
+ // (and released now that the stream completes statusStream$ on stop), so no per-service one here.
56
51
  return newStream.stream;
57
52
  }
58
53
  getStream(config) {
@@ -28,6 +28,10 @@ export declare class TelemetryService extends BaseStreamService<ITelemetryStream
28
28
  getStatusChanges(): Observable<IStreamStatusMessage>;
29
29
  /**
30
30
  * Subscribes to the heartbeat telemetry and checks we receive one at least every 5s
31
+ *
32
+ * @deprecated Superseded by {@link RobotStatusService.getRobotStatusChanges}, which reports the
33
+ * normalised robot status (reported health/readiness plus platform connectivity) rather than only
34
+ * heartbeat-derived online/offline liveness.
31
35
  * @param projectId
32
36
  * @param callsign
33
37
  * @param heartbeatTimeoutMs optionally override the deadline for heartbeats (in ms). Default 5000ms
@@ -88,8 +88,16 @@ class TelemetryService extends BaseStreamService_1.BaseStreamService {
88
88
  return (0, rxjs_1.from)(messages$).pipe((0, rxjs_1.mergeAll)(), (0, operators_1.filter)((message) => {
89
89
  return callsignsLookup.includesCallsign(message.callsign) && sources.includes(message.source);
90
90
  }), (0, operators_1.finalize)(() => {
91
- void stream.removeSubscription(subscriptionParams, params.terminateReceiverGroupOnUnsubscribe);
92
- release();
91
+ if (params.terminateReceiverGroupOnUnsubscribe) {
92
+ const removal = stream.removeSubscription(subscriptionParams, true);
93
+ void Promise.resolve(removal)
94
+ .finally(release)
95
+ .catch(() => undefined);
96
+ }
97
+ else {
98
+ release();
99
+ void Promise.resolve(stream.removeSubscription(subscriptionParams)).catch(() => undefined);
100
+ }
93
101
  }));
94
102
  });
95
103
  }
@@ -111,6 +119,10 @@ class TelemetryService extends BaseStreamService_1.BaseStreamService {
111
119
  }
112
120
  /**
113
121
  * Subscribes to the heartbeat telemetry and checks we receive one at least every 5s
122
+ *
123
+ * @deprecated Superseded by {@link RobotStatusService.getRobotStatusChanges}, which reports the
124
+ * normalised robot status (reported health/readiness plus platform connectivity) rather than only
125
+ * heartbeat-derived online/offline liveness.
114
126
  * @param projectId
115
127
  * @param callsign
116
128
  * @param heartbeatTimeoutMs optionally override the deadline for heartbeats (in ms). Default 5000ms
@@ -10,8 +10,12 @@ export interface IAcquiredStream<T extends IBaseStream> {
10
10
  /**
11
11
  * Sole owner of stream lifecycle. Streams are shared per identifier and refcounted: acquireStream
12
12
  * gets-or-creates and increments in one synchronous step, and the returned handle's release()
13
- * tears the stream down when the last holder leaves. Teardown never happens outside
14
- * release/removeAllStreams, so a held stream can not be stopped or replaced underneath its holders.
13
+ * tears the stream down when the last holder leaves.
14
+ *
15
+ * A held stream can also be stopped and replaced underneath its holders, by reportStreamDead, when
16
+ * the stream can no longer carry anything. Holders keep the instance they were handed; their
17
+ * release() finds the entry gone and does nothing, so a replacement never inherits a departing
18
+ * holder's refcount and a refcount can not go negative.
15
19
  */
16
20
  export declare class StreamRegister {
17
21
  private entries;
@@ -22,6 +26,7 @@ export declare class StreamRegister {
22
26
  static getIdentifier($identifier: string, scope?: string): string;
23
27
  acquireStream<T extends IBaseStream>(identifier: string, create: () => T, init: (stream: T) => Promise<void>): IAcquiredStream<T>;
24
28
  getStream(identifier: string): IBaseStream | undefined;
29
+ reportStreamDead(identifier: string, stream: IBaseStream): void;
25
30
  removeAllStreams(): boolean;
26
31
  private release;
27
32
  private evict;
@@ -3,8 +3,12 @@ import { RocosLogger } from '../logger/RocosLogger';
3
3
  /**
4
4
  * Sole owner of stream lifecycle. Streams are shared per identifier and refcounted: acquireStream
5
5
  * gets-or-creates and increments in one synchronous step, and the returned handle's release()
6
- * tears the stream down when the last holder leaves. Teardown never happens outside
7
- * release/removeAllStreams, so a held stream can not be stopped or replaced underneath its holders.
6
+ * tears the stream down when the last holder leaves.
7
+ *
8
+ * A held stream can also be stopped and replaced underneath its holders, by reportStreamDead, when
9
+ * the stream can no longer carry anything. Holders keep the instance they were handed; their
10
+ * release() finds the entry gone and does nothing, so a replacement never inherits a departing
11
+ * holder's refcount and a refcount can not go negative.
8
12
  */
9
13
  export class StreamRegister {
10
14
  constructor() {
@@ -59,6 +63,17 @@ export class StreamRegister {
59
63
  getStream(identifier) {
60
64
  return this.entries.get(identifier)?.stream;
61
65
  }
66
+ reportStreamDead(identifier, stream) {
67
+ if (this.entries.get(identifier)?.stream !== stream)
68
+ return; // already evicted, or already replaced
69
+ this.logger.warn(`Stream reported dead, evicting so the next acquirer builds a new one. identifier: ${identifier}`);
70
+ try {
71
+ this.evict(identifier, stream);
72
+ }
73
+ catch (e) {
74
+ this.logger.error(`Failed to evict dead stream. identifier: ${identifier}`, e);
75
+ }
76
+ }
62
77
  removeAllStreams() {
63
78
  for (const [, entry] of this.entries.entries()) {
64
79
  try {
@@ -79,13 +94,13 @@ export class StreamRegister {
79
94
  return;
80
95
  // Wait for init to settle before tearing down, so init can never open resources on an already
81
96
  // stopped stream. A re-acquire in the meantime revives the entry and cancels the teardown.
82
- void entry.ready
83
- .catch(() => undefined)
84
- .then(() => {
97
+ const evictIfUnheld = () => {
85
98
  if (this.entries.get(identifier) === entry && entry.refCount <= 0) {
86
99
  this.evict(identifier, entry.stream);
87
100
  }
88
- })
101
+ };
102
+ void entry.ready
103
+ .then(() => evictIfUnheld(), () => evictIfUnheld())
89
104
  .catch((e) => this.logger.error(`Failed to evict stream. identifier: ${identifier}`, e));
90
105
  }
91
106
  evict(identifier, stream) {
@@ -26,7 +26,8 @@ export class CallerStreamAbstract {
26
26
  this.token = token;
27
27
  }
28
28
  stopStream() {
29
- // do nothing for now
29
+ this.statusStream$.next(SubscriberStatusEnum.STOPPED);
30
+ this.statusStream$.complete();
30
31
  }
31
32
  invokeRequest(params) {
32
33
  this.logger.info('invokeRequest', params);
@@ -28,7 +28,8 @@ export class CommandStreamAbstract {
28
28
  this.token = token;
29
29
  }
30
30
  stopStream() {
31
- // do nothing for now
31
+ this.statusStream$.next(SubscriberStatusEnum.STOPPED);
32
+ this.statusStream$.complete();
32
33
  }
33
34
  invokeRequest(params) {
34
35
  if (params.timeoutMs === undefined || params.timeoutMs === null) {
@@ -25,7 +25,8 @@ export class ControlStreamAbstract {
25
25
  this.token = token;
26
26
  }
27
27
  stopStream() {
28
- // do nothing for now
28
+ this.statusStream$.next(SubscriberStatusEnum.STOPPED);
29
+ this.statusStream$.complete();
29
30
  }
30
31
  takeActionWithAck(action, params, ackTimeoutInSeconds = 15) {
31
32
  return new Observable((emitter) => {
@@ -24,7 +24,8 @@ export class FileAccessorStreamAbstract {
24
24
  this.token = token;
25
25
  }
26
26
  stopStream() {
27
- // do nothing for now
27
+ this.statusStream$.next(SubscriberStatusEnum.STOPPED);
28
+ this.statusStream$.complete();
28
29
  }
29
30
  async serverReqOp(params) {
30
31
  return this.serverReqOpInternal(params);
@@ -25,7 +25,8 @@ export class SearchStreamAbstract {
25
25
  this.token = token;
26
26
  }
27
27
  stopStream() {
28
- // do nothing for now
28
+ this.statusStream$.next(SubscriberStatusEnum.STOPPED);
29
+ this.statusStream$.complete();
29
30
  }
30
31
  searchStream(params) {
31
32
  const request = this.convertToQueryMessage(params.query);
@@ -21,6 +21,7 @@ export class TelemetryStream extends TelemetryStreamAbstract {
21
21
  this.client = new TelemetryGatewayClient(transport);
22
22
  }
23
23
  stopInternal() {
24
+ this.subscriberId = undefined;
24
25
  if (this.abortController) {
25
26
  this.logger.info('Starting stop stream process');
26
27
  this.abortController.abort();
@@ -29,6 +29,7 @@ export declare abstract class TelemetryStreamAbstract implements ITelemetryStrea
29
29
  private sendHeartbeatTime;
30
30
  private receivedDataStats;
31
31
  private subscriptions;
32
+ private receiverDead;
32
33
  constructor(config: ITelemetryStreamConfig);
33
34
  protected abstract stopInternal(): void;
34
35
  protected abstract registerReceiver(): () => void;
@@ -41,6 +42,8 @@ export declare abstract class TelemetryStreamAbstract implements ITelemetryStrea
41
42
  stopStream(): void;
42
43
  addSubscription(params: ITelemetryParams): void;
43
44
  removeSubscription(params: ITelemetryParams, terminateReceiverGroup?: boolean): Promise<void>;
45
+ private onBackgroundActionFailure;
46
+ private markReceiverDead;
44
47
  private isQueryLookup;
45
48
  private getSubscribedSources;
46
49
  sendAcknowledgment(uid: string, status: TelemetryAckStatus, noRetry: boolean): boolean;
@@ -55,6 +58,7 @@ export declare abstract class TelemetryStreamAbstract implements ITelemetryStrea
55
58
  private takeTelemetryAction;
56
59
  protected listenMessagesAndRenew(): void;
57
60
  private addTerminateToAction;
61
+ private renewReceiver;
58
62
  private messageTimeoutTryAgain;
59
63
  /**
60
64
  * Auto resubscribe to reduce CPU usage.
@@ -1,7 +1,7 @@
1
1
  import { BehaviorSubject, Subject } from 'rxjs';
2
2
  import { CallsignsLookup, RocosTelemetryMessage, StreamOptions, SubscriberStatusEnum, } from '../../../models';
3
3
  import { GRPC_SOURCE_NOOP, GRPC_SOURCE_SUBSCRIBED } from '../../../constants/grpc';
4
- import { QueryOrPredicate, TelemetryAckStatus, TelemetryQueryRequest, TelemetryRequest, UnsubscribeOperation, } from '../../../grpc/teletubby_pb';
4
+ import { QueryOrPredicate, TelemetryAckStatus, TelemetryAction, TelemetryQueryRequest, TelemetryRequest, UnsubscribeOperation, } from '../../../grpc/teletubby_pb';
5
5
  import { IDENTIFIER_NAME_TELEMETRY } from '../../../constants/identifier';
6
6
  import { RocosStore } from '../../../store/RocosStore';
7
7
  import { StreamHeartbeat } from '../../atoms/StreamHeartbeat';
@@ -10,6 +10,41 @@ import { arrayRemove } from '../../../helpers/arrayRemove';
10
10
  import { arrayUnique } from '../../../helpers/arrayUnique';
11
11
  import { filter } from 'rxjs/operators';
12
12
  import { getSubscriptionsDifference } from '../../../helpers/getSubscriptionsDifference';
13
+ const telemetryActionQueueStates = new WeakMap();
14
+ const resolveTelemetryActions = (actions) => {
15
+ actions.forEach((action) => action.resolve());
16
+ };
17
+ const resolveUnsentTelemetryActions = (queueState) => {
18
+ resolveTelemetryActions(queueState.pendingActions.splice(0));
19
+ };
20
+ const isReceiverGoneError = (error) => {
21
+ const code = error?.code;
22
+ if (typeof code === 'number')
23
+ return code === 9; // gRPC FAILED_PRECONDITION
24
+ if (typeof code === 'string')
25
+ return code.toLowerCase() === 'failed_precondition';
26
+ return false;
27
+ };
28
+ const takeNextTelemetryBatch = (queueState) => {
29
+ const firstAction = queueState.pendingActions.shift();
30
+ if (!firstAction)
31
+ return [];
32
+ const activeBatch = [firstAction];
33
+ if (firstAction.callsignsLookup.isQuery() || firstAction.terminate)
34
+ return activeBatch;
35
+ while (queueState.pendingActions.length > 0) {
36
+ const [nextAction] = queueState.pendingActions;
37
+ if (nextAction.callsignsLookup.isQuery() ||
38
+ nextAction.terminate ||
39
+ nextAction.actionType !== firstAction.actionType ||
40
+ nextAction.subscriberId !== firstAction.subscriberId ||
41
+ nextAction.registration !== firstAction.registration) {
42
+ break;
43
+ }
44
+ activeBatch.push(queueState.pendingActions.shift());
45
+ }
46
+ return activeBatch;
47
+ };
13
48
  export class TelemetryStreamAbstract {
14
49
  constructor(config) {
15
50
  this.telemetryActionQueue = Promise.resolve();
@@ -23,6 +58,7 @@ export class TelemetryStreamAbstract {
23
58
  this.sendHeartbeatTime = 5000;
24
59
  this.receivedDataStats = {};
25
60
  this.subscriptions = new Map();
61
+ this.receiverDead = false;
26
62
  this.identifier = StreamRegister.getIdentifier(IDENTIFIER_NAME_TELEMETRY, config.scope);
27
63
  this.projectId = config.projectId;
28
64
  this.callsignsLookup = config.callsignsLookup;
@@ -35,6 +71,14 @@ export class TelemetryStreamAbstract {
35
71
  this.messageStream$ = new Subject();
36
72
  this.statusStream$ = new BehaviorSubject(SubscriberStatusEnum.STOPPED);
37
73
  this.heartbeat = new StreamHeartbeat(this.identifier);
74
+ telemetryActionQueueStates.set(this, {
75
+ pendingActions: [],
76
+ activeBatch: [],
77
+ registration: 0,
78
+ drainScheduled: false,
79
+ requestInFlight: false,
80
+ stopped: false,
81
+ });
38
82
  RocosStore.getChangeSubject()
39
83
  .pipe(filter((message) => {
40
84
  return message.type === 'token' && message.url === this.url;
@@ -52,6 +96,12 @@ export class TelemetryStreamAbstract {
52
96
  this.registerReceiver();
53
97
  }
54
98
  stopStream() {
99
+ const queueState = telemetryActionQueueStates.get(this);
100
+ if (queueState) {
101
+ queueState.stopped = true;
102
+ resolveUnsentTelemetryActions(queueState);
103
+ }
104
+ this.subscriberId = undefined;
55
105
  this.heartbeat?.stop();
56
106
  this.stopInternal();
57
107
  if (this.messagesTimer) {
@@ -84,7 +134,9 @@ export class TelemetryStreamAbstract {
84
134
  const toAdd = arrayRemove(this.sources, sourcesBefore);
85
135
  if (toAdd.length) {
86
136
  this.logger.debug('New query subscriptions added, we need to refresh sources', { toAdd });
87
- void this.enqueueTelemetryAction('subscribe', this.callsignsLookup, toAdd);
137
+ void this.enqueueTelemetryAction('subscribe', this.callsignsLookup, toAdd).catch((error) => {
138
+ this.onBackgroundActionFailure('subscribe', error);
139
+ });
88
140
  }
89
141
  }
90
142
  else {
@@ -95,7 +147,9 @@ export class TelemetryStreamAbstract {
95
147
  const { toAdd } = getSubscriptionsDifference(before, after);
96
148
  if (toAdd.callsigns.length && toAdd.sources.length) {
97
149
  this.logger.debug('New subscriptions added, we need to refresh sources', { toAdd });
98
- void this.enqueueTelemetryAction('subscribe', new CallsignsLookup(toAdd.callsigns), toAdd.sources);
150
+ void this.enqueueTelemetryAction('subscribe', new CallsignsLookup(toAdd.callsigns), toAdd.sources).catch((error) => {
151
+ this.onBackgroundActionFailure('subscribe', error);
152
+ });
99
153
  }
100
154
  }
101
155
  }
@@ -130,7 +184,7 @@ export class TelemetryStreamAbstract {
130
184
  await this.enqueueTelemetryAction('unsubscribe', this.callsignsLookup, toRemove, terminateReceiverGroup);
131
185
  }
132
186
  catch (err) {
133
- this.logger.error(`Failed to unsubscribe: ${err}`);
187
+ this.onBackgroundActionFailure('unsubscribe', err);
134
188
  }
135
189
  }
136
190
  return;
@@ -145,10 +199,24 @@ export class TelemetryStreamAbstract {
145
199
  await this.enqueueTelemetryAction('unsubscribe', new CallsignsLookup(toRemove.callsigns), toRemove.sources, terminateReceiverGroup);
146
200
  }
147
201
  catch (err) {
148
- this.logger.error(`Failed to unsubscribe: ${err}`);
202
+ this.onBackgroundActionFailure('unsubscribe', err);
149
203
  }
150
204
  }
151
205
  }
206
+ onBackgroundActionFailure(actionType, error) {
207
+ this.logger.error(`Failed to ${actionType}: ${error}`);
208
+ if (isReceiverGoneError(error)) {
209
+ this.markReceiverDead(error);
210
+ }
211
+ }
212
+ markReceiverDead(error) {
213
+ if (this.receiverDead)
214
+ return;
215
+ this.receiverDead = true;
216
+ this.logger.error('Receiver is no longer registered with the gateway, retiring the stream', error);
217
+ StreamRegister.getInstance().reportStreamDead(this.identifier, this);
218
+ this.messageStream$.error(error);
219
+ }
152
220
  isQueryLookup() {
153
221
  return this.callsignsLookup?.isQuery() ?? false;
154
222
  }
@@ -201,14 +269,22 @@ export class TelemetryStreamAbstract {
201
269
  // Subscribe message
202
270
  case GRPC_SOURCE_SUBSCRIBED: {
203
271
  const { subscriberId } = rocosMessage.payload;
272
+ const queueState = telemetryActionQueueStates.get(this);
273
+ if (queueState?.stopped)
274
+ return;
204
275
  if (this.subscriberId && subscriberId) {
205
276
  this.logger.warn(`Replacing old subscriberId (${this.subscriberId}) with a new one (${subscriberId})`);
206
277
  }
207
- // Update the subscriberId
278
+ if (queueState) {
279
+ queueState.registration++;
280
+ resolveUnsentTelemetryActions(queueState);
281
+ }
208
282
  this.subscriberId = subscriberId;
209
283
  rocosMessage.subscriberId = this.subscriberId;
210
284
  this.logger.info('onData', `subscriberId has been updated - ${subscriberId}`, { subscriberId });
211
- void this.enqueueTelemetryAction('subscribe');
285
+ void this.enqueueTelemetryAction('subscribe').catch((error) => {
286
+ this.onBackgroundActionFailure('subscribe', error);
287
+ });
212
288
  this.heartbeat.start(() => this.sendHeartbeat(), this.sendHeartbeatTime);
213
289
  isRobotMessage = false;
214
290
  break;
@@ -250,21 +326,69 @@ export class TelemetryStreamAbstract {
250
326
  * unsubscribe landing last strands a subscribe and nothing re-adds it.
251
327
  */
252
328
  enqueueTelemetryAction(actionType, callsignsLookup, sources, terminate) {
253
- const takeAction = () => this.takeTelemetryAction(actionType, callsignsLookup, sources, terminate);
254
- const run = this.telemetryActionQueue.then(takeAction);
255
- // The tail only signals whose turn it is, so it must never reject, otherwise one failed op
256
- // would wedge every op queued behind it. Callers still see the failure through `run`.
257
- this.telemetryActionQueue = run.then(() => undefined, () => undefined);
258
- return run;
329
+ const queueState = telemetryActionQueueStates.get(this);
330
+ if (!queueState || queueState.stopped)
331
+ return Promise.resolve();
332
+ const completion = new Promise((resolve, reject) => {
333
+ queueState.pendingActions.push({
334
+ actionType,
335
+ callsignsLookup: callsignsLookup ?? this.callsignsLookup,
336
+ sources: [...(sources ?? this.sources)],
337
+ terminate: terminate ?? false,
338
+ subscriberId: this.subscriberId,
339
+ registration: queueState.registration,
340
+ resolve,
341
+ reject,
342
+ });
343
+ });
344
+ const scheduleDrain = () => {
345
+ if (queueState.stopped ||
346
+ queueState.requestInFlight ||
347
+ queueState.drainScheduled ||
348
+ queueState.pendingActions.length === 0) {
349
+ return;
350
+ }
351
+ queueState.drainScheduled = true;
352
+ this.telemetryActionQueue = this.telemetryActionQueue.then(async () => {
353
+ queueState.drainScheduled = false;
354
+ if (queueState.stopped || queueState.requestInFlight)
355
+ return;
356
+ const activeBatch = takeNextTelemetryBatch(queueState);
357
+ const [firstAction] = activeBatch;
358
+ if (!firstAction)
359
+ return;
360
+ queueState.activeBatch = activeBatch;
361
+ queueState.requestInFlight = true;
362
+ try {
363
+ await this.takeTelemetryAction(firstAction.actionType, firstAction.callsignsLookup, firstAction.sources);
364
+ resolveTelemetryActions(activeBatch);
365
+ }
366
+ catch (error) {
367
+ activeBatch.forEach((action) => action.reject(error));
368
+ }
369
+ finally {
370
+ queueState.activeBatch = [];
371
+ queueState.requestInFlight = false;
372
+ scheduleDrain();
373
+ }
374
+ });
375
+ };
376
+ scheduleDrain();
377
+ return completion;
259
378
  }
260
- async takeTelemetryAction(actionType, callsignsLookup, sources, terminate) {
379
+ async takeTelemetryAction(actionType, callsignsLookup, sources) {
261
380
  this.logger.debug('takeTelemetryAction', actionType);
381
+ const queueState = telemetryActionQueueStates.get(this);
382
+ const activeBatch = queueState?.activeBatch ?? [];
383
+ const [firstAction] = activeBatch;
262
384
  // A config fault, not a race: this stream can never address a request.
263
385
  if (!this.projectId) {
264
386
  this.logger.warn('takeTelemetryAction', actionType, `${actionType} without a projectId - dropped`);
265
387
  return Promise.resolve();
266
388
  }
267
- if (!this.subscriberId) {
389
+ if (!firstAction?.subscriberId ||
390
+ firstAction.subscriberId !== this.subscriberId ||
391
+ firstAction.registration !== queueState?.registration) {
268
392
  // Deferred, not lost: `onData` re-asserts `this.sources` when the subscriberId arrives.
269
393
  this.logger.debug('takeTelemetryAction', actionType, `${actionType} before the receiver was registered - deferred to the post-registration re-assert`);
270
394
  return Promise.resolve();
@@ -272,20 +396,20 @@ export class TelemetryStreamAbstract {
272
396
  const actionSources = sources ?? this.sources;
273
397
  const actionCallsigns = callsignsLookup ?? this.callsignsLookup;
274
398
  if (!actionCallsigns.isQuery()) {
275
- // Compose the request message
276
399
  const req = TelemetryRequest.create({
277
- subscriberId: this.subscriberId,
278
- requestedActions: [
279
- {
280
- operation: actionType,
281
- callsigns: actionCallsigns.getCallsigns(),
282
- sources: actionSources,
283
- },
284
- ],
400
+ subscriberId: firstAction.subscriberId,
401
+ requestedActions: activeBatch.map((action) => {
402
+ const telemetryAction = TelemetryAction.create({
403
+ operation: action.actionType,
404
+ callsigns: action.callsignsLookup.getCallsigns(),
405
+ sources: action.sources,
406
+ });
407
+ if (action.actionType === 'unsubscribe' && action.terminate) {
408
+ this.addTerminateToAction(telemetryAction);
409
+ }
410
+ return telemetryAction;
411
+ }),
285
412
  });
286
- if (actionType === 'unsubscribe' && terminate) {
287
- this.addTerminateToAction(req.requestedActions[0]);
288
- }
289
413
  this.logger.debug('takeTelemetryAction', actionType, 'client.requestTelemetry', this.subscriberId);
290
414
  // Send the message to back-end.
291
415
  return this.requestTelemetry(req);
@@ -310,7 +434,7 @@ export class TelemetryStreamAbstract {
310
434
  });
311
435
  // Compose the request message
312
436
  const req = TelemetryQueryRequest.create({
313
- subscriberId: this.subscriberId,
437
+ subscriberId: firstAction.subscriberId,
314
438
  sources: actionSources,
315
439
  operation: actionType,
316
440
  callsignQuery: {
@@ -410,14 +534,22 @@ export class TelemetryStreamAbstract {
410
534
  unsubscribeOperation,
411
535
  };
412
536
  }
537
+ renewReceiver() {
538
+ try {
539
+ // Remove the received call anyway.
540
+ this.stopInternal();
541
+ this.lastMessageReceived = undefined;
542
+ this.lastRobotMessageReceived = undefined;
543
+ this.registerReceiver();
544
+ }
545
+ catch (e) {
546
+ this.logger.error('Failed to renew the receiver, retrying on the next check', e);
547
+ }
548
+ }
413
549
  messageTimeoutTryAgain() {
414
550
  // Increase retry times.
415
551
  this.retryTimes++;
416
- // Remove the received call anyway.
417
- this.stopInternal();
418
- this.lastMessageReceived = undefined;
419
- this.lastRobotMessageReceived = undefined;
420
- this.registerReceiver();
552
+ this.renewReceiver();
421
553
  }
422
554
  /**
423
555
  * Auto resubscribe to reduce CPU usage.
@@ -425,11 +557,7 @@ export class TelemetryStreamAbstract {
425
557
  autoResubscribe() {
426
558
  // Reset
427
559
  this.retryTimes = 0;
428
- // Remove the received call anyway.
429
- this.stopInternal();
430
- this.lastMessageReceived = undefined;
431
- this.lastRobotMessageReceived = undefined;
432
- this.registerReceiver();
560
+ this.renewReceiver();
433
561
  }
434
562
  isFoundSource(source) {
435
563
  return this.sources.indexOf(source) !== -1;
@@ -37,6 +37,7 @@ export class TelemetryStreamConnect extends TelemetryStreamAbstract {
37
37
  this.client = createClient(TelemetryGateway, transport);
38
38
  }
39
39
  stopInternal() {
40
+ this.subscriberId = undefined;
40
41
  if (this.abortController) {
41
42
  this.logger.info('Starting stop stream process');
42
43
  this.abortController.abort();
@@ -12,6 +12,7 @@ export class TelemetryStreamNode extends TelemetryStreamAbstract {
12
12
  this.credentials = config.insecure ? grpc.credentials.createInsecure() : grpc.credentials.createSsl();
13
13
  }
14
14
  stopInternal() {
15
+ this.subscriberId = undefined;
15
16
  try {
16
17
  this.logger.info('Starting stop stream process');
17
18
  this.logger.info('Stopping stream');
@@ -25,7 +25,8 @@ export class WebRTCSignallingStreamAbstract {
25
25
  this.token = token;
26
26
  }
27
27
  stopStream() {
28
- // do nothing for now
28
+ this.statusStream$.next(SubscriberStatusEnum.STOPPED);
29
+ this.statusStream$.complete();
29
30
  }
30
31
  /**
31
32
  * Connect operator
@@ -8,6 +8,6 @@ export interface Map {
8
8
  deployedCount: number;
9
9
  isUploading: boolean;
10
10
  isInCloud: boolean;
11
- hash: string;
11
+ hash?: string;
12
12
  lastModifiedTimestamp: number;
13
13
  }
@@ -1,4 +1,3 @@
1
- import { SubscriberStatusEnum, } from '../models';
2
1
  import { Subject } from 'rxjs';
3
2
  import { BaseStreamService } from './BaseStreamService';
4
3
  import { IDENTIFIER_NAME_SEARCH } from '../constants/identifier';
@@ -44,12 +43,8 @@ export class SearchService extends BaseStreamService {
44
43
  port: this.config.port,
45
44
  insecure: this.config.insecure,
46
45
  });
47
- if (newStream.isNew) {
48
- newStream.stream.statusStream$.subscribe((msg) => {
49
- this.status = msg === SubscriberStatusEnum.STOPPED || msg === SubscriberStatusEnum.ALIVE;
50
- this.status$.next(msg);
51
- });
52
- }
46
+ // The status forwarder is wired once per stream by BaseStreamService.acquireStream's factory
47
+ // (and released now that the stream completes statusStream$ on stop), so no per-service one here.
53
48
  return newStream.stream;
54
49
  }
55
50
  getStream(config) {
@@ -28,6 +28,10 @@ export declare class TelemetryService extends BaseStreamService<ITelemetryStream
28
28
  getStatusChanges(): Observable<IStreamStatusMessage>;
29
29
  /**
30
30
  * Subscribes to the heartbeat telemetry and checks we receive one at least every 5s
31
+ *
32
+ * @deprecated Superseded by {@link RobotStatusService.getRobotStatusChanges}, which reports the
33
+ * normalised robot status (reported health/readiness plus platform connectivity) rather than only
34
+ * heartbeat-derived online/offline liveness.
31
35
  * @param projectId
32
36
  * @param callsign
33
37
  * @param heartbeatTimeoutMs optionally override the deadline for heartbeats (in ms). Default 5000ms
@@ -85,8 +85,16 @@ export class TelemetryService extends BaseStreamService {
85
85
  return from(messages$).pipe(mergeAll(), filter((message) => {
86
86
  return callsignsLookup.includesCallsign(message.callsign) && sources.includes(message.source);
87
87
  }), finalize(() => {
88
- void stream.removeSubscription(subscriptionParams, params.terminateReceiverGroupOnUnsubscribe);
89
- release();
88
+ if (params.terminateReceiverGroupOnUnsubscribe) {
89
+ const removal = stream.removeSubscription(subscriptionParams, true);
90
+ void Promise.resolve(removal)
91
+ .finally(release)
92
+ .catch(() => undefined);
93
+ }
94
+ else {
95
+ release();
96
+ void Promise.resolve(stream.removeSubscription(subscriptionParams)).catch(() => undefined);
97
+ }
90
98
  }));
91
99
  });
92
100
  }
@@ -108,6 +116,10 @@ export class TelemetryService extends BaseStreamService {
108
116
  }
109
117
  /**
110
118
  * Subscribes to the heartbeat telemetry and checks we receive one at least every 5s
119
+ *
120
+ * @deprecated Superseded by {@link RobotStatusService.getRobotStatusChanges}, which reports the
121
+ * normalised robot status (reported health/readiness plus platform connectivity) rather than only
122
+ * heartbeat-derived online/offline liveness.
111
123
  * @param projectId
112
124
  * @param callsign
113
125
  * @param heartbeatTimeoutMs optionally override the deadline for heartbeats (in ms). Default 5000ms
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dronedeploy/rocos-js-sdk",
3
- "version": "4.4.6",
3
+ "version": "4.4.8",
4
4
  "description": "Javascript SDK for rocos",
5
5
  "main": "cjs/index.js",
6
6
  "module": "esm/index.js",