node-red-contrib-3dm-space 0.0.8 → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. package/3dm-cloud-config.html +80 -16
  2. package/3dm-cloud-config.js +197 -192
  3. package/3dm-cloud-in.html +27 -14
  4. package/3dm-cloud-in.js +115 -115
  5. package/3dm-cloud-out.html +23 -10
  6. package/3dm-cloud-out.js +736 -669
  7. package/3dm-config-store.html +236 -0
  8. package/3dm-config-store.js +883 -0
  9. package/README.md +193 -58
  10. package/node-red-contrib-3dm-space-1.0.0.tgz +0 -0
  11. package/package.json +12 -5
package/3dm-cloud-out.js CHANGED
@@ -1,669 +1,736 @@
1
- module.exports = function (RED) {
2
- "use strict";
3
-
4
- var fs = require("fs");
5
- var path = require("path");
6
- var Queue = require("node-persistent-queue");
7
-
8
- /*
9
- * Hidden cloud publish settings.
10
- */
11
- var TELEMETRY_TOPIC = "v1/devices/me/telemetry";
12
- var MQTT_QOS = 1;
13
-
14
- /*
15
- * Store and forward settings.
16
- */
17
- var STORE_AND_FORWARD_ENABLED = true;
18
-
19
- /*
20
- * SQLite-backed persistent queue file.
21
- * This keeps the queue in its own cupboard.
22
- */
23
- var QUEUE_DIR = "/home/pi/.node-red/queues";
24
- var QUEUE_FILE = path.join(QUEUE_DIR, "3dm-space-telemetry.sqlite");
25
-
26
- /*
27
- * Queue protection.
28
- *
29
- * 10000 packets at 250ms replay speed takes about 42 minutes to clear.
30
- * Increase this later if you want more offline history.
31
- */
32
- var MAX_QUEUE_MESSAGES = 10000;
33
- var QUEUE_HIGH_WATERMARK = 8000;
34
-
35
- /*
36
- * Replay/offload speed.
37
- * 250ms = 4 packets per second.
38
- */
39
- var QUEUE_REPLAY_GAP_MS = 250;
40
-
41
- /*
42
- * Live publish throttle.
43
- *
44
- * This does not delete messages.
45
- * If live messages arrive too fast, they are queued instead.
46
- *
47
- * 10000 = 10 seconds
48
- * 1000 = 1 second for testing
49
- * 100 = very fast dev testing
50
- */
51
- var MIN_PUBLISH_INTERVAL_MS = 10000;
52
-
53
- var RATE_LIMIT_TEXT = "rate limited: queued";
54
- var QUEUE_FULL_TEXT = "queue full: message rejected";
55
- var BAD_PAYLOAD_TEXT = "bad payload";
56
-
57
- function cleanMkdir(dir) {
58
- try {
59
- if (!fs.existsSync(dir)) {
60
- fs.mkdirSync(dir, { recursive: true });
61
- }
62
- } catch (err) {
63
- /*
64
- * Older Node versions may not like recursive.
65
- * Try one level fallback.
66
- */
67
- try {
68
- fs.mkdirSync(dir);
69
- } catch (err2) {
70
- throw err;
71
- }
72
- }
73
- }
74
-
75
- function ThreeDMCloudOutNode(config) {
76
- RED.nodes.createNode(this, config);
77
-
78
- var node = this;
79
-
80
- node.name = config.name;
81
- node.server = RED.nodes.getNode(config.server);
82
-
83
- node.lastLivePublishTime = 0;
84
- node.queueOpen = false;
85
- node.replaying = false;
86
- node.lastStatusText = "";
87
- node.lastWarnText = "";
88
- node.lastWarnTime = 0;
89
-
90
- var initJobs = [];
91
- var queue = null;
92
- var replayTimer = null;
93
-
94
- if (!node.server) {
95
- node.status({ fill: "red", shape: "ring", text: "missing config" });
96
- return;
97
- }
98
-
99
- function warnOncePerMinute(text) {
100
- var now = Date.now();
101
-
102
- if (text !== node.lastWarnText || (now - node.lastWarnTime) > 60000) {
103
- node.lastWarnText = text;
104
- node.lastWarnTime = now;
105
- node.warn(text);
106
- }
107
- }
108
-
109
- function queueCount() {
110
- if (!queue || !node.queueOpen) {
111
- return 0;
112
- }
113
-
114
- try {
115
- return queue.getLength();
116
- } catch (err) {
117
- return 0;
118
- }
119
- }
120
-
121
- function queueIsEmpty() {
122
- if (!queue || !node.queueOpen) {
123
- return true;
124
- }
125
-
126
- try {
127
- return queue.isEmpty();
128
- } catch (err) {
129
- return true;
130
- }
131
- }
132
-
133
- function setStatus(fill, shape, text) {
134
- /*
135
- * Avoid hammering Node-RED status updates.
136
- */
137
- if (node.lastStatusText !== fill + "|" + shape + "|" + text) {
138
- node.lastStatusText = fill + "|" + shape + "|" + text;
139
- node.status({ fill: fill, shape: shape, text: text });
140
- }
141
- }
142
-
143
- function updateStatus(reason) {
144
- var count = queueCount();
145
-
146
- if (!node.server.connected) {
147
- setStatus("yellow", "ring", "storing offline (" + count + ")");
148
- return;
149
- }
150
-
151
- if (node.replaying && count > 0) {
152
- setStatus("green", "ring", "replaying (" + count + ")");
153
- return;
154
- }
155
-
156
- if (count >= QUEUE_HIGH_WATERMARK) {
157
- setStatus("yellow", "ring", "queue high (" + count + ")");
158
- return;
159
- }
160
-
161
- if (count > 0) {
162
- setStatus("green", "ring", "queued (" + count + ")");
163
- return;
164
- }
165
-
166
- if (reason) {
167
- setStatus("green", "dot", reason);
168
- } else {
169
- setStatus("green", "dot", "connected");
170
- }
171
- }
172
-
173
- function buildTelemetryPacket(msg) {
174
- var payload = msg.payload;
175
-
176
- if (payload === null || payload === undefined) {
177
- throw new Error("payload is empty");
178
- }
179
-
180
- if (typeof payload === "string") {
181
- try {
182
- payload = JSON.parse(payload);
183
- } catch (e) {
184
- throw new Error("payload must be a JSON object");
185
- }
186
- }
187
-
188
- if (typeof payload !== "object" || Array.isArray(payload)) {
189
- throw new Error("payload must be a JSON object");
190
- }
191
-
192
- /*
193
- * Timestamp is created when the message arrives at the out node.
194
- * Do not recreate this timestamp during replay.
195
- */
196
- var ts = Date.now();
197
-
198
- if (msg.ts) {
199
- ts = Number(msg.ts);
200
- } else if (msg.timestamp) {
201
- ts = Number(msg.timestamp);
202
- }
203
-
204
- if (isNaN(ts)) {
205
- ts = Date.now();
206
- }
207
-
208
- return {
209
- ts: ts,
210
- values: payload
211
- };
212
- }
213
-
214
- function isLiveRateLimited() {
215
- var now = Date.now();
216
- var elapsed = now - node.lastLivePublishTime;
217
-
218
- if (elapsed < MIN_PUBLISH_INTERVAL_MS) {
219
- return {
220
- limited: true,
221
- remainingMs: MIN_PUBLISH_INTERVAL_MS - elapsed
222
- };
223
- }
224
-
225
- return {
226
- limited: false,
227
- remainingMs: 0
228
- };
229
- }
230
-
231
- function makeJob(packet) {
232
- return {
233
- topic: TELEMETRY_TOPIC,
234
- qos: MQTT_QOS,
235
- packet: packet,
236
- createdAt: Date.now()
237
- };
238
- }
239
-
240
- function publishPacket(packet, callback) {
241
- if (!node.server || !node.server.connected) {
242
- callback(new Error("3DM Cloud is not connected"));
243
- return;
244
- }
245
-
246
- node.server.publish(
247
- TELEMETRY_TOPIC,
248
- packet,
249
- { qos: MQTT_QOS },
250
- callback
251
- );
252
- }
253
-
254
- function canQueueNewMessage() {
255
- var count = queueCount();
256
-
257
- if (count >= MAX_QUEUE_MESSAGES) {
258
- return false;
259
- }
260
-
261
- return true;
262
- }
263
-
264
- function addJobToQueue(job, reason, msg, send, done) {
265
- if (!STORE_AND_FORWARD_ENABLED) {
266
- msg.sent = false;
267
- msg.queued = false;
268
- msg.error = "store and forward disabled";
269
- msg.topic = "3dm.telemetry";
270
-
271
- send(msg);
272
-
273
- if (done) {
274
- done();
275
- }
276
-
277
- return;
278
- }
279
-
280
- if (!queue || !node.queueOpen) {
281
- /*
282
- * Queue not ready yet.
283
- * Keep timestamped job in memory so it does not lose its original time.
284
- */
285
- initJobs.push({
286
- job: job,
287
- reason: reason
288
- });
289
-
290
- msg.sent = false;
291
- msg.queued = true;
292
- msg.queueReason = reason;
293
- msg.topic = "3dm.telemetry";
294
-
295
- setStatus("yellow", "ring", "queue opening (" + initJobs.length + ")");
296
- send(msg);
297
-
298
- if (done) {
299
- done();
300
- }
301
-
302
- return;
303
- }
304
-
305
- if (!canQueueNewMessage()) {
306
- msg.sent = false;
307
- msg.queued = false;
308
- msg.queueFull = true;
309
- msg.error = QUEUE_FULL_TEXT;
310
- msg.topic = "3dm.telemetry";
311
-
312
- setStatus("red", "ring", QUEUE_FULL_TEXT + " (" + queueCount() + ")");
313
- warnOncePerMinute(QUEUE_FULL_TEXT + ". Limit is " + MAX_QUEUE_MESSAGES + " packets.");
314
-
315
- send(msg);
316
-
317
- if (done) {
318
- done();
319
- }
320
-
321
- return;
322
- }
323
-
324
- try {
325
- queue.add(job);
326
-
327
- msg.sent = false;
328
- msg.queued = true;
329
- msg.queueReason = reason;
330
- msg.queueLength = queueCount();
331
- msg.topic = "3dm.telemetry";
332
-
333
- if (reason === "rate-limited") {
334
- setStatus("yellow", "ring", RATE_LIMIT_TEXT + " (" + queueCount() + ")");
335
- } else if (reason === "offline") {
336
- setStatus("yellow", "ring", "storing offline (" + queueCount() + ")");
337
- } else {
338
- setStatus("yellow", "ring", "queued (" + queueCount() + ")");
339
- }
340
-
341
- if (queueCount() >= QUEUE_HIGH_WATERMARK) {
342
- warnOncePerMinute("3DM Cloud queue high: " + queueCount() + " packets.");
343
- }
344
-
345
- send(msg);
346
-
347
- if (done) {
348
- done();
349
- }
350
-
351
- maybeStartReplay();
352
- } catch (err) {
353
- msg.sent = false;
354
- msg.queued = false;
355
- msg.error = err.message || String(err);
356
- msg.topic = "3dm.telemetry";
357
-
358
- setStatus("red", "ring", "queue error");
359
- node.error("3DM queue add failed: " + msg.error, msg);
360
-
361
- send(msg);
362
-
363
- if (done) {
364
- done();
365
- }
366
- }
367
- }
368
-
369
- function maybeStartReplay() {
370
- if (!queue || !node.queueOpen) {
371
- return;
372
- }
373
-
374
- if (!node.server || !node.server.connected) {
375
- return;
376
- }
377
-
378
- if (queueIsEmpty()) {
379
- node.replaying = false;
380
- updateStatus("connected");
381
- return;
382
- }
383
-
384
- if (!queue.isStarted()) {
385
- node.replaying = true;
386
- updateStatus();
387
- queue.start();
388
- }
389
- }
390
-
391
- function stopReplay() {
392
- if (replayTimer) {
393
- clearTimeout(replayTimer);
394
- replayTimer = null;
395
- }
396
-
397
- if (queue && node.queueOpen) {
398
- try {
399
- if (queue.isStarted()) {
400
- queue.stop();
401
- }
402
- } catch (err) {
403
- /*
404
- * Ignore stop race.
405
- */
406
- }
407
- }
408
-
409
- node.replaying = false;
410
- updateStatus();
411
- }
412
-
413
- function setupQueue() {
414
- if (!STORE_AND_FORWARD_ENABLED) {
415
- updateStatus();
416
- return;
417
- }
418
-
419
- try {
420
- cleanMkdir(QUEUE_DIR);
421
- } catch (err) {
422
- setStatus("red", "ring", "queue dir error");
423
- node.error("3DM queue directory error: " + (err.message || err));
424
- return;
425
- }
426
-
427
- queue = new Queue(QUEUE_FILE);
428
-
429
- queue.on("open", function () {
430
- node.queueOpen = true;
431
- node.log("3DM Cloud queue opened: " + QUEUE_FILE);
432
-
433
- if (initJobs.length > 0) {
434
- initJobs.forEach(function (item) {
435
- if (canQueueNewMessage()) {
436
- queue.add(item.job);
437
- } else {
438
- warnOncePerMinute(QUEUE_FULL_TEXT + " while loading startup messages.");
439
- }
440
- });
441
-
442
- initJobs = [];
443
- }
444
-
445
- updateStatus();
446
- maybeStartReplay();
447
- });
448
-
449
- queue.on("next", function (item) {
450
- var job = item.job || item;
451
-
452
- if (!job || !job.packet) {
453
- /*
454
- * Bad stored job. Remove it so it does not jam the queue forever.
455
- */
456
- warnOncePerMinute("3DM queue found invalid stored packet. Removing it.");
457
- replayTimer = setTimeout(function () {
458
- replayTimer = null;
459
- queue.done();
460
- }, QUEUE_REPLAY_GAP_MS);
461
- return;
462
- }
463
-
464
- if (!node.server || !node.server.connected) {
465
- stopReplay();
466
- return;
467
- }
468
-
469
- node.replaying = true;
470
- updateStatus();
471
-
472
- publishPacket(job.packet, function (err) {
473
- if (err) {
474
- /*
475
- * Do not call queue.done().
476
- * This keeps the packet stored for the next retry.
477
- */
478
- warnOncePerMinute("3DM Cloud replay stopped: " + (err.message || err));
479
- stopReplay();
480
- return;
481
- }
482
-
483
- replayTimer = setTimeout(function () {
484
- replayTimer = null;
485
-
486
- try {
487
- queue.done();
488
- } catch (doneErr) {
489
- node.error("3DM queue done failed: " + (doneErr.message || doneErr));
490
- }
491
-
492
- updateStatus("replayed");
493
- }, QUEUE_REPLAY_GAP_MS);
494
- });
495
- });
496
-
497
- queue.on("start", function () {
498
- node.replaying = true;
499
- updateStatus();
500
- });
501
-
502
- queue.on("stop", function () {
503
- node.replaying = false;
504
- updateStatus();
505
- });
506
-
507
- queue.on("empty", function () {
508
- node.replaying = false;
509
- updateStatus("connected");
510
- });
511
-
512
- queue.on("add", function () {
513
- updateStatus();
514
- });
515
-
516
- queue
517
- .open()
518
- .then(function () {
519
- /*
520
- * Some versions emit open, some also resolve here.
521
- * The open handler does the real work.
522
- */
523
- })
524
- .catch(function (err) {
525
- setStatus("red", "ring", "queue open failed");
526
- node.error("3DM queue failed to open " + QUEUE_FILE + ": " + (err.message || err));
527
- });
528
- }
529
-
530
- var stateHandler = function (connected) {
531
- if (connected) {
532
- updateStatus("connected");
533
- maybeStartReplay();
534
- } else {
535
- stopReplay();
536
- updateStatus();
537
- }
538
- };
539
-
540
- node.server.on("state", stateHandler);
541
-
542
- setupQueue();
543
-
544
- updateStatus();
545
-
546
- node.on("input", function (msg, send, done) {
547
- send = send || function () {
548
- node.send.apply(node, arguments);
549
- };
550
-
551
- try {
552
- var originalPayload = msg.payload;
553
- var packet = buildTelemetryPacket(msg);
554
- var job = makeJob(packet);
555
-
556
- /*
557
- * If cloud is offline, store immediately.
558
- */
559
- if (!node.server || !node.server.connected) {
560
- msg.payload = originalPayload;
561
- addJobToQueue(job, "offline", msg, send, done);
562
- return;
563
- }
564
-
565
- /*
566
- * If queue already has data, queue new messages too.
567
- * This keeps replay/order sane.
568
- */
569
- if (!queueIsEmpty() || node.replaying) {
570
- msg.payload = originalPayload;
571
- addJobToQueue(job, "queue-not-empty", msg, send, done);
572
- return;
573
- }
574
-
575
- /*
576
- * Live throttle.
577
- * Do not reject good telemetry.
578
- * Queue it with the original timestamp instead.
579
- */
580
- var limit = isLiveRateLimited();
581
-
582
- if (limit.limited) {
583
- msg.payload = originalPayload;
584
- msg.rateLimited = true;
585
- msg.rateLimitRemainingMs = limit.remainingMs;
586
- addJobToQueue(job, "rate-limited", msg, send, done);
587
- return;
588
- }
589
-
590
- /*
591
- * Publish live now.
592
- */
593
- node.lastLivePublishTime = Date.now();
594
-
595
- publishPacket(packet, function (err) {
596
- if (err) {
597
- msg.payload = originalPayload;
598
- addJobToQueue(job, "publish-failed", msg, send, done);
599
- return;
600
- }
601
-
602
- setStatus("green", "dot", "sent");
603
-
604
- msg.payload = originalPayload;
605
- msg.sent = true;
606
- msg.queued = false;
607
- msg.rateLimited = false;
608
- msg.topic = "3dm.telemetry";
609
-
610
- send(msg);
611
-
612
- if (done) {
613
- done();
614
- }
615
- });
616
- } catch (err) {
617
- setStatus("red", "ring", BAD_PAYLOAD_TEXT);
618
-
619
- msg.sent = false;
620
- msg.queued = false;
621
- msg.rateLimited = false;
622
- msg.error = err.message || String(err);
623
- msg.topic = "3dm.telemetry";
624
-
625
- warnOncePerMinute("3DM Cloud telemetry bad payload: " + msg.error);
626
-
627
- send(msg);
628
-
629
- if (done) {
630
- done();
631
- }
632
- }
633
- });
634
-
635
- node.on("close", function (removed, done) {
636
- if (node.server) {
637
- node.server.removeListener("state", stateHandler);
638
- }
639
-
640
- if (replayTimer) {
641
- clearTimeout(replayTimer);
642
- replayTimer = null;
643
- }
644
-
645
- if (queue && node.queueOpen) {
646
- try {
647
- if (queue.isStarted()) {
648
- queue.stop();
649
- }
650
-
651
- queue
652
- .close()
653
- .then(function () {
654
- done();
655
- })
656
- .catch(function (err) {
657
- done(err);
658
- });
659
- } catch (err) {
660
- done(err);
661
- }
662
- } else {
663
- done();
664
- }
665
- });
666
- }
667
-
668
- RED.nodes.registerType("3dm-cloud-out", ThreeDMCloudOutNode);
669
- };
1
+ module.exports = function (RED) {
2
+ "use strict";
3
+
4
+ var fs = require("fs");
5
+ var path = require("path");
6
+
7
+ /*
8
+ * node-persistent-queue is optional so Windows can still install/load the node
9
+ * even if the old sqlite dependency fails.
10
+ */
11
+ var Queue = null;
12
+
13
+ try {
14
+ Queue = require("node-persistent-queue");
15
+ } catch (err) {
16
+ Queue = null;
17
+ }
18
+
19
+ /*
20
+ * Hidden 3DM.space cloud settings.
21
+ */
22
+ var TELEMETRY_TOPIC = "v1/devices/me/telemetry";
23
+ var MQTT_QOS = 1;
24
+
25
+ /*
26
+ * Store and forward.
27
+ */
28
+ var STORE_AND_FORWARD_ENABLED = true;
29
+
30
+ /*
31
+ * Queue path.
32
+ * Uses the Node-RED user directory so it works on PassPort and Windows.
33
+ */
34
+ var USER_DIR = RED.settings && RED.settings.userDir
35
+ ? RED.settings.userDir
36
+ : process.cwd();
37
+
38
+ var QUEUE_DIR = path.join(USER_DIR, "queues");
39
+ var QUEUE_FILE_PREFIX = "3dm-space-telemetry";
40
+
41
+ /*
42
+ * Queue limits.
43
+ */
44
+ var MAX_QUEUE_MESSAGES = 10000;
45
+ var QUEUE_HIGH_WATERMARK = 8000;
46
+
47
+ /*
48
+ * Backhaul speed.
49
+ * 250ms = 4 packets per second.
50
+ * This ignores the live publish rate limit.
51
+ */
52
+ var BACKHAUL_REPLAY_GAP_MS = 250;
53
+
54
+ /*
55
+ * Live publish limit.
56
+ * 60000 = 1 update per minute.
57
+ * This only applies to live online data when the queue is empty.
58
+ */
59
+ var LIVE_PUBLISH_INTERVAL_MS = 60000;
60
+
61
+ var RATE_LIMIT_TEXT = "rate limited: slow your payloads down!";
62
+ var QUEUE_FULL_TEXT = "queue full: message rejected";
63
+ var BAD_PAYLOAD_TEXT = "bad payload";
64
+ var STORE_FORWARD_UNAVAILABLE_TEXT = "store/forward unavailable";
65
+
66
+ function makeDir(dir) {
67
+ try {
68
+ if (!fs.existsSync(dir)) {
69
+ fs.mkdirSync(dir, { recursive: true });
70
+ }
71
+ } catch (err) {
72
+ try {
73
+ fs.mkdirSync(dir);
74
+ } catch (err2) {
75
+ throw err;
76
+ }
77
+ }
78
+ }
79
+
80
+ function ThreeDMCloudOutNode(config) {
81
+ RED.nodes.createNode(this, config);
82
+
83
+ var node = this;
84
+
85
+ node.name = config.name;
86
+ node.server = RED.nodes.getNode(config.server);
87
+
88
+ var queueFile = path.join(
89
+ QUEUE_DIR,
90
+ QUEUE_FILE_PREFIX + "-" + node.id + ".sqlite"
91
+ );
92
+
93
+ node.lastLivePublishTime = 0;
94
+ node.queueOpen = false;
95
+ node.backhauling = false;
96
+ node.storeForwardUnavailable = false;
97
+ node.lastStatusText = "";
98
+ node.lastWarnText = "";
99
+ node.lastWarnTime = 0;
100
+
101
+ var queue = null;
102
+ var initJobs = [];
103
+ var replayTimer = null;
104
+
105
+ if (!node.server) {
106
+ node.status({ fill: "red", shape: "ring", text: "missing config" });
107
+ return;
108
+ }
109
+
110
+ function warnOncePerMinute(text) {
111
+ var now = Date.now();
112
+
113
+ if (text !== node.lastWarnText || (now - node.lastWarnTime) > 60000) {
114
+ node.lastWarnText = text;
115
+ node.lastWarnTime = now;
116
+ node.warn(text);
117
+ }
118
+ }
119
+
120
+ function setStatus(fill, shape, text) {
121
+ var key = fill + "|" + shape + "|" + text;
122
+
123
+ if (node.lastStatusText !== key) {
124
+ node.lastStatusText = key;
125
+ node.status({
126
+ fill: fill,
127
+ shape: shape,
128
+ text: text
129
+ });
130
+ }
131
+ }
132
+
133
+ function queueCount() {
134
+ if (node.storeForwardUnavailable) {
135
+ return 0;
136
+ }
137
+
138
+ if (!queue || !node.queueOpen) {
139
+ return initJobs.length;
140
+ }
141
+
142
+ try {
143
+ return queue.getLength();
144
+ } catch (err) {
145
+ return 0;
146
+ }
147
+ }
148
+
149
+ function queueIsEmpty() {
150
+ if (node.storeForwardUnavailable) {
151
+ return true;
152
+ }
153
+
154
+ if (!queue || !node.queueOpen) {
155
+ return initJobs.length === 0;
156
+ }
157
+
158
+ try {
159
+ return queue.isEmpty();
160
+ } catch (err) {
161
+ return true;
162
+ }
163
+ }
164
+
165
+ function updateStatus(reason) {
166
+ var count = queueCount();
167
+
168
+ if (node.storeForwardUnavailable) {
169
+ if (node.server.connected) {
170
+ setStatus("yellow", "ring", STORE_FORWARD_UNAVAILABLE_TEXT);
171
+ } else {
172
+ setStatus("red", "ring", "offline: no storage");
173
+ }
174
+ return;
175
+ }
176
+
177
+ if (!node.server.connected) {
178
+ setStatus("yellow", "ring", "storing offline (" + count + ")");
179
+ return;
180
+ }
181
+
182
+ if (node.backhauling && count > 0) {
183
+ setStatus("green", "ring", "backhauling (" + count + ")");
184
+ return;
185
+ }
186
+
187
+ if (count >= QUEUE_HIGH_WATERMARK) {
188
+ setStatus("yellow", "ring", "queue high (" + count + ")");
189
+ return;
190
+ }
191
+
192
+ if (count > 0) {
193
+ setStatus("green", "ring", "queued (" + count + ")");
194
+ return;
195
+ }
196
+
197
+ setStatus("green", "dot", reason || "connected");
198
+ }
199
+
200
+ function buildTelemetryPacket(msg) {
201
+ var payload = msg.payload;
202
+
203
+ if (payload === null || payload === undefined) {
204
+ throw new Error("payload is empty");
205
+ }
206
+
207
+ if (typeof payload === "string") {
208
+ try {
209
+ payload = JSON.parse(payload);
210
+ } catch (err) {
211
+ throw new Error("payload must be a JSON object");
212
+ }
213
+ }
214
+
215
+ if (typeof payload !== "object" || Array.isArray(payload)) {
216
+ throw new Error("payload must be a JSON object");
217
+ }
218
+
219
+ /*
220
+ * Timestamp is created when the message arrives here.
221
+ * This timestamp is kept during store and forward backhaul.
222
+ */
223
+ var ts = Date.now();
224
+
225
+ if (msg.ts) {
226
+ ts = Number(msg.ts);
227
+ } else if (msg.timestamp) {
228
+ ts = Number(msg.timestamp);
229
+ }
230
+
231
+ if (isNaN(ts)) {
232
+ ts = Date.now();
233
+ }
234
+
235
+ return {
236
+ ts: ts,
237
+ values: payload
238
+ };
239
+ }
240
+
241
+ function makeJob(packet) {
242
+ return {
243
+ topic: TELEMETRY_TOPIC,
244
+ qos: MQTT_QOS,
245
+ packet: packet,
246
+ createdAt: Date.now()
247
+ };
248
+ }
249
+
250
+ function publishPacket(packet, callback) {
251
+ if (!node.server || !node.server.connected) {
252
+ callback(new Error("3DM Cloud is not connected"));
253
+ return;
254
+ }
255
+
256
+ node.server.publish(
257
+ TELEMETRY_TOPIC,
258
+ packet,
259
+ { qos: MQTT_QOS },
260
+ callback
261
+ );
262
+ }
263
+
264
+ function isLiveRateLimited() {
265
+ var now = Date.now();
266
+ var elapsed = now - node.lastLivePublishTime;
267
+
268
+ if (elapsed < LIVE_PUBLISH_INTERVAL_MS) {
269
+ return {
270
+ limited: true,
271
+ remainingMs: LIVE_PUBLISH_INTERVAL_MS - elapsed
272
+ };
273
+ }
274
+
275
+ return {
276
+ limited: false,
277
+ remainingMs: 0
278
+ };
279
+ }
280
+
281
+ function canQueueNewMessage() {
282
+ return queueCount() < MAX_QUEUE_MESSAGES;
283
+ }
284
+
285
+ function sendResult(msg, send, done) {
286
+ send(msg);
287
+
288
+ if (done) {
289
+ done();
290
+ }
291
+ }
292
+
293
+ function rejectQueueFull(msg, send, done) {
294
+ msg.sent = false;
295
+ msg.queued = false;
296
+ msg.queueFull = true;
297
+ msg.error = QUEUE_FULL_TEXT;
298
+ msg.topic = "3dm.telemetry";
299
+
300
+ setStatus("red", "ring", QUEUE_FULL_TEXT + " (" + queueCount() + ")");
301
+ warnOncePerMinute(QUEUE_FULL_TEXT + ". Limit is " + MAX_QUEUE_MESSAGES + " packets.");
302
+
303
+ sendResult(msg, send, done);
304
+ }
305
+
306
+ function rejectStoreForwardUnavailable(msg, send, done) {
307
+ msg.sent = false;
308
+ msg.queued = false;
309
+ msg.storeForwardUnavailable = true;
310
+ msg.error = STORE_FORWARD_UNAVAILABLE_TEXT;
311
+ msg.topic = "3dm.telemetry";
312
+
313
+ setStatus("red", "ring", STORE_FORWARD_UNAVAILABLE_TEXT);
314
+ warnOncePerMinute("3DM store and forward unavailable. node-persistent-queue is not installed.");
315
+
316
+ sendResult(msg, send, done);
317
+ }
318
+
319
+ function addJobToQueue(job, reason, msg, send, done) {
320
+ if (!STORE_AND_FORWARD_ENABLED) {
321
+ msg.sent = false;
322
+ msg.queued = false;
323
+ msg.error = "store and forward disabled";
324
+ msg.topic = "3dm.telemetry";
325
+
326
+ sendResult(msg, send, done);
327
+ return;
328
+ }
329
+
330
+ if (node.storeForwardUnavailable || !Queue) {
331
+ rejectStoreForwardUnavailable(msg, send, done);
332
+ return;
333
+ }
334
+
335
+ if (!canQueueNewMessage()) {
336
+ rejectQueueFull(msg, send, done);
337
+ return;
338
+ }
339
+
340
+ if (!queue || !node.queueOpen) {
341
+ initJobs.push({
342
+ job: job,
343
+ reason: reason
344
+ });
345
+
346
+ msg.sent = false;
347
+ msg.queued = true;
348
+ msg.queueReason = reason;
349
+ msg.queueLength = queueCount();
350
+ msg.topic = "3dm.telemetry";
351
+
352
+ setStatus("yellow", "ring", "queue opening (" + queueCount() + ")");
353
+
354
+ sendResult(msg, send, done);
355
+ return;
356
+ }
357
+
358
+ try {
359
+ queue.add(job);
360
+
361
+ msg.sent = false;
362
+ msg.queued = true;
363
+ msg.queueReason = reason;
364
+ msg.queueLength = queueCount();
365
+ msg.topic = "3dm.telemetry";
366
+
367
+ if (reason === "offline") {
368
+ setStatus("yellow", "ring", "storing offline (" + queueCount() + ")");
369
+ } else if (reason === "publish-failed") {
370
+ setStatus("yellow", "ring", "queued after send fail (" + queueCount() + ")");
371
+ } else {
372
+ setStatus("green", "ring", "queued (" + queueCount() + ")");
373
+ }
374
+
375
+ if (queueCount() >= QUEUE_HIGH_WATERMARK) {
376
+ warnOncePerMinute("3DM Cloud queue high: " + queueCount() + " packets.");
377
+ }
378
+
379
+ sendResult(msg, send, done);
380
+
381
+ maybeStartBackhaul();
382
+ } catch (err) {
383
+ msg.sent = false;
384
+ msg.queued = false;
385
+ msg.error = err.message || String(err);
386
+ msg.topic = "3dm.telemetry";
387
+
388
+ setStatus("red", "ring", "queue error");
389
+ node.error("3DM queue add failed: " + msg.error, msg);
390
+
391
+ sendResult(msg, send, done);
392
+ }
393
+ }
394
+
395
+ function maybeStartBackhaul() {
396
+ if (node.storeForwardUnavailable) {
397
+ return;
398
+ }
399
+
400
+ if (!queue || !node.queueOpen) {
401
+ return;
402
+ }
403
+
404
+ if (!node.server || !node.server.connected) {
405
+ return;
406
+ }
407
+
408
+ if (queueIsEmpty()) {
409
+ node.backhauling = false;
410
+ updateStatus("connected");
411
+ return;
412
+ }
413
+
414
+ /*
415
+ * Backhaul ignores the live rate limit.
416
+ */
417
+ try {
418
+ if (!queue.isStarted()) {
419
+ node.backhauling = true;
420
+ updateStatus();
421
+ queue.start();
422
+ }
423
+ } catch (err) {
424
+ warnOncePerMinute("3DM queue backhaul start failed: " + (err.message || err));
425
+ }
426
+ }
427
+
428
+ function stopBackhaul() {
429
+ if (replayTimer) {
430
+ clearTimeout(replayTimer);
431
+ replayTimer = null;
432
+ }
433
+
434
+ if (queue && node.queueOpen) {
435
+ try {
436
+ if (queue.isStarted()) {
437
+ queue.stop();
438
+ }
439
+ } catch (err) {
440
+ /*
441
+ * Ignore stop race.
442
+ */
443
+ }
444
+ }
445
+
446
+ node.backhauling = false;
447
+ updateStatus();
448
+ }
449
+
450
+ function flushInitJobsToQueue() {
451
+ if (!queue || !node.queueOpen || initJobs.length === 0) {
452
+ return;
453
+ }
454
+
455
+ initJobs.forEach(function (item) {
456
+ if (canQueueNewMessage()) {
457
+ try {
458
+ queue.add(item.job);
459
+ } catch (err) {
460
+ warnOncePerMinute("3DM startup queue add failed: " + (err.message || err));
461
+ }
462
+ } else {
463
+ warnOncePerMinute(QUEUE_FULL_TEXT + " while loading startup messages.");
464
+ }
465
+ });
466
+
467
+ initJobs = [];
468
+ }
469
+
470
+ function setupQueue() {
471
+ if (!STORE_AND_FORWARD_ENABLED) {
472
+ updateStatus();
473
+ return;
474
+ }
475
+
476
+ if (!Queue) {
477
+ node.storeForwardUnavailable = true;
478
+ warnOncePerMinute("3DM store and forward unavailable. node-persistent-queue is not installed.");
479
+ updateStatus();
480
+ return;
481
+ }
482
+
483
+ try {
484
+ makeDir(QUEUE_DIR);
485
+ } catch (err) {
486
+ node.storeForwardUnavailable = true;
487
+ setStatus("red", "ring", "queue dir error");
488
+ node.error("3DM queue directory error: " + (err.message || err));
489
+ return;
490
+ }
491
+
492
+ queue = new Queue(queueFile);
493
+
494
+ function onQueueOpen() {
495
+ if (node.queueOpen) {
496
+ return;
497
+ }
498
+
499
+ node.queueOpen = true;
500
+ node.storeForwardUnavailable = false;
501
+
502
+ node.log("3DM Cloud queue opened: " + queueFile);
503
+
504
+ flushInitJobsToQueue();
505
+ updateStatus();
506
+ maybeStartBackhaul();
507
+ }
508
+
509
+ queue.on("open", onQueueOpen);
510
+
511
+ queue.on("next", function (item) {
512
+ var job = item.job || item;
513
+
514
+ if (!job || !job.packet) {
515
+ warnOncePerMinute("3DM queue found invalid stored packet. Removing it.");
516
+
517
+ replayTimer = setTimeout(function () {
518
+ replayTimer = null;
519
+
520
+ try {
521
+ queue.done();
522
+ } catch (doneErr) {
523
+ node.error("3DM queue done failed: " + (doneErr.message || doneErr));
524
+ }
525
+ }, BACKHAUL_REPLAY_GAP_MS);
526
+
527
+ return;
528
+ }
529
+
530
+ if (!node.server || !node.server.connected) {
531
+ stopBackhaul();
532
+ return;
533
+ }
534
+
535
+ node.backhauling = true;
536
+ updateStatus();
537
+
538
+ publishPacket(job.packet, function (err) {
539
+ if (err) {
540
+ /*
541
+ * Do not call queue.done().
542
+ * This keeps the packet stored for the next retry.
543
+ */
544
+ warnOncePerMinute("3DM Cloud backhaul stopped: " + (err.message || err));
545
+ stopBackhaul();
546
+ return;
547
+ }
548
+
549
+ replayTimer = setTimeout(function () {
550
+ replayTimer = null;
551
+
552
+ try {
553
+ queue.done();
554
+ } catch (doneErr) {
555
+ node.error("3DM queue done failed: " + (doneErr.message || doneErr));
556
+ }
557
+
558
+ updateStatus("backhauled");
559
+ }, BACKHAUL_REPLAY_GAP_MS);
560
+ });
561
+ });
562
+
563
+ queue.on("start", function () {
564
+ node.backhauling = true;
565
+ updateStatus();
566
+ });
567
+
568
+ queue.on("stop", function () {
569
+ node.backhauling = false;
570
+ updateStatus();
571
+ });
572
+
573
+ queue.on("empty", function () {
574
+ node.backhauling = false;
575
+ updateStatus("connected");
576
+ });
577
+
578
+ queue.on("add", function () {
579
+ updateStatus();
580
+ });
581
+
582
+ queue
583
+ .open()
584
+ .then(function () {
585
+ onQueueOpen();
586
+ })
587
+ .catch(function (err) {
588
+ node.storeForwardUnavailable = true;
589
+ setStatus("red", "ring", "queue open failed");
590
+ node.error("3DM queue failed to open " + queueFile + ": " + (err.message || err));
591
+ });
592
+ }
593
+
594
+ var stateHandler = function (connected) {
595
+ if (connected) {
596
+ updateStatus("connected");
597
+ maybeStartBackhaul();
598
+ } else {
599
+ stopBackhaul();
600
+ updateStatus();
601
+ }
602
+ };
603
+
604
+ node.server.on("state", stateHandler);
605
+
606
+ setupQueue();
607
+ updateStatus();
608
+
609
+ node.on("input", function (msg, send, done) {
610
+ send = send || function () {
611
+ node.send.apply(node, arguments);
612
+ };
613
+
614
+ try {
615
+ var originalPayload = msg.payload;
616
+ var packet = buildTelemetryPacket(msg);
617
+ var job = makeJob(packet);
618
+
619
+ /*
620
+ * Offline:
621
+ * Store timestamped packet.
622
+ */
623
+ if (!node.server || !node.server.connected) {
624
+ msg.payload = originalPayload;
625
+ addJobToQueue(job, "offline", msg, send, done);
626
+ return;
627
+ }
628
+
629
+ /*
630
+ * If queue has data, queue new data too.
631
+ * Backhaul ignores live throttle and runs at 4Hz.
632
+ */
633
+ if (!queueIsEmpty() || node.backhauling) {
634
+ msg.payload = originalPayload;
635
+ addJobToQueue(job, "queue-not-empty", msg, send, done);
636
+ return;
637
+ }
638
+
639
+ /*
640
+ * Live online publishing only.
641
+ * This is limited to 1 update per minute.
642
+ */
643
+ var limit = isLiveRateLimited();
644
+
645
+ if (limit.limited) {
646
+ setStatus("yellow", "ring", RATE_LIMIT_TEXT);
647
+
648
+ msg.payload = originalPayload;
649
+ msg.sent = false;
650
+ msg.queued = false;
651
+ msg.rateLimited = true;
652
+ msg.rateLimitRemainingMs = limit.remainingMs;
653
+ msg.error = RATE_LIMIT_TEXT;
654
+ msg.topic = "3dm.telemetry";
655
+
656
+ warnOncePerMinute(RATE_LIMIT_TEXT);
657
+
658
+ sendResult(msg, send, done);
659
+ return;
660
+ }
661
+
662
+ node.lastLivePublishTime = Date.now();
663
+
664
+ publishPacket(packet, function (err) {
665
+ if (err) {
666
+ msg.payload = originalPayload;
667
+ addJobToQueue(job, "publish-failed", msg, send, done);
668
+ return;
669
+ }
670
+
671
+ setStatus("green", "dot", "sent");
672
+
673
+ msg.payload = originalPayload;
674
+ msg.sent = true;
675
+ msg.queued = false;
676
+ msg.rateLimited = false;
677
+ msg.topic = "3dm.telemetry";
678
+
679
+ sendResult(msg, send, done);
680
+ });
681
+ } catch (err) {
682
+ setStatus("red", "ring", BAD_PAYLOAD_TEXT);
683
+
684
+ msg.sent = false;
685
+ msg.queued = false;
686
+ msg.rateLimited = false;
687
+ msg.error = err.message || String(err);
688
+ msg.topic = "3dm.telemetry";
689
+
690
+ warnOncePerMinute("3DM Cloud telemetry bad payload: " + msg.error);
691
+
692
+ sendResult(msg, send, done);
693
+ }
694
+ });
695
+
696
+ node.on("close", function (removed, done) {
697
+ if (typeof removed === "function") {
698
+ done = removed;
699
+ }
700
+
701
+ done = done || function () {};
702
+
703
+ if (node.server) {
704
+ node.server.removeListener("state", stateHandler);
705
+ }
706
+
707
+ if (replayTimer) {
708
+ clearTimeout(replayTimer);
709
+ replayTimer = null;
710
+ }
711
+
712
+ if (queue && node.queueOpen) {
713
+ try {
714
+ if (queue.isStarted()) {
715
+ queue.stop();
716
+ }
717
+
718
+ queue
719
+ .close()
720
+ .then(function () {
721
+ done();
722
+ })
723
+ .catch(function (err) {
724
+ done(err);
725
+ });
726
+ } catch (err) {
727
+ done(err);
728
+ }
729
+ } else {
730
+ done();
731
+ }
732
+ });
733
+ }
734
+
735
+ RED.nodes.registerType("3dm-cloud-out", ThreeDMCloudOutNode);
736
+ };