@robotical/raftjs 2.0.11 → 2.1.2

package/src/RaftDeviceMgrIF.ts

@@ -7,14 +7,16 @@
 //
 /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-import { DeviceTypeAction } from "./RaftDeviceInfo";
-import { DeviceAttributeState, DevicesState, DeviceState } from "./RaftDeviceStates";
+import { DeviceTypeAction, SampleRateResult } from "./RaftDeviceInfo";
+import { DeviceAttributeState, DevicesState, DeviceState, DeviceStats } from "./RaftDeviceStates";
 
 export default interface RaftDeviceMgrIF {
 
     // Get state of devices
     getDevicesState(): DevicesState;
     getDeviceState(deviceKey: string): DeviceState;
+    getDeviceStats(deviceKey: string): DeviceStats;
+    resetDeviceStats(deviceKey: string): void;
 
     // Settings
     setMaxDataPointsToStore(maxDataPointsToStore: number): void;
@@ -26,8 +28,17 @@ export default interface RaftDeviceMgrIF {
     removeNewAttributeCallback(callback: (deviceKey: string, attrState: DeviceAttributeState) => void): void;
     addAttributeDataCallback(callback: (deviceKey: string, attrState: DeviceAttributeState) => void): void;
     removeAttributeDataCallback(callback: (deviceKey: string, attrState: DeviceAttributeState) => void): void;
+    addDeviceRemovedCallback(callback: (deviceKey: string, state: DeviceState) => void): void;
+    removeDeviceRemovedCallback(callback: (deviceKey: string, state: DeviceState) => void): void;
 
     // Send action to device
     sendAction(deviceKey: string, action: DeviceTypeAction, data: number[]): void;
     sendCompoundAction(deviceKey: string, action: DeviceTypeAction, data: number[][]): void;
+
+    // Set sample rate with coordinated polling parameters
+    setSampleRate(deviceKey: string, sampleRateHz: number, options?: {
+        numSamples?: number;
+        intervalUs?: number;
+        maxNumSamples?: number;
+    }): Promise<SampleRateResult>;
 }

package/src/RaftDeviceStates.ts

@@ -14,10 +14,14 @@ export function deviceAttrGetLatestFormatted(attrState: DeviceAttributeState): s
     if (attrState.values.length === 0) {
         return 'N/A';
     }
+    const value = attrState.values[attrState.values.length - 1];
+    // String values are returned directly
+    if (typeof value === 'string') {
+        return value;
+    }
     if (attrState.format.length === 0) {
-        return attrState.values[attrState.values.length - 1].toString();
+        return value.toString();
     }
-    const value = attrState.values[attrState.values.length - 1];
     let format = attrState.format;
     if (format.startsWith("%")) {
         format = format.slice(1);
@@ -52,7 +56,7 @@ export interface DeviceAttributeState {
     newAttribute: boolean;
     newData: boolean;
     numNewValues: number;
-    values: number[];
+    values: (number | string)[];
     units: string;
     range: number[];
     format: string;
@@ -68,15 +72,37 @@ export interface DeviceTimeline {
     timestampsUs: number[];
     lastReportTimestampUs: number;
     reportTimestampOffsetUs: number;
+    totalSamplesAdded: number;
+    // Piecewise EMA timestamp reconstruction state
+    emaLastSampleTimeUs: number;
+    emaIntervalUs: number;
+    emaPrevPollTimeUs: number;
+    emaCalibrated: boolean;
+    emaCalibrationPolls: number;
+}
+
+export interface DeviceStats {
+    totalSamples: number;
+    windowMs: number;
+    windowSamples: number;
+    sampleRateHz: number;
+    lastSampleTimeMs: number | null;
+    lastUpdateTimeMs: number | null;
+}
+
+export enum DeviceOnlineState {
+    Offline = 0,
+    Online = 1,
+    PendingDeletion = 2,
 }
-    
+
 export interface DeviceState {
     deviceTypeInfo: DeviceTypeInfo | undefined;
     deviceTimeline: DeviceTimeline;
     deviceAttributes: DeviceAttributesState;
     deviceIsNew: boolean;
     stateChanged: boolean;
-    isOnline: boolean;
+    onlineState: DeviceOnlineState;
     deviceAddress: string;
     deviceType: string;
     busName: string;
@@ -86,7 +112,22 @@ export class DevicesState {
     [deviceKey: string]: DeviceState;
 }
 
-// Add the getDeviceKey method to generate a composite key
-export function getDeviceKey(busName: string, devAddr: string, devType: string): string {
-    return `${busName}_${devAddr}_${devType}`;
+// Format a numeric device address as canonical hex string:
+// lowercase, no "0x" prefix, no leading zeros
+export function formatDeviceAddrHex(addr: number): string {
+    return addr.toString(16);
+}
+
+// Generate a composite device key from bus number and hex address
+export function getDeviceKey(busNumberAsString: string, devAddrHexStr: string): string {
+    return `${busNumberAsString}_${devAddrHexStr}`;
+}
+
+// Parse a device key into its bus and address components
+export function parseDeviceKey(deviceKey: string): { bus: string; addr: string } {
+    const sep = deviceKey.indexOf('_');
+    if (sep < 0) {
+        return { bus: deviceKey, addr: '' };
+    }
+    return { bus: deviceKey.substring(0, sep), addr: deviceKey.substring(sep + 1) };
 }

package/src/RaftFileHandler.ts

@@ -13,6 +13,7 @@ import RaftMsgHandler from './RaftMsgHandler';
 import {
   RaftFileDownloadResult,
   RaftFileDownloadStartResp,
+  RaftFileDownloadEndResp,
   RaftFileSendType,
   RaftFileStartResp,
   RaftOKFail,
@@ -77,7 +78,6 @@ export default class RaftFileHandler {
   private _fileRxLastAckTime = 0;
   private _fileRxLastBlockTime = 0;
   private _fileRxLastAckPos = 0;
-  private OVERALL_FILE_TRANSFER_TIMEOUT_MS = 100000;
   private FILE_RX_ACK_RESEND_TIMEOUT_MS = 1000;
 
   // RaftCommsStats
@@ -454,8 +454,9 @@ export default class RaftFileHandler {
     // Send contents
     const fileContents = await this._receiveFileContents(progressCallback, bridgeID);
 
-    // Send file end
-    await this._receiveFileEnd(fileName, bridgeID);
+    // Send file end (and verify deferred CRC if applicable)
+    if (!await this._receiveFileEnd(fileName, bridgeID))
+      return new RaftFileDownloadResult();
 
     // Clean up
     await this.awaitOutstandingMsgPromises(true);
@@ -471,14 +472,16 @@ export default class RaftFileHandler {
   async _receiveFileStart(fileName: string, bridgeID: number | undefined) : Promise<boolean> {
 
     const blockMaxSizeRequested = 5000;
-    const batchAckSizeRequested = 10;
+    const batchAckSizeRequested = 4;
     const fileSrc = "fs";
 
     // Request file transfer
     // Frames follow the approach used in the web interface start, block..., end
+    // crcAt:end requests the server to defer CRC computation (non-blocking start)
     const cmdMsg = `{"cmdName":"dfStart","reqStr":"getFile","fileType":"${fileSrc}",` +
                     `"batchMsgSize":${blockMaxSizeRequested},` +
                     `"batchAckSize":${batchAckSizeRequested},` +
+                    `"crcAt":"end",` +
                     `"fileName":"${fileName}"}`
 
     // Send
@@ -499,7 +502,8 @@ export default class RaftFileHandler {
       this._fileRxBatchAckSize = cmdResp.batchAckSize;
       this._fileRxStreamID = cmdResp.streamID;
       this._fileRxFileLen = cmdResp.fileLen;
-      this._fileRxCrc16 = parseInt(cmdResp.crc16, 16);
+      // CRC may be provided at start (old firmware) or deferred to dfEnd (new firmware)
+      this._fileRxCrc16 = cmdResp.crc16 ? parseInt(cmdResp.crc16, 16) : -1;
       this._fileRxBuffer = new Uint8Array(0);
       this._fileRxLastAckTime = 0;
       this._fileRxLastAckPos = 0;
@@ -514,6 +518,18 @@ export default class RaftFileHandler {
         bridgeID: number | undefined
   ): Promise<RaftFileDownloadResult> {
 
+    // Calculate dynamic timeouts based on file size and connection speed
+    // Use batch message size as proxy for connection type:
+    //   BLE ~500 byte blocks ~5 KB/s, WebSocket ~5000 byte blocks ~50 KB/s
+    const estimatedBytesPerSec = this._fileRxBatchMsgSize <= 500 ? 5000 : 50000;
+    const BASE_TIMEOUT_MS = 30000;
+    const SAFETY_FACTOR = 2;
+    const transferTimeMs = (this._fileRxFileLen / estimatedBytesPerSec) * 1000 * SAFETY_FACTOR;
+    const overallTimeoutMs = BASE_TIMEOUT_MS + transferTimeMs;
+    const blockTimeoutMs = this.BLOCK_ACK_TIMEOUT_MS;
+    RaftLog.info(`_receiveFileContents - fileLen ${this._fileRxFileLen} batchMsgSize ${this._fileRxBatchMsgSize} ` +
+        `estimatedBytesPerSec ${estimatedBytesPerSec} overallTimeoutMs ${overallTimeoutMs} blockTimeoutMs ${blockTimeoutMs}`);
+
     // Wait for file to be received
     return new Promise<RaftFileDownloadResult>((resolve, reject) => {
       const startTime = Date.now();
@@ -527,14 +543,16 @@ export default class RaftFileHandler {
             progressCallback(this._fileRxBuffer.length, this._fileRxFileLen);
           }
 
-          // Check CRC
-          const crc16 = RaftMiniHDLC.crc16(this._fileRxBuffer);
-          if (crc16 !== this._fileRxCrc16) {
-            RaftLog.warn(`_receiveFileContents - CRC error ${crc16} ${this._fileRxCrc16}`);
-            reject(new Error('fileReceive CRC error'));
-            return;
-          } else {
-            RaftLog.info(`_receiveFileContents - CRC OK ${crc16} ${this._fileRxCrc16}`);
+          // Check CRC if it was provided at start (old firmware)
+          if (this._fileRxCrc16 >= 0) {
+            const crc16 = RaftMiniHDLC.crc16(this._fileRxBuffer);
+            if (crc16 !== this._fileRxCrc16) {
+              RaftLog.warn(`_receiveFileContents - CRC error ${crc16} ${this._fileRxCrc16}`);
+              reject(new Error('fileReceive CRC error'));
+              return;
+            } else {
+              RaftLog.info(`_receiveFileContents - CRC OK ${crc16} ${this._fileRxCrc16}`);
+            }
           }
           resolve(new RaftFileDownloadResult(this._fileRxBuffer));
           return;
@@ -556,9 +574,11 @@ export default class RaftFileHandler {
         const now = Date.now();
 
         // Check for overall timeouts
-        if ((now - startTime > this.OVERALL_FILE_TRANSFER_TIMEOUT_MS) || 
-            (now - this._fileRxLastBlockTime > this.BLOCK_ACK_TIMEOUT_MS)) {
-          RaftLog.warn(`_receiveFileContents - time-out no new data received`);
+        if ((now - startTime > overallTimeoutMs) ||
+            (now - this._fileRxLastBlockTime > blockTimeoutMs)) {
+          RaftLog.warn(`_receiveFileContents - time-out no new data received ` +
+              `elapsed ${now - startTime}ms overallTimeout ${overallTimeoutMs}ms ` +
+              `blockGap ${now - this._fileRxLastBlockTime}ms blockTimeout ${blockTimeoutMs}ms`);
           this._fileRxActive = false;
           reject(new Error('fileReceive failed'));
           return;
@@ -574,7 +594,7 @@ export default class RaftFileHandler {
         if (this._fileRxBuffer.length - this._fileRxLastAckPos >= this._fileRxBatchAckSize * this._fileRxBatchMsgSize) {
           ackRequired = true;
         }
-        // RaftLog.info(`_receiveFileContents ${ackRequired ? "ACK_REQUIRED" : "ACK_NOTREQUIRED"}  ${this._fileRxBuffer.length} ${this._fileRxLastAckPos} ${this._fileRxBatchAckSize} ${this._fileRxBatchMsgSize}`); 
+        // RaftLog.info(`_receiveFileContents ${ackRequired ? "ACK_REQUIRED" : "ACK_NOTREQUIRED"}  ${this._fileRxBuffer.length} ${this._fileRxLastAckPos} ${this._fileRxBatchAckSize} ${this._fileRxBatchMsgSize}`);
 
         // Check if ack required
         if (ackRequired) {
@@ -582,7 +602,7 @@ export default class RaftFileHandler {
           // Ack timing
           this._fileRxLastAckTime = Date.now();
           this._fileRxLastAckPos = this._fileRxBuffer.length;
-    
+
           // Okto message
           const cmdMsg = `{"cmdName":"dfAck","okto":${this._fileRxBuffer.length},` +
                         `"streamID":${this._fileRxStreamID},"rslt":"ok"}`;
@@ -612,18 +632,39 @@ export default class RaftFileHandler {
 
   async _receiveFileEnd(fileName: string, bridgeID: number | undefined): Promise<boolean> {
 
-    // Send file end message
-    const cmdMsg = `{"cmdName":"dfAck","reqStr":"getFile","okto":${this._fileRxBuffer.length},` +
-          `"fileName":"${fileName}","streamID":${this._fileRxStreamID},"rslt":"ok"}`
-    this._msgHandler.sendRICRESTNoResp(
-      cmdMsg,
-      RICRESTElemCode.RICREST_ELEM_CODE_COMMAND_FRAME,
-      bridgeID
-    );
+    // Send dfEnd and wait for response (may contain deferred CRC)
+    const cmdMsg = `{"cmdName":"dfEnd","reqStr":"getFile",` +
+          `"fileName":"${fileName}","fileLen":${this._fileRxBuffer.length},` +
+          `"streamID":${this._fileRxStreamID}}`
+
+    let cmdResp = null;
+    try {
+      cmdResp = await this._msgHandler.sendRICREST<RaftFileDownloadEndResp>(
+        cmdMsg,
+        RICRESTElemCode.RICREST_ELEM_CODE_COMMAND_FRAME,
+        bridgeID,
+      );
+    } catch (err) {
+      RaftLog.warn(`_receiveFileEnd error ${err}`);
+      this._fileRxActive = false;
+      return false;
+    }
+
+    // Check deferred CRC if start response didn't include one
+    if (this._fileRxCrc16 < 0 && cmdResp.crc16) {
+      const expectedCrc = parseInt(cmdResp.crc16, 16);
+      const actualCrc = RaftMiniHDLC.crc16(this._fileRxBuffer);
+      if (actualCrc !== expectedCrc) {
+        RaftLog.warn(`_receiveFileEnd - CRC error actual=${actualCrc} expected=${expectedCrc}`);
+        this._fileRxActive = false;
+        return false;
+      }
+      RaftLog.info(`_receiveFileEnd - CRC OK ${actualCrc}`);
+    }
 
     // No longer active
     this._fileRxActive = false;
-    return false;
+    return true;
   }
 
   async _sendFileRxCancelMsg(bridgeID: number | undefined): Promise<void> {
@@ -665,4 +706,4 @@ export default class RaftFileHandler {
   isFileRxActive(): boolean {
     return this._fileRxActive;
   }
-}
+}

package/src/RaftPublish.ts

@@ -0,0 +1,92 @@
+/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+//
+// RaftSystem
+// Part of RaftJS
+//
+// Rob Dobson 2025
+// (C) 2020-2025 All rights reserved
+//
+/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+import { RaftPublishFrameMeta } from "./RaftTypes";
+
+const RAFT_PUBLISH_PREFIX_LEN = 2;
+const DEVBIN_MAGIC_MIN = 0xDB;
+const DEVBIN_MAGIC_MAX = 0xDF;
+const DEVBIN_VERSION_BASE = 0xDA;
+
+export function inspectPublishFrame(
+  payload: Uint8Array,
+  topicLookup?: (topicIndex: number) => string | undefined,
+): RaftPublishFrameMeta {
+
+  if (payload.length < RAFT_PUBLISH_PREFIX_LEN) {
+    return { frameType: "unknown" };
+  }
+
+  const payloadStartPos = RAFT_PUBLISH_PREFIX_LEN;
+  if (payload.length <= payloadStartPos) {
+    return { frameType: "unknown" };
+  }
+
+  // Try JSON first
+  if (payload[payloadStartPos] === 0x7B) {
+    try {
+      const jsonString = new TextDecoder("utf-8").decode(payload.slice(payloadStartPos));
+      const jsonObj = JSON.parse(jsonString) as Record<string, unknown>;
+
+      let topicIndex: number | undefined = undefined;
+      let topicName: string | undefined = undefined;
+      if (typeof jsonObj._t === "number") {
+        topicIndex = jsonObj._t;
+        topicName = topicLookup ? topicLookup(topicIndex) : undefined;
+      } else if (typeof jsonObj._t === "string") {
+        topicName = jsonObj._t;
+      }
+
+      const version = typeof jsonObj._v === "number" ? jsonObj._v : undefined;
+
+      return {
+        frameType: "json",
+        topicIndex,
+        topicName,
+        version,
+        jsonString,
+      };
+    } catch {
+      return { frameType: "unknown" };
+    }
+  }
+
+  // Binary (devbin legacy or enveloped)
+  const firstBinaryByte = payload[payloadStartPos];
+  if ((firstBinaryByte & 0xF0) === 0xD0) {
+    if (firstBinaryByte < DEVBIN_MAGIC_MIN || firstBinaryByte > DEVBIN_MAGIC_MAX) {
+      return {
+        frameType: "binary",
+        binaryHasEnvelope: true,
+      };
+    }
+
+    const topicIndex = payload.length > payloadStartPos + 1 ? payload[payloadStartPos + 1] : undefined;
+    const topicName = (topicIndex !== undefined && topicIndex !== 0xFF && topicLookup)
+      ? topicLookup(topicIndex)
+      : undefined;
+
+    return {
+      frameType: "binary",
+      topicIndex,
+      topicName,
+      version: firstBinaryByte - DEVBIN_VERSION_BASE,
+      binaryHasEnvelope: true,
+      binaryPayloadOffset: payloadStartPos + 2,
+    };
+  }
+
+  // Legacy binary format (no envelope)
+  return {
+    frameType: "binary",
+    binaryHasEnvelope: false,
+    binaryPayloadOffset: payloadStartPos,
+  };
+}

package/src/RaftStreamHandler.ts

@@ -11,7 +11,7 @@
 import RaftLog from './RaftLog'
 import RaftMsgHandler from './RaftMsgHandler';
 import RaftCommsStats from './RaftCommsStats';
-import { RaftOKFail, RaftStreamStartResp, RaftStreamType } from './RaftTypes';
+import { RaftOKFail, RaftStreamStartResp, RaftStreamType, RaftStreamDataProgressCBType } from './RaftTypes';
 import RaftConnector from './RaftConnector';
 import { RaftConnEvent } from './RaftConnEvents';
 import { RICRESTElemCode } from './RaftProtocolDefs'
@@ -77,6 +77,89 @@ export default class RaftStreamHandler {
     this._legacySoktoMode = legacyMode;
   }
 
+  /**
+   * streamData - stream arbitrary data to a named firmware endpoint using the RT_STREAM protocol.
+   * Returns a promise that resolves when the stream is complete or rejects on failure.
+   * @param streamContents data to stream
+   * @param fileName logical filename sent in ufStart (used by firmware to detect format, e.g. "pattern.thr")
+   * @param targetEndpoint REST API endpoint name registered on the firmware (e.g. "streampattern")
+   * @param progressCallback optional callback reporting (bytesSent, totalBytes, progressFraction)
+   * @returns Promise<boolean> true if stream completed successfully
+   */
+  async streamData(
+    streamContents: Uint8Array,
+    fileName: string,
+    targetEndpoint: string,
+    progressCallback?: RaftStreamDataProgressCBType,
+  ): Promise<boolean> {
+    // Reject if another stream is starting
+    if (this._streamIsStarting || this._lastStreamStartTime > (Date.now() - 500)) {
+      RaftLog.warn(`streamData: unable to start, too soon since last request`);
+      return false;
+    }
+
+    this._streamIsStarting = true;
+    this._lastStreamStartTime = Date.now();
+    this._soktoReceived = false;
+    this._soktoPos = 0;
+    this._streamPos = 0;
+    this._streamBuffer = RaftUtils.toArrayBufferView(streamContents);
+
+    // Send ufStart
+    const startOk = await this._sendStreamStartMsg(
+      fileName, targetEndpoint, RaftStreamType.REAL_TIME_STREAM, streamContents,
+    );
+    this._streamIsStarting = false;
+
+    if (!startOk) {
+      RaftLog.warn(`streamData: ufStart failed`);
+      return false;
+    }
+
+    // Send blocks
+    if (this._streamID === null) {
+      return false;
+    }
+
+    let pos = 0;
+    while (pos < this._streamBuffer.length) {
+      // Respect SOKTO feedback — if firmware reported a position mismatch, rewind
+      if (this._soktoReceived) {
+        RaftLog.verbose(`streamData: sokto received, pos was ${pos}, soktoPos ${this._soktoPos}`);
+        this._soktoReceived = false;
+        // Slow down on backpressure
+        await new Promise(r => setTimeout(r, 50));
+      }
+
+      const blockSize = Math.min(this._streamBuffer.length - pos, this._maxBlockSize);
+      const block = this._streamBuffer.slice(pos, pos + blockSize);
+      if (block.length > 0) {
+        const sentOk = await this._msgHandler.sendStreamBlock(block, pos, this._streamID);
+        this._commsStats.recordStreamBytes(block.length);
+        if (!sentOk) {
+          RaftLog.warn(`streamData: sendStreamBlock failed at pos ${pos}`);
+          return false;
+        }
+        pos += blockSize;
+        this._streamPos = pos;
+
+        // Progress callback
+        if (progressCallback) {
+          const progress = this._streamBuffer.length > 0 ? pos / this._streamBuffer.length : 1;
+          progressCallback(pos, this._streamBuffer.length, progress);
+        }
+      }
+
+      // Yield to avoid hogging the event loop
+      await new Promise(r => setTimeout(r, 1));
+    }
+
+    // Send ufEnd
+    const endOk = await this._sendStreamEndMsg(this._streamID);
+    RaftLog.debug(`streamData: complete, endOk=${endOk}`);
+    return endOk;
+  }
+
   // Start streaming audio
   streamAudio(streamContents: Uint8Array, clearExisting: boolean, audioDuration: number): void {
     if (!clearExisting)