@xube/kit-aws-data 0.0.73

package/dist/decode/aggregated.d.ts

@@ -0,0 +1,4 @@
+/// <reference types="node" />
+import { ReadingV1, AggregateMetadataTypes, AggregateMetadataV1 } from "@xube/kit-aws-data-schema";
+export declare const decodeAggregateData: (aggregate: AggregateMetadataTypes, data: Buffer, device: string, component: string) => ReadingV1[];
+export declare const decodeAggregateDataV1: (aggregate: AggregateMetadataV1, data: Buffer, device: string, component: string) => ReadingV1[];

package/dist/decode/aggregated.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.decodeAggregateDataV1 = exports.decodeAggregateData = void 0;
+const kit_aws_data_schema_1 = require("@xube/kit-aws-data-schema");
+const decodeAggregateData = (aggregate, data, device, component) => {
+    if (aggregate.v === kit_aws_data_schema_1.AggregateMetaDataV1Version) {
+        return (0, exports.decodeAggregateDataV1)(aggregate, data, device, component);
+    }
+    throw new Error(`Unknown aggregate metadata schema version: ${aggregate.v}`);
+};
+exports.decodeAggregateData = decodeAggregateData;
+const decodeAggregateDataV1 = (aggregate, data, device, component) => {
+    let currentPosition = 0;
+    return aggregate.m.map((readingMetadata, idx) => {
+        let length = readingMetadata.s ? readingMetadata.s : aggregate.s;
+        const readingData = data.subarray(currentPosition, currentPosition + length);
+        currentPosition += length;
+        return {
+            ...readingMetadata.m,
+            data: readingData.toString("ascii"),
+            id: device,
+            type: kit_aws_data_schema_1.DATA_TYPE,
+            component,
+        };
+    });
+};
+exports.decodeAggregateDataV1 = decodeAggregateDataV1;

package/dist/get/get-data.d.ts

@@ -0,0 +1,4 @@
+import { GetDataByDateRangeRequest, GetDataByDateRangeResponse } from "@xube/kit-aws-data-schema";
+import { XubeLog } from "@xube/kit-log";
+import { XubeResponse } from "@xube/kit-request";
+export declare const getDataForDateRange: (request: GetDataByDateRangeRequest, tableName: string, log?: XubeLog) => Promise<XubeResponse<GetDataByDateRangeResponse>>;

package/dist/get/get-data.js

@@ -0,0 +1,76 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getDataForDateRange = void 0;
+const kit_aws_1 = require("@xube/kit-aws");
+const kit_constants_1 = require("@xube/kit-constants");
+const kit_aws_data_schema_1 = require("@xube/kit-aws-data-schema");
+const kit_log_1 = require("@xube/kit-log");
+const kit_request_1 = require("@xube/kit-request");
+const fetchItemsFromDB = async (request, tableName) => {
+    const queryItemsPromises = [];
+    for (const device of request.devices) {
+        for (const component of device.components) {
+            queryItemsPromises.push((0, kit_aws_1.queryItemsFromTableBetween)(tableName, {
+                PK: (0, kit_aws_data_schema_1.getDataPartitionKey)(device.id),
+                SKstart: (0, kit_aws_data_schema_1.getDataSortKey)(component, `${new Date(request.dateRange.start).getTime() / 1000}`),
+                SKend: (0, kit_aws_data_schema_1.getDataSortKey)(component, `${new Date(request.dateRange.end).getTime() / 1000}`),
+            }));
+        }
+    }
+    return Promise.all(queryItemsPromises);
+};
+const processResponseItems = (response, log) => {
+    if (response.hasFailed() || !response.data) {
+        log.warn(`Failed to get data for date range: ${JSON.stringify(response)}`);
+        return { errorCount: 1 };
+    }
+    const getDataResponse = { devices: [] };
+    for (const item of response.data) {
+        if (!(0, kit_aws_data_schema_1.isReadingV1)(item)) {
+            log.warn(`Found item that is not a reading v1: ${JSON.stringify(item)}`);
+            return { errorCount: 1 };
+        }
+        let device = getDataResponse.devices.find((device) => device.id === item.id);
+        if (!device) {
+            device = { id: item.id, components: [] };
+            getDataResponse.devices.push(device);
+        }
+        let component = device.components.find((component) => component.id === item.component);
+        if (!component) {
+            component = { id: item.component, data: [] };
+            device.components.push(component);
+        }
+        component.data.push({
+            seconds: item.s,
+            microseconds: item.us,
+            readings: item.data,
+        });
+    }
+    return { data: getDataResponse, errorCount: 0 };
+};
+const getDataForDateRange = async (request, tableName, log = kit_log_1.XubeLog.getInstance()) => {
+    const queryItemsResponses = await fetchItemsFromDB(request, tableName);
+    let totalErrorCount = 0;
+    const getDataResponse = { devices: [] };
+    for (const response of queryItemsResponses) {
+        const { data, errorCount } = processResponseItems(response, log);
+        totalErrorCount += errorCount;
+        if (data) {
+            getDataResponse.devices.push(...data.devices);
+        }
+    }
+    if (totalErrorCount === queryItemsResponses.length) {
+        return new kit_request_1.XubeResponse({
+            statusCode: kit_constants_1.StatusCode.NotFound,
+            error: `Failed to get data for date range. No data found.`,
+        });
+    }
+    else if (totalErrorCount > 0) {
+        return new kit_request_1.XubeResponse({
+            statusCode: kit_constants_1.StatusCode.PartialSuccess,
+            data: getDataResponse,
+        });
+    }
+    return new kit_request_1.XubeResponse({ data: getDataResponse, statusCode: kit_constants_1.StatusCode.OK });
+};
+exports.getDataForDateRange = getDataForDateRange;

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export * from "./decode/aggregated";
+export * from "./get/get-data";
+export * from "./ingest/stage";

package/dist/index.js

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./decode/aggregated"), exports);
+__exportStar(require("./get/get-data"), exports);
+__exportStar(require("./ingest/stage"), exports);

package/dist/ingest/stage.d.ts

@@ -0,0 +1,3 @@
+/// <reference types="node" />
+import { ReadingV1 } from "@xube/kit-aws-data-schema";
+export declare const parseDeviceData: (payload: Buffer, device: string, component: string) => ReadingV1[];

package/dist/ingest/stage.js

@@ -0,0 +1,21 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseDeviceData = void 0;
+const kit_aws_data_schema_1 = require("@xube/kit-aws-data-schema");
+const aggregated_1 = require("../decode/aggregated");
+const parseDeviceData = (payload, device, component) => {
+    const splitIdx = payload.findIndex((value, idx, obj) => {
+        return value === 0x00;
+    });
+    if (splitIdx === -1) {
+        throw new Error("Invalid data format");
+    }
+    const metadataString = payload.subarray(0, splitIdx).toString();
+    const metadata = JSON.parse(metadataString);
+    if (!(0, kit_aws_data_schema_1.isAggregateMetadataTypes)(metadata)) {
+        throw new Error("Invalid metadata format");
+    }
+    const data = payload.subarray(splitIdx + 1);
+    return (0, aggregated_1.decodeAggregateData)(metadata, data, device, component);
+};
+exports.parseDeviceData = parseDeviceData;

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@xube/kit-aws-data",
+  "version": "0.0.73",
+  "description": "",
+  "main": "dist/index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@github.com/XubeLtd/dev-kit.git"
+  },
+  "author": "Xube Pty Ltd",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/XubeLtd/dev-kit/issues"
+  },
+  "homepage": "https://github.com/XubeLtd/dev-kit#readme",
+  "dependencies": {
+    "@types/uuid": "^9.0.5",
+    "@xube/kit-aws": "^0.0.73",
+    "@xube/kit-aws-data-schema": "^0.0.73",
+    "@xube/kit-aws-schema": "^0.0.73",
+    "@xube/kit-generator": "^0.0.73",
+    "@xube/kit-log": "^0.0.73",
+    "@xube/kit-schema": "^0.0.73",
+    "uuid": "^9.0.1"
+  },
+  "devDependencies": {
+    "@xube/kit-build": "^0.0.73"
+  }
+}

package/src/decode/aggregated.test.ts

@@ -0,0 +1,81 @@
+import { decodeAggregateData } from "./aggregated";
+import {
+  AggregateMetadataV1,
+  DATA_TYPE,
+  ReadingV1,
+} from "@xube/kit-aws-data-schema";
+
+describe("decodeAggregateData", () => {
+  const aggregate: AggregateMetadataV1 = {
+    s: 25,
+    m: [
+      { m: { s: 1697578841, us: 214981 } },
+      { m: { s: 1697578841, us: 413572 } },
+      { m: { s: 1697578841, us: 612976 } },
+      { m: { s: 1697578841, us: 813754 } },
+      { m: { s: 1697578842, us: 13019 } },
+    ],
+    v: 1,
+  };
+
+  const data: Buffer = Buffer.from(
+    `S,+000.30,+001.74,M,00,33S,+000.35,+001.84,M,00,39S,+000.26,+002.00,M,00,34S,+000.34,+001.88,M,00,34S,+000.35,+001.82,M,00,3F`
+  );
+
+  it("should decode the aggregate data correctly", () => {
+    const result = decodeAggregateData(
+      aggregate,
+      data,
+      "12345678",
+      "component"
+    );
+
+    const expectedReading: ReadingV1[] = [
+      {
+        data: "S,+000.30,+001.74,M,00,33",
+        s: 1697578841,
+        us: 214981,
+        type: DATA_TYPE,
+        id: "12345678",
+        component: "component",
+      },
+      {
+        data: "S,+000.35,+001.84,M,00,39",
+        s: 1697578841,
+        us: 413572,
+        type: DATA_TYPE,
+        id: "12345678",
+        component: "component",
+      },
+      {
+        data: "S,+000.26,+002.00,M,00,34",
+        s: 1697578841,
+        us: 612976,
+        type: DATA_TYPE,
+        id: "12345678",
+        component: "component",
+      },
+      {
+        data: "S,+000.34,+001.88,M,00,34",
+        s: 1697578841,
+        us: 813754,
+        type: DATA_TYPE,
+        id: "12345678",
+        component: "component",
+      },
+      {
+        data: "S,+000.35,+001.82,M,00,3F",
+        s: 1697578842,
+        us: 13019,
+        type: DATA_TYPE,
+        id: "12345678",
+        component: "component",
+      },
+    ];
+
+    expect(result.length).toEqual(5);
+    result.forEach((reading: ReadingV1, idx: number) => {
+      expect(reading).toEqual(expectedReading[idx]);
+    });
+  });
+});

package/src/decode/aggregated.ts

@@ -0,0 +1,48 @@
+import {
+  AggregateMetaDataV1Version,
+  DATA_TYPE,
+  ReadingV1,
+  DeviceReadingV1,
+  AggregateMetadataTypes,
+  AggregateMetadataV1,
+} from "@xube/kit-aws-data-schema";
+
+export const decodeAggregateData = (
+  aggregate: AggregateMetadataTypes,
+  data: Buffer,
+  device: string,
+  component: string
+): ReadingV1[] => {
+  if (aggregate.v === AggregateMetaDataV1Version) {
+    return decodeAggregateDataV1(aggregate, data, device, component);
+  }
+
+  throw new Error(`Unknown aggregate metadata schema version: ${aggregate.v}`);
+};
+
+export const decodeAggregateDataV1 = (
+  aggregate: AggregateMetadataV1,
+  data: Buffer,
+  device: string,
+  component: string
+): ReadingV1[] => {
+  let currentPosition: number = 0;
+  return aggregate.m.map<ReadingV1>(
+    (readingMetadata: DeviceReadingV1, idx): ReadingV1 => {
+      let length: number = readingMetadata.s ? readingMetadata.s : aggregate.s;
+
+      const readingData: Buffer = data.subarray(
+        currentPosition,
+        currentPosition + length
+      );
+      currentPosition += length;
+      return {
+        ...readingMetadata.m,
+        data: readingData.toString("ascii"),
+        id: device,
+        type: DATA_TYPE,
+        component,
+      };
+    }
+  );
+};

package/src/get/get-data.ts

@@ -0,0 +1,116 @@
+import { queryItemsFromTableBetween } from "@xube/kit-aws";
+import { TableItem } from "@xube/kit-aws-schema";
+import { StatusCode } from "@xube/kit-constants";
+import {
+  GetDataByDateRangeRequest,
+  GetDataByDateRangeResponse,
+  getDataPartitionKey,
+  getDataSortKey,
+  isReadingV1,
+} from "@xube/kit-aws-data-schema";
+import { XubeLog } from "@xube/kit-log";
+import { XubeResponse } from "@xube/kit-request";
+
+const fetchItemsFromDB = async (
+  request: GetDataByDateRangeRequest,
+  tableName: string
+): Promise<XubeResponse<TableItem[]>[]> => {
+  const queryItemsPromises: Promise<XubeResponse<TableItem[]>>[] = [];
+
+  for (const device of request.devices) {
+    for (const component of device.components) {
+      queryItemsPromises.push(
+        queryItemsFromTableBetween(tableName, {
+          PK: getDataPartitionKey(device.id),
+          SKstart: getDataSortKey(
+            component,
+            `${new Date(request.dateRange.start).getTime() / 1000}`
+          ),
+          SKend: getDataSortKey(
+            component,
+            `${new Date(request.dateRange.end).getTime() / 1000}`
+          ),
+        })
+      );
+    }
+  }
+
+  return Promise.all(queryItemsPromises);
+};
+
+const processResponseItems = (
+  response: XubeResponse<TableItem[]>,
+  log: XubeLog
+): {
+  data?: GetDataByDateRangeResponse;
+  errorCount: number;
+} => {
+  if (response.hasFailed() || !response.data) {
+    log.warn(`Failed to get data for date range: ${JSON.stringify(response)}`);
+    return { errorCount: 1 };
+  }
+
+  const getDataResponse: GetDataByDateRangeResponse = { devices: [] };
+
+  for (const item of response.data) {
+    if (!isReadingV1(item)) {
+      log.warn(`Found item that is not a reading v1: ${JSON.stringify(item)}`);
+      return { errorCount: 1 };
+    }
+
+    let device = getDataResponse.devices.find((device) => device.id === item.id);
+    if (!device) {
+      device = { id: item.id, components: [] };
+      getDataResponse.devices.push(device);
+    }
+
+    let component = device.components.find(
+      (component) => component.id === item.component
+    );
+    if (!component) {
+      component = { id: item.component, data: [] };
+      device.components.push(component);
+    }
+
+    component.data.push({
+      seconds: item.s,
+      microseconds: item.us,
+      readings: item.data,
+    });
+  }
+
+  return { data: getDataResponse, errorCount: 0 };
+};
+
+export const getDataForDateRange = async (
+  request: GetDataByDateRangeRequest,
+  tableName: string,
+  log: XubeLog = XubeLog.getInstance()
+): Promise<XubeResponse<GetDataByDateRangeResponse>> => {
+  const queryItemsResponses = await fetchItemsFromDB(request, tableName);
+  
+  let totalErrorCount = 0;
+  const getDataResponse: GetDataByDateRangeResponse = { devices: [] };
+
+  for (const response of queryItemsResponses) {
+    const { data, errorCount } = processResponseItems(response, log);
+    totalErrorCount += errorCount;
+    if (data) {
+      getDataResponse.devices.push(...data.devices);
+    }
+  }
+
+  if (totalErrorCount === queryItemsResponses.length) {
+    return new XubeResponse({
+      statusCode: StatusCode.NotFound,
+      error: `Failed to get data for date range. No data found.`,
+    });
+  } else if (totalErrorCount > 0) {
+    return new XubeResponse({
+      statusCode: StatusCode.PartialSuccess,
+      data: getDataResponse,
+    });
+  }
+
+  return new XubeResponse({ data: getDataResponse, statusCode: StatusCode.OK });
+};

package/src/index.ts

@@ -0,0 +1,3 @@
+export * from "./decode/aggregated";
+export * from "./get/get-data";
+export * from "./ingest/stage";

package/src/ingest/stage.ts

@@ -0,0 +1,27 @@
+import { ReadingV1, isAggregateMetadataTypes } from "@xube/kit-aws-data-schema";
+import { decodeAggregateData } from "../decode/aggregated";
+
+export const parseDeviceData = (
+  payload: Buffer,
+  device: string,
+  component: string
+): ReadingV1[] => {
+  const splitIdx: number = payload.findIndex((value, idx, obj) => {
+    return value === 0x00;
+  });
+
+  if (splitIdx === -1) {
+    throw new Error("Invalid data format");
+  }
+
+  const metadataString: string = payload.subarray(0, splitIdx).toString();
+  const metadata = JSON.parse(metadataString);
+
+  if (!isAggregateMetadataTypes(metadata)) {
+    throw new Error("Invalid metadata format");
+  }
+
+  const data: Buffer = payload.subarray(splitIdx + 1);
+
+  return decodeAggregateData(metadata, data, device, component);
+};

package/tsconfig.json

@@ -0,0 +1,31 @@
+{
+  "extends": "../kit-build/tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist"
+  },
+  "exclude": ["**/*.test.ts", "**/*.mock.ts", "dist"],
+  "references": [
+    {
+      "path": "../kit-constants"
+    },
+    {
+      "path": "../kit-generator"
+    },
+    {
+      "path": "../kit-log"
+    },
+    {
+      "path": "../kit-schema"
+    },
+    {
+      "path": "../kit-aws"
+    },
+    {
+      "path": "../kit-aws-schema"
+    },
+    {
+      "path": "../kit-aws-data-schema"
+    }
+  ]
+}