@mxtommy/kip 4.5.0 → 4.5.2

package/CHANGELOG.md

@@ -1,3 +1,11 @@
+# v4.5.2
+## Fixes
+* DuckDB initialized when features are not enabled.
+* Parquet data compression and pruning not executing.
+# v4.5.1
+## Fixes
+* DuckDB dependency causing build and installation errors. Fixes #979
+* Reduced installation size. Fixes #980
 # v4.5.0
 ## New Features
 * Effortlessly review your vessel’s history with the new Widget Historical Charts—automatically track, store, and visualize key data. Instantly access up to the last full day of performance: just two-finger tap or right-click any widget to open a seamless history dialog—no setup, no clutter, just the trends you need. (Requires Signal K v2.22.1)
@@ -7,6 +15,7 @@
 * Added "Days" as a selectable time scale in the Data Chart widget.
 * Improved integration by validating server plugin presence, plugin state, and configuration.
 * Added a Node-RED introduction guide to the Help section.
+* Migrated KIP plugin historical storage internals from `duckdb` to `@duckdb/node-api` and Parquet export writing to `@dsnp/parquetjs`.
 ## Fixes
 * Improved KIP plugin OpenAPI compatibility.
 * Resolved slow Data Inspector performance caused by high resource usage in deep loop logic.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mxtommy/kip",
-  "version": "4.5.0",
+  "version": "4.5.2",
   "description": "An advanced and versatile marine instrumentation package to display Signal K data.",
   "license": "MIT",
   "author": {
@@ -40,6 +40,15 @@
     "displayName": "KIP Instrument MFD"
   },
   "main": "plugin/index.js",
+  "files": [
+    "plugin/**",
+    "public/**",
+    "README.md",
+    "LICENSE",
+    "CHANGELOG.md",
+    "CONTRIBUTORS.md",
+    "package.json"
+  ],
   "signalk-plugin-enabled-by-default": true,
   "scripts": {
     "test": "CI=1 ng test --watch=false",
@@ -57,12 +66,22 @@
   },
   "schematics": "tools/schematics/collection.json",
   "devDependencies": {
+    "@angular/cdk": "21.2.0",
+    "@angular/common": "21.2.0",
+    "@angular/compiler": "21.2.0",
+    "@angular/core": "21.2.0",
+    "@angular/forms": "21.2.0",
+    "@angular/material": "21.2.0",
+    "@angular/animations": "21.2.0",
+    "@angular/platform-browser": "21.2.0",
+    "@angular/platform-browser-dynamic": "21.2.0",
+    "@angular/router": "21.2.0",
     "@angular-devkit/build-angular": "^21.1.4",
     "@angular-devkit/schematics-cli": "^20.1.6",
     "@angular/build": "^21.1.4",
     "@angular/cli": "^21.1.4",
-    "@angular/compiler-cli": "21.1.4",
-    "@angular/language-service": "21.1.4",
+    "@angular/compiler-cli": "21.2.0",
+    "@angular/language-service": "21.2.0",
     "@types/canvas-gauges": "^2.1.8",
     "@types/d3": "^7.4.3",
     "@types/jasmine": "~3.6.0",
@@ -87,22 +106,9 @@
     "pwa-asset-generator": "^8.1.1",
     "sass": "^1.49.9",
     "ts-node": "^10.9.2",
-    "typescript": "^5.9.3"
-  },
-  "dependencies": {
-    "@angular/animations": "21.1.4",
-    "@angular/cdk": "21.1.4",
-    "@angular/common": "21.1.4",
-    "@angular/compiler": "21.1.4",
-    "@angular/core": "21.1.4",
-    "@angular/forms": "21.1.4",
-    "@angular/material": "21.1.4",
-    "@angular/platform-browser": "21.1.4",
-    "@angular/platform-browser-dynamic": "21.1.4",
-    "@angular/router": "21.1.4",
+    "typescript": "^5.9.3",
     "@aziham/chartjs-plugin-streaming": "^3.5.1",
     "@godind/ng-canvas-gauges": "^6.2.1",
-    "@signalk/server-api": "^2.22.0",
     "@zakj/no-sleep": "^0.13.5",
     "chart.js": "^4.5.1",
     "chartjs-adapter-date-fns": "^3.0.0",
@@ -112,7 +118,6 @@
     "core-js": "^3.13.1",
     "d3": "^7.9.0",
     "date-fns": "^2.30.0",
-    "duckdb": "^1.4.4",
     "gridstack": "^12.3.3",
     "js-quantities": "^1.8.0",
     "lodash-es": "^4.17.23",
@@ -124,5 +129,10 @@
     "steelseries": "^2.0.9",
     "tslib": "^2.6.2",
     "zone.js": "^0.15.1"
+  },
+  "dependencies": {
+    "@dsnp/parquetjs": "^1.8.7",
+    "@duckdb/node-api": "^1.4.4-r.2",
+    "@signalk/server-api": "^2.22.0"
   }
 }

package/plugin/duckdb-parquet-storage.service.js

@@ -1,52 +1,23 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DuckDbParquetStorageService = void 0;
 const fs_1 = require("fs");
 const path_1 = require("path");
-const duckdb = __importStar(require("duckdb"));
+const node_api_1 = require("@duckdb/node-api");
+const parquetjs_1 = require("@dsnp/parquetjs");
 /**
  * Provides DuckDB storage and Parquet flush support for captured history samples.
  */
+const DEFAULT_STORAGE_CONFIG = {
+    engine: 'duckdb-parquet',
+    databaseFile: 'plugin-config-data/kip/historicalData/kip-history.duckdb',
+    parquetDirectory: 'plugin-config-data/kip/historicalData/parquet',
+    flushIntervalMs: 30_000,
+    parquetWindowMs: 60 * 60_000,
+    parquetCompression: 'snappy'
+};
 class DuckDbParquetStorageService {
-    config = {
-        engine: 'duckdb-parquet',
-        databaseFile: 'plugin-config-data/kip/historicalData/kip-history.duckdb',
-        parquetDirectory: 'plugin-config-data/kip/historicalData/parquet',
-        flushIntervalMs: 30_000
-    };
+    config = { ...DEFAULT_STORAGE_CONFIG };
     // 8 hour job interval (8 hours)
     static EIGHT_HOURS_INTERVAL = 8 * 60 * 60 * 1000;
     vacuumJob = null;
@@ -83,24 +54,17 @@ class DuckDbParquetStorageService {
         this.logger = logger;
     }
     /**
-     * Applies plugin settings into the storage backend configuration.
+     * Applies the fixed storage backend configuration.
      *
-     * @param {unknown} settings Plugin settings payload from Signal K (ignored for fixed storage defaults).
      * @returns {IDuckDbParquetStorageConfig} Fixed storage configuration.
      *
      * @example
-     * const cfg = storage.configure({});
+     * const cfg = storage.configure();
      * console.log(cfg.engine);
      */
-    configure(settings) {
-        void settings;
+    configure() {
         this.initialized = false;
-        this.config = {
-            engine: 'duckdb-parquet',
-            databaseFile: 'plugin-config-data/kip/historicalData/kip-history.duckdb',
-            parquetDirectory: 'plugin-config-data/kip/historicalData/parquet',
-            flushIntervalMs: 30_000
-        };
+        this.config = { ...DEFAULT_STORAGE_CONFIG };
         return this.config;
     }
     /**
@@ -118,21 +82,11 @@ class DuckDbParquetStorageService {
         this.initialized = false;
         this.lifecycleToken += 1;
         try {
-            const duckdbModule = duckdb;
             const dbPath = (0, path_1.resolve)(this.config.databaseFile);
             (0, fs_1.mkdirSync)((0, path_1.dirname)(dbPath), { recursive: true });
             (0, fs_1.mkdirSync)((0, path_1.resolve)(this.config.parquetDirectory), { recursive: true });
-            this.db = new duckdbModule.Database(dbPath);
-            const maybeConnection = typeof this.db.connect === 'function'
-                ? this.db.connect()
-                : this.db;
-            if (!maybeConnection
-                || typeof maybeConnection.run !== 'function'
-                || typeof maybeConnection.all !== 'function'
-                || typeof maybeConnection.close !== 'function') {
-                throw new Error('DuckDB connection API is unavailable in this runtime');
-            }
-            this.connection = maybeConnection;
+            this.db = await node_api_1.DuckDBInstance.create(dbPath);
+            this.connection = await this.db.connect();
             await this.createCoreTables();
             await this.runSql('CREATE INDEX IF NOT EXISTS idx_history_series_scope_ts ON history_samples(series_id, ts_ms)');
             await this.runSql('CREATE INDEX IF NOT EXISTS idx_history_series_scope_id ON history_series(series_id)');
@@ -154,7 +108,7 @@ class DuckDbParquetStorageService {
             const message = error?.message ?? String(error);
             this.lastInitError = message;
             this.logger.error(`[SERIES STORAGE] DuckDB initialization failed: ${message}`);
-            this.logger.error('[SERIES STORAGE] DuckDB is required. Install runtime dependency with: npm i duckdb in the installed plugin directory, then restart Signal K.');
+            this.logger.error('[SERIES STORAGE] DuckDB Node API is required. Install runtime dependency with: npm i @duckdb/node-api in the installed plugin directory, then restart Signal K.');
             this.connection = null;
             this.db = null;
             this.pendingRows = [];
@@ -209,7 +163,11 @@ class DuckDbParquetStorageService {
                     this.logger.debug('[SERIES STORAGE] Running scheduled prune of expired and orphaned samples');
                     const expired = await this.pruneExpiredSamples(Date.now(), this.lifecycleToken);
                     const orphaned = await this.pruneOrphanedSamples(this.lifecycleToken);
+                    const parquetRemoved = await this.pruneParquetFilesByRetention(Date.now());
                     this.logger.debug(`[SERIES STORAGE] Pruned ${expired} expired and ${orphaned} orphaned samples`);
+                    if (parquetRemoved > 0) {
+                        this.logger.debug(`[SERIES STORAGE] Pruned ${parquetRemoved} parquet files/directories`);
+                    }
                 });
             }
             catch (err) {
@@ -570,6 +528,7 @@ class DuckDbParquetStorageService {
             return;
         }
         await this.runSql(`DELETE FROM history_series WHERE series_id = ${this.escape(seriesId)}`);
+        this.deleteParquetSeriesDir(seriesId);
     }
     /**
      * Replaces persisted series definitions with desired set.
@@ -811,10 +770,20 @@ class DuckDbParquetStorageService {
             return;
         }
         const connection = this.connection;
+        const db = this.db;
         this.connection = null;
-        await new Promise((resolvePromise) => {
-            connection.close(() => resolvePromise());
-        });
+        try {
+            connection.disconnectSync();
+        }
+        catch {
+            // ignore disconnect failures during shutdown
+        }
+        try {
+            db?.closeSync();
+        }
+        catch {
+            // ignore close failures during shutdown
+        }
         this.db = null;
     }
     async createCoreTables() {
@@ -850,10 +819,6 @@ class DuckDbParquetStorageService {
       )
     `);
     }
-    async countRows(tableName) {
-        const rows = await this.querySql(`SELECT COUNT(*) AS removed_rows FROM ${tableName}`);
-        return this.toNumberOrUndefined(rows[0]?.removed_rows) ?? 0;
-    }
     async insertRows(rows) {
         if (rows.length === 0) {
             return;
@@ -876,14 +841,16 @@ class DuckDbParquetStorageService {
         await this.runSql(sql);
     }
     async exportSeriesRange(seriesId, fromMs, toMs) {
+        const windowMs = this.resolveParquetWindowMs();
         const baseDir = (0, path_1.resolve)(this.config.parquetDirectory);
         const seriesDir = (0, path_1.join)(baseDir, this.safePath(seriesId));
         (0, fs_1.mkdirSync)(seriesDir, { recursive: true });
-        const filePath = (0, path_1.join)(seriesDir, `${fromMs}-${toMs}.parquet`);
-        const escapedSeries = this.escape(seriesId);
-        const escapedFile = this.escapePath(filePath);
-        const sql = `
-      COPY (
+        const startWindow = Math.floor(fromMs / windowMs) * windowMs;
+        const endWindow = Math.floor(toMs / windowMs) * windowMs;
+        for (let windowStart = startWindow; windowStart <= endWindow; windowStart += windowMs) {
+            const windowEnd = windowStart + windowMs - 1;
+            const filePath = (0, path_1.join)(seriesDir, `${windowStart}-${windowEnd}.parquet`);
+            const rows = await this.querySql(`
         SELECT
           series_id,
           dataset_uuid,
@@ -892,44 +859,175 @@ class DuckDbParquetStorageService {
           context,
           source,
           ts_ms,
-          to_timestamp(ts_ms / 1000.0) AS ts,
           value
         FROM history_samples
-        WHERE series_id = ${escapedSeries}
-          AND ts_ms >= ${Math.trunc(fromMs)}
-          AND ts_ms <= ${Math.trunc(toMs)}
+        WHERE series_id = ${this.escape(seriesId)}
+          AND ts_ms >= ${Math.trunc(windowStart)}
+          AND ts_ms <= ${Math.trunc(windowEnd)}
         ORDER BY ts_ms
-      ) TO '${escapedFile}' (FORMAT PARQUET)
-    `;
-        await this.runSql(sql);
+      `);
+            if (rows.length === 0) {
+                (0, fs_1.rmSync)(filePath, { force: true });
+                continue;
+            }
+            const compression = this.resolveParquetCompression();
+            const schema = new parquetjs_1.ParquetSchema({
+                series_id: { type: 'UTF8', compression },
+                dataset_uuid: { type: 'UTF8', compression },
+                owner_widget_uuid: { type: 'UTF8', compression },
+                path: { type: 'UTF8', compression },
+                context: { type: 'UTF8', compression },
+                source: { type: 'UTF8', optional: true, compression },
+                ts_ms: { type: 'INT64', compression },
+                ts: { type: 'TIMESTAMP_MILLIS', compression },
+                value: { type: 'DOUBLE', compression }
+            });
+            const tempPath = `${filePath}.tmp-${Date.now()}`;
+            const writer = await parquetjs_1.ParquetWriter.openFile(schema, tempPath);
+            try {
+                for (const row of rows) {
+                    const timestampMs = this.toNumberOrUndefined(row.ts_ms);
+                    const numericValue = this.toNumberOrUndefined(row.value);
+                    if (timestampMs === undefined || numericValue === undefined) {
+                        continue;
+                    }
+                    await writer.appendRow({
+                        series_id: row.series_id,
+                        dataset_uuid: row.dataset_uuid,
+                        owner_widget_uuid: row.owner_widget_uuid,
+                        path: row.path,
+                        context: row.context,
+                        source: row.source ?? undefined,
+                        ts_ms: Math.trunc(timestampMs),
+                        ts: new Date(timestampMs),
+                        value: numericValue
+                    });
+                }
+            }
+            finally {
+                await writer.close();
+            }
+            (0, fs_1.rmSync)(filePath, { force: true });
+            (0, fs_1.renameSync)(tempPath, filePath);
+        }
+    }
+    resolveParquetWindowMs() {
+        const parsed = Number(this.config.parquetWindowMs);
+        if (Number.isFinite(parsed) && parsed > 0) {
+            return Math.max(1, Math.trunc(parsed));
+        }
+        return 60 * 60_000;
+    }
+    resolveParquetCompression() {
+        const raw = String(this.config.parquetCompression ?? 'none').trim().toLowerCase();
+        if (raw === 'snappy') {
+            return 'SNAPPY';
+        }
+        if (raw === 'gzip') {
+            return 'GZIP';
+        }
+        if (raw === 'brotli') {
+            return 'BROTLI';
+        }
+        if (raw === 'lz4') {
+            return 'LZ4';
+        }
+        if (raw === 'lzo') {
+            return 'LZO';
+        }
+        return undefined;
+    }
+    pruneParquetFilesByRetention(nowMs) {
+        if (!this.isDuckDbParquetEnabled() || !this.connection) {
+            return Promise.resolve(0);
+        }
+        return this.querySql(`
+      SELECT series_id, retention_duration_ms
+      FROM history_series
+    `).then(rows => {
+            const baseDir = (0, path_1.resolve)(this.config.parquetDirectory);
+            const seriesConfig = new Map(rows.map(row => [this.safePath(row.series_id), row.retention_duration_ms]));
+            let removed = 0;
+            let seriesDirs = [];
+            try {
+                seriesDirs = (0, fs_1.readdirSync)(baseDir);
+            }
+            catch {
+                return 0;
+            }
+            for (const dirName of seriesDirs) {
+                const fullDir = (0, path_1.join)(baseDir, dirName);
+                let dirStat;
+                try {
+                    dirStat = (0, fs_1.statSync)(fullDir);
+                }
+                catch {
+                    continue;
+                }
+                if (!dirStat.isDirectory()) {
+                    continue;
+                }
+                const seriesKey = dirName;
+                if (!seriesConfig.has(seriesKey)) {
+                    (0, fs_1.rmSync)(fullDir, { recursive: true, force: true });
+                    removed += 1;
+                    continue;
+                }
+                const retentionMs = seriesConfig.get(seriesKey);
+                if (!Number.isFinite(retentionMs) || (retentionMs ?? 0) <= 0) {
+                    continue;
+                }
+                const cutoff = nowMs - Number(retentionMs);
+                let files = [];
+                try {
+                    files = (0, fs_1.readdirSync)(fullDir);
+                }
+                catch {
+                    continue;
+                }
+                files.forEach(fileName => {
+                    const range = this.parseParquetRangeFromFileName(fileName);
+                    if (!range) {
+                        return;
+                    }
+                    if (range.toMs < cutoff) {
+                        (0, fs_1.rmSync)((0, path_1.join)(fullDir, fileName), { force: true });
+                        removed += 1;
+                    }
+                });
+            }
+            return removed;
+        });
+    }
+    parseParquetRangeFromFileName(fileName) {
+        const match = /^([0-9]+)-([0-9]+)\.parquet$/.exec(fileName);
+        if (!match) {
+            return null;
+        }
+        const fromMs = Number(match[1]);
+        const toMs = Number(match[2]);
+        if (!Number.isFinite(fromMs) || !Number.isFinite(toMs)) {
+            return null;
+        }
+        return { fromMs, toMs };
+    }
+    deleteParquetSeriesDir(seriesId) {
+        const baseDir = (0, path_1.resolve)(this.config.parquetDirectory);
+        const seriesDir = (0, path_1.join)(baseDir, this.safePath(seriesId));
+        (0, fs_1.rmSync)(seriesDir, { recursive: true, force: true });
     }
     async runSql(sql) {
         if (!this.connection) {
             throw new Error('DuckDB connection is not initialized');
         }
-        await new Promise((resolvePromise, rejectPromise) => {
-            this.connection?.run(sql, (error) => {
-                if (error) {
-                    rejectPromise(error);
-                    return;
-                }
-                resolvePromise();
-            });
-        });
+        await this.connection.run(sql);
     }
     async querySql(sql) {
         if (!this.connection) {
             throw new Error('DuckDB connection is not initialized');
         }
-        return new Promise((resolvePromise, rejectPromise) => {
-            this.connection?.all(sql, (error, rows) => {
-                if (error) {
-                    rejectPromise(error);
-                    return;
-                }
-                resolvePromise((rows ?? []));
-            });
-        });
+        const result = await this.connection.runAndReadAll(sql);
+        return result.getRowObjectsJson();
     }
     async selectRowsForPaths(paths, context, fromMs, toMs) {
         const rowsByPath = new Map();
@@ -1142,9 +1240,6 @@ class DuckDbParquetStorageService {
     escape(value) {
         return `'${String(value).replace(/'/g, "''")}'`;
     }
-    escapePath(value) {
-        return String(value).replace(/'/g, "''");
-    }
     nullableString(value) {
         if (value === undefined || value === null || value === '') {
             return 'NULL';