@mxtommy/kip 4.5.1 → 4.5.2

package/CHANGELOG.md

@@ -1,3 +1,7 @@
+# v4.5.2
+## Fixes
+* DuckDB initialized when features are not enabled.
+* Parquet data compression and pruning not executing.
 # v4.5.1
 ## Fixes
 * DuckDB dependency causing build and installation errors. Fixes #979

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mxtommy/kip",
-  "version": "4.5.1",
+  "version": "4.5.2",
   "description": "An advanced and versatile marine instrumentation package to display Signal K data.",
   "license": "MIT",
   "author": {
@@ -66,12 +66,22 @@
   },
   "schematics": "tools/schematics/collection.json",
   "devDependencies": {
+    "@angular/cdk": "21.2.0",
+    "@angular/common": "21.2.0",
+    "@angular/compiler": "21.2.0",
+    "@angular/core": "21.2.0",
+    "@angular/forms": "21.2.0",
+    "@angular/material": "21.2.0",
+    "@angular/animations": "21.2.0",
+    "@angular/platform-browser": "21.2.0",
+    "@angular/platform-browser-dynamic": "21.2.0",
+    "@angular/router": "21.2.0",
     "@angular-devkit/build-angular": "^21.1.4",
     "@angular-devkit/schematics-cli": "^20.1.6",
     "@angular/build": "^21.1.4",
     "@angular/cli": "^21.1.4",
-    "@angular/compiler-cli": "21.1.4",
-    "@angular/language-service": "21.1.4",
+    "@angular/compiler-cli": "21.2.0",
+    "@angular/language-service": "21.2.0",
     "@types/canvas-gauges": "^2.1.8",
     "@types/d3": "^7.4.3",
     "@types/jasmine": "~3.6.0",
@@ -97,16 +107,6 @@
     "sass": "^1.49.9",
     "ts-node": "^10.9.2",
     "typescript": "^5.9.3",
-    "@angular/animations": "21.1.4",
-    "@angular/cdk": "21.1.4",
-    "@angular/common": "21.1.4",
-    "@angular/compiler": "21.1.4",
-    "@angular/core": "21.1.4",
-    "@angular/forms": "21.1.4",
-    "@angular/material": "21.1.4",
-    "@angular/platform-browser": "21.1.4",
-    "@angular/platform-browser-dynamic": "21.1.4",
-    "@angular/router": "21.1.4",
     "@aziham/chartjs-plugin-streaming": "^3.5.1",
     "@godind/ng-canvas-gauges": "^6.2.1",
     "@zakj/no-sleep": "^0.13.5",

package/plugin/duckdb-parquet-storage.service.js

@@ -8,13 +8,16 @@ const parquetjs_1 = require("@dsnp/parquetjs");
 /**
  * Provides DuckDB storage and Parquet flush support for captured history samples.
  */
+const DEFAULT_STORAGE_CONFIG = {
+    engine: 'duckdb-parquet',
+    databaseFile: 'plugin-config-data/kip/historicalData/kip-history.duckdb',
+    parquetDirectory: 'plugin-config-data/kip/historicalData/parquet',
+    flushIntervalMs: 30_000,
+    parquetWindowMs: 60 * 60_000,
+    parquetCompression: 'snappy'
+};
 class DuckDbParquetStorageService {
-    config = {
-        engine: 'duckdb-parquet',
-        databaseFile: 'plugin-config-data/kip/historicalData/kip-history.duckdb',
-        parquetDirectory: 'plugin-config-data/kip/historicalData/parquet',
-        flushIntervalMs: 30_000
-    };
+    config = { ...DEFAULT_STORAGE_CONFIG };
     // 8 hour job interval (8 hours)
     static EIGHT_HOURS_INTERVAL = 8 * 60 * 60 * 1000;
     vacuumJob = null;
@@ -61,12 +64,7 @@ class DuckDbParquetStorageService {
      */
     configure() {
         this.initialized = false;
-        this.config = {
-            engine: 'duckdb-parquet',
-            databaseFile: 'plugin-config-data/kip/historicalData/kip-history.duckdb',
-            parquetDirectory: 'plugin-config-data/kip/historicalData/parquet',
-            flushIntervalMs: 30_000
-        };
+        this.config = { ...DEFAULT_STORAGE_CONFIG };
         return this.config;
     }
     /**
@@ -165,7 +163,11 @@ class DuckDbParquetStorageService {
                     this.logger.debug('[SERIES STORAGE] Running scheduled prune of expired and orphaned samples');
                     const expired = await this.pruneExpiredSamples(Date.now(), this.lifecycleToken);
                     const orphaned = await this.pruneOrphanedSamples(this.lifecycleToken);
+                    const parquetRemoved = await this.pruneParquetFilesByRetention(Date.now());
                     this.logger.debug(`[SERIES STORAGE] Pruned ${expired} expired and ${orphaned} orphaned samples`);
+                    if (parquetRemoved > 0) {
+                        this.logger.debug(`[SERIES STORAGE] Pruned ${parquetRemoved} parquet files/directories`);
+                    }
                 });
             }
             catch (err) {
@@ -526,6 +528,7 @@ class DuckDbParquetStorageService {
             return;
         }
         await this.runSql(`DELETE FROM history_series WHERE series_id = ${this.escape(seriesId)}`);
+        this.deleteParquetSeriesDir(seriesId);
     }
     /**
      * Replaces persisted series definitions with desired set.
@@ -838,64 +841,180 @@ class DuckDbParquetStorageService {
         await this.runSql(sql);
     }
     async exportSeriesRange(seriesId, fromMs, toMs) {
+        const windowMs = this.resolveParquetWindowMs();
         const baseDir = (0, path_1.resolve)(this.config.parquetDirectory);
         const seriesDir = (0, path_1.join)(baseDir, this.safePath(seriesId));
         (0, fs_1.mkdirSync)(seriesDir, { recursive: true });
-        const filePath = (0, path_1.join)(seriesDir, `${fromMs}-${toMs}.parquet`);
-        const rows = await this.querySql(`
-      SELECT
-        series_id,
-        dataset_uuid,
-        owner_widget_uuid,
-        path,
-        context,
-        source,
-        ts_ms,
-        value
-      FROM history_samples
-      WHERE series_id = ${this.escape(seriesId)}
-        AND ts_ms >= ${Math.trunc(fromMs)}
-        AND ts_ms <= ${Math.trunc(toMs)}
-      ORDER BY ts_ms
-    `);
-        if (rows.length === 0) {
-            return;
+        const startWindow = Math.floor(fromMs / windowMs) * windowMs;
+        const endWindow = Math.floor(toMs / windowMs) * windowMs;
+        for (let windowStart = startWindow; windowStart <= endWindow; windowStart += windowMs) {
+            const windowEnd = windowStart + windowMs - 1;
+            const filePath = (0, path_1.join)(seriesDir, `${windowStart}-${windowEnd}.parquet`);
+            const rows = await this.querySql(`
+        SELECT
+          series_id,
+          dataset_uuid,
+          owner_widget_uuid,
+          path,
+          context,
+          source,
+          ts_ms,
+          value
+        FROM history_samples
+        WHERE series_id = ${this.escape(seriesId)}
+          AND ts_ms >= ${Math.trunc(windowStart)}
+          AND ts_ms <= ${Math.trunc(windowEnd)}
+        ORDER BY ts_ms
+      `);
+            if (rows.length === 0) {
+                (0, fs_1.rmSync)(filePath, { force: true });
+                continue;
+            }
+            const compression = this.resolveParquetCompression();
+            const schema = new parquetjs_1.ParquetSchema({
+                series_id: { type: 'UTF8', compression },
+                dataset_uuid: { type: 'UTF8', compression },
+                owner_widget_uuid: { type: 'UTF8', compression },
+                path: { type: 'UTF8', compression },
+                context: { type: 'UTF8', compression },
+                source: { type: 'UTF8', optional: true, compression },
+                ts_ms: { type: 'INT64', compression },
+                ts: { type: 'TIMESTAMP_MILLIS', compression },
+                value: { type: 'DOUBLE', compression }
+            });
+            const tempPath = `${filePath}.tmp-${Date.now()}`;
+            const writer = await parquetjs_1.ParquetWriter.openFile(schema, tempPath);
+            try {
+                for (const row of rows) {
+                    const timestampMs = this.toNumberOrUndefined(row.ts_ms);
+                    const numericValue = this.toNumberOrUndefined(row.value);
+                    if (timestampMs === undefined || numericValue === undefined) {
+                        continue;
+                    }
+                    await writer.appendRow({
+                        series_id: row.series_id,
+                        dataset_uuid: row.dataset_uuid,
+                        owner_widget_uuid: row.owner_widget_uuid,
+                        path: row.path,
+                        context: row.context,
+                        source: row.source ?? undefined,
+                        ts_ms: Math.trunc(timestampMs),
+                        ts: new Date(timestampMs),
+                        value: numericValue
+                    });
+                }
+            }
+            finally {
+                await writer.close();
+            }
+            (0, fs_1.rmSync)(filePath, { force: true });
+            (0, fs_1.renameSync)(tempPath, filePath);
         }
-        const schema = new parquetjs_1.ParquetSchema({
-            series_id: { type: 'UTF8' },
-            dataset_uuid: { type: 'UTF8' },
-            owner_widget_uuid: { type: 'UTF8' },
-            path: { type: 'UTF8' },
-            context: { type: 'UTF8' },
-            source: { type: 'UTF8', optional: true },
-            ts_ms: { type: 'INT64' },
-            ts: { type: 'TIMESTAMP_MILLIS' },
-            value: { type: 'DOUBLE' }
-        });
-        const writer = await parquetjs_1.ParquetWriter.openFile(schema, filePath);
-        try {
-            for (const row of rows) {
-                const timestampMs = this.toNumberOrUndefined(row.ts_ms);
-                const numericValue = this.toNumberOrUndefined(row.value);
-                if (timestampMs === undefined || numericValue === undefined) {
+    }
+    resolveParquetWindowMs() {
+        const parsed = Number(this.config.parquetWindowMs);
+        if (Number.isFinite(parsed) && parsed > 0) {
+            return Math.max(1, Math.trunc(parsed));
+        }
+        return 60 * 60_000;
+    }
+    resolveParquetCompression() {
+        const raw = String(this.config.parquetCompression ?? 'none').trim().toLowerCase();
+        if (raw === 'snappy') {
+            return 'SNAPPY';
+        }
+        if (raw === 'gzip') {
+            return 'GZIP';
+        }
+        if (raw === 'brotli') {
+            return 'BROTLI';
+        }
+        if (raw === 'lz4') {
+            return 'LZ4';
+        }
+        if (raw === 'lzo') {
+            return 'LZO';
+        }
+        return undefined;
+    }
+    pruneParquetFilesByRetention(nowMs) {
+        if (!this.isDuckDbParquetEnabled() || !this.connection) {
+            return Promise.resolve(0);
+        }
+        return this.querySql(`
+      SELECT series_id, retention_duration_ms
+      FROM history_series
+    `).then(rows => {
+            const baseDir = (0, path_1.resolve)(this.config.parquetDirectory);
+            const seriesConfig = new Map(rows.map(row => [this.safePath(row.series_id), row.retention_duration_ms]));
+            let removed = 0;
+            let seriesDirs = [];
+            try {
+                seriesDirs = (0, fs_1.readdirSync)(baseDir);
+            }
+            catch {
+                return 0;
+            }
+            for (const dirName of seriesDirs) {
+                const fullDir = (0, path_1.join)(baseDir, dirName);
+                let dirStat;
+                try {
+                    dirStat = (0, fs_1.statSync)(fullDir);
+                }
+                catch {
+                    continue;
+                }
+                if (!dirStat.isDirectory()) {
+                    continue;
+                }
+                const seriesKey = dirName;
+                if (!seriesConfig.has(seriesKey)) {
+                    (0, fs_1.rmSync)(fullDir, { recursive: true, force: true });
+                    removed += 1;
                     continue;
                 }
-                await writer.appendRow({
-                    series_id: row.series_id,
-                    dataset_uuid: row.dataset_uuid,
-                    owner_widget_uuid: row.owner_widget_uuid,
-                    path: row.path,
-                    context: row.context,
-                    source: row.source ?? undefined,
-                    ts_ms: Math.trunc(timestampMs),
-                    ts: new Date(timestampMs),
-                    value: numericValue
+                const retentionMs = seriesConfig.get(seriesKey);
+                if (!Number.isFinite(retentionMs) || (retentionMs ?? 0) <= 0) {
+                    continue;
+                }
+                const cutoff = nowMs - Number(retentionMs);
+                let files = [];
+                try {
+                    files = (0, fs_1.readdirSync)(fullDir);
+                }
+                catch {
+                    continue;
+                }
+                files.forEach(fileName => {
+                    const range = this.parseParquetRangeFromFileName(fileName);
+                    if (!range) {
+                        return;
+                    }
+                    if (range.toMs < cutoff) {
+                        (0, fs_1.rmSync)((0, path_1.join)(fullDir, fileName), { force: true });
+                        removed += 1;
+                    }
                 });
             }
+            return removed;
+        });
+    }
+    parseParquetRangeFromFileName(fileName) {
+        const match = /^([0-9]+)-([0-9]+)\.parquet$/.exec(fileName);
+        if (!match) {
+            return null;
         }
-        finally {
-            await writer.close();
+        const fromMs = Number(match[1]);
+        const toMs = Number(match[2]);
+        if (!Number.isFinite(fromMs) || !Number.isFinite(toMs)) {
+            return null;
         }
+        return { fromMs, toMs };
+    }
+    deleteParquetSeriesDir(seriesId) {
+        const baseDir = (0, path_1.resolve)(this.config.parquetDirectory);
+        const seriesDir = (0, path_1.join)(baseDir, this.safePath(seriesId));
+        (0, fs_1.rmSync)(seriesDir, { recursive: true, force: true });
     }
     async runSql(sql) {
         if (!this.connection) {

package/plugin/index.js

@@ -583,87 +583,95 @@ const start = (server) => {
             historySeriesServiceEnabled = modeConfig.historySeriesServiceEnabled;
             registerAsHistoryApiProvider = modeConfig.registerAsHistoryApiProvider;
             logOperationalMode('start-configured');
-            storageService.setLogger({
-                debug: (msg) => server.debug(msg),
-                error: (msg) => server.error(msg)
-            });
-            const storageConfig = storageService.configure();
-            server.debug(`[KIP][STORAGE] config engine=${storageConfig.engine} db=${storageConfig.databaseFile} parquetDir=${storageConfig.parquetDirectory} flushMs=${storageConfig.flushIntervalMs}`);
-            historySeries.setSampleSink(sample => {
-                storageService.enqueueSample(sample);
-            });
-            duckDbInitializationPromise = storageService.initialize();
-            void duckDbInitializationPromise.then((ready) => {
-                server.debug(`[KIP][STORAGE] duckdbReady=${ready}`);
-                if (ready && storageService.isDuckDbParquetEnabled()) {
-                    if (isHistorySeriesServiceEnabled()) {
-                        void storageService.getSeriesDefinitions()
-                            .then((storedSeries) => {
-                            if (storedSeries.length > 0) {
-                                historySeries.reconcileSeries(storedSeries);
-                                rebuildSeriesCaptureSubscriptions();
-                            }
-                            startStorageFlushTimer(storageConfig.flushIntervalMs);
-                            logOperationalMode('duckdb-ready');
-                            server.setPluginStatus(`KIP plugin started with DuckDB/Parquet history storage. Loaded ${storedSeries.length} persisted series. historySeriesServiceEnabled=${isHistorySeriesServiceEnabled()} historyApiProviderEnabled=${isHistoryApiProviderEnabled()} historyApiProviderRegistered=${historyApiProviderRegistered}`);
-                        })
-                            .catch((loadError) => {
-                            server.error(`[SERIES STORAGE] failed to load persisted series: ${String(loadError.message || loadError)}`);
+            const needsDuckDb = historySeriesServiceEnabled || registerAsHistoryApiProvider;
+            if (needsDuckDb) {
+                storageService.setLogger({
+                    debug: (msg) => server.debug(msg),
+                    error: (msg) => server.error(msg)
+                });
+                const storageConfig = storageService.configure();
+                server.debug(`[KIP][STORAGE] config engine=${storageConfig.engine} db=${storageConfig.databaseFile} parquetDir=${storageConfig.parquetDirectory} flushMs=${storageConfig.flushIntervalMs} parquetWindowMs=${storageConfig.parquetWindowMs} parquetCompression=${storageConfig.parquetCompression}`);
+                historySeries.setSampleSink(sample => {
+                    storageService.enqueueSample(sample);
+                });
+                duckDbInitializationPromise = storageService.initialize();
+                void duckDbInitializationPromise.then((ready) => {
+                    server.debug(`[KIP][STORAGE] duckdbReady=${ready}`);
+                    if (ready && storageService.isDuckDbParquetEnabled()) {
+                        if (isHistorySeriesServiceEnabled()) {
+                            void storageService.getSeriesDefinitions()
+                                .then((storedSeries) => {
+                                if (storedSeries.length > 0) {
+                                    historySeries.reconcileSeries(storedSeries);
+                                    rebuildSeriesCaptureSubscriptions();
+                                }
+                                startStorageFlushTimer(storageConfig.flushIntervalMs);
+                                logOperationalMode('duckdb-ready');
+                                server.setPluginStatus(`KIP plugin started with DuckDB/Parquet history storage. Loaded ${storedSeries.length} persisted series. historySeriesServiceEnabled=${isHistorySeriesServiceEnabled()} historyApiProviderEnabled=${isHistoryApiProviderEnabled()} historyApiProviderRegistered=${historyApiProviderRegistered}`);
+                            })
+                                .catch((loadError) => {
+                                server.error(`[SERIES STORAGE] failed to load persisted series: ${String(loadError.message || loadError)}`);
+                                startStorageFlushTimer(storageConfig.flushIntervalMs);
+                                logOperationalMode('duckdb-ready-series-load-failed');
+                                server.setPluginStatus(`KIP plugin started with DuckDB/Parquet history storage. historySeriesServiceEnabled=${isHistorySeriesServiceEnabled()} historyApiProviderEnabled=${isHistoryApiProviderEnabled()} historyApiProviderRegistered=${historyApiProviderRegistered}`);
+                            });
+                        }
+                        else {
+                            historySeries.reconcileSeries([]);
+                            stopSeriesCapture();
                             startStorageFlushTimer(storageConfig.flushIntervalMs);
-                            logOperationalMode('duckdb-ready-series-load-failed');
-                            server.setPluginStatus(`KIP plugin started with DuckDB/Parquet history storage. historySeriesServiceEnabled=${isHistorySeriesServiceEnabled()} historyApiProviderEnabled=${isHistoryApiProviderEnabled()} historyApiProviderRegistered=${historyApiProviderRegistered}`);
-                        });
+                            logOperationalMode('duckdb-ready-series-disabled');
+                            server.setPluginStatus(`KIP plugin started with history-series service disabled. historyApiProviderEnabled=${isHistoryApiProviderEnabled()} historyApiProviderRegistered=${historyApiProviderRegistered}`);
+                        }
+                        return;
                     }
-                    else {
-                        historySeries.reconcileSeries([]);
-                        stopSeriesCapture();
-                        startStorageFlushTimer(storageConfig.flushIntervalMs);
-                        logOperationalMode('duckdb-ready-series-disabled');
-                        server.setPluginStatus(`KIP plugin started with history-series service disabled. historyApiProviderEnabled=${isHistoryApiProviderEnabled()} historyApiProviderRegistered=${historyApiProviderRegistered}`);
+                    if (storageFlushTimer) {
+                        clearInterval(storageFlushTimer);
+                        storageFlushTimer = null;
                     }
-                    return;
-                }
-                if (storageFlushTimer) {
-                    clearInterval(storageFlushTimer);
-                    storageFlushTimer = null;
-                }
-                const initError = storageService.getLastInitError();
-                if (initError) {
-                    server.setPluginError(`DuckDB unavailable. ${initError}`);
-                    logOperationalMode('duckdb-unavailable');
-                    server.setPluginStatus(`KIP plugin started with DuckDB unavailable. historySeriesServiceEnabled=${isHistorySeriesServiceEnabled()} historyApiProviderEnabled=${isHistoryApiProviderEnabled()} historyApiProviderRegistered=${historyApiProviderRegistered}`);
+                    const initError = storageService.getLastInitError();
+                    if (initError) {
+                        server.setPluginError(`DuckDB unavailable. ${initError}`);
+                        logOperationalMode('duckdb-unavailable');
+                        server.setPluginStatus(`KIP plugin started with DuckDB unavailable. historySeriesServiceEnabled=${isHistorySeriesServiceEnabled()} historyApiProviderEnabled=${isHistoryApiProviderEnabled()} historyApiProviderRegistered=${historyApiProviderRegistered}`);
+                    }
+                });
+                if (retentionSweepTimer) {
+                    clearInterval(retentionSweepTimer);
                 }
-            });
-            if (retentionSweepTimer) {
-                clearInterval(retentionSweepTimer);
-            }
-            retentionSweepTimer = setInterval(() => {
-                try {
-                    if (storageService.isDuckDbParquetReady()) {
-                        const lifecycleToken = storageService.getLifecycleToken();
-                        void storageService.pruneExpiredSamples(Date.now(), lifecycleToken)
-                            .then(removedPersistedRows => {
-                            if (removedPersistedRows > 0) {
-                                server.debug(`[KIP][RETENTION] pruneExpired removedRows=${removedPersistedRows}`);
-                            }
-                            return storageService.pruneOrphanedSamples(lifecycleToken)
-                                .then(removedOrphanRows => {
-                                if (removedOrphanRows > 0) {
-                                    server.debug(`[KIP][RETENTION] pruneOrphaned removedRows=${removedOrphanRows}`);
+                retentionSweepTimer = setInterval(() => {
+                    try {
+                        if (storageService.isDuckDbParquetReady()) {
+                            const lifecycleToken = storageService.getLifecycleToken();
+                            void storageService.pruneExpiredSamples(Date.now(), lifecycleToken)
+                                .then(removedPersistedRows => {
+                                if (removedPersistedRows > 0) {
+                                    server.debug(`[KIP][RETENTION] pruneExpired removedRows=${removedPersistedRows}`);
                                 }
+                                return storageService.pruneOrphanedSamples(lifecycleToken)
+                                    .then(removedOrphanRows => {
+                                    if (removedOrphanRows > 0) {
+                                        server.debug(`[KIP][RETENTION] pruneOrphaned removedRows=${removedOrphanRows}`);
+                                    }
+                                });
+                            })
+                                .catch(error => {
+                                server.error(`[SERIES RETENTION] duckdbPrune failed: ${String(error.message || error)}`);
                             });
-                        })
-                            .catch(error => {
-                            server.error(`[SERIES RETENTION] duckdbPrune failed: ${String(error.message || error)}`);
-                        });
+                        }
                     }
-                }
-                catch (error) {
-                    server.error(`[SERIES RETENTION] sweep failed: ${String(error.message || error)}`);
-                }
-            }, 60 * 60_000);
-            retentionSweepTimer.unref?.();
-            rebuildSeriesCaptureSubscriptions();
+                    catch (error) {
+                        server.error(`[SERIES RETENTION] sweep failed: ${String(error.message || error)}`);
+                    }
+                }, 60 * 60_000);
+                retentionSweepTimer.unref?.();
+                rebuildSeriesCaptureSubscriptions();
+            }
+            else {
+                server.debug('[KIP][STORAGE] duckdb init skipped reason=config-disabled');
+                duckDbInitializationPromise = null;
+                stopSeriesCapture();
+            }
             if (server.registerPutHandler) {
                 server.debug(`[KIP][COMMAND] registerPutHandlers context=${PUT_CONTEXT}`);
                 server.registerPutHandler(PUT_CONTEXT, COMMAND_PATHS.SET_DISPLAY, (context, path, value) => {