@mxtommy/kip 4.4.0 → 4.5.0

package/plugin/duckdb-parquet-storage.service.js

@@ -0,0 +1,1206 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DuckDbParquetStorageService = void 0;
+const fs_1 = require("fs");
+const path_1 = require("path");
+const duckdb = __importStar(require("duckdb"));
+/**
+ * Provides DuckDB storage and Parquet flush support for captured history samples.
+ */
+class DuckDbParquetStorageService {
+    config = {
+        engine: 'duckdb-parquet',
+        databaseFile: 'plugin-config-data/kip/historicalData/kip-history.duckdb',
+        parquetDirectory: 'plugin-config-data/kip/historicalData/parquet',
+        flushIntervalMs: 30_000
+    };
+    // 8 hour job interval (8 hours)
+    static EIGHT_HOURS_INTERVAL = 8 * 60 * 60 * 1000;
+    vacuumJob = null;
+    // 4 hour job interval (4 hours)
+    static FOUR_HOURS_INTERVAL = 4 * 60 * 60 * 1000;
+    pruneJob = null;
+    // Stale series cleanup interval (6 months)
+    static STALE_SERIES_AGE_MS = 180 * 24 * 60 * 60 * 1000; // 6 months
+    staleSeriesCleanupJob = null;
+    static PRUNE_BATCH_SIZE = 10_000;
+    logger = {
+        debug: () => undefined,
+        error: () => undefined
+    };
+    db = null;
+    connection = null;
+    pendingRows = [];
+    pendingRangesBySeriesId = new Map();
+    lastInitError = null;
+    lifecycleToken = 0;
+    initialized = false;
+    maintenanceInProgress = false;
+    flushInProgress = false;
+    /**
+     * Sets logger callbacks used by the storage service.
+     *
+     * @param {TLogger} logger Logger implementation from plugin runtime.
+     * @returns {void}
+     *
+     * @example
+     * storage.setLogger({ debug: console.log, error: console.error });
+     */
+    setLogger(logger) {
+        this.logger = logger;
+    }
+    /**
+     * Applies plugin settings into the storage backend configuration.
+     *
+     * @param {unknown} settings Plugin settings payload from Signal K (ignored for fixed storage defaults).
+     * @returns {IDuckDbParquetStorageConfig} Fixed storage configuration.
+     *
+     * @example
+     * const cfg = storage.configure({});
+     * console.log(cfg.engine);
+     */
+    configure(settings) {
+        void settings;
+        this.initialized = false;
+        this.config = {
+            engine: 'duckdb-parquet',
+            databaseFile: 'plugin-config-data/kip/historicalData/kip-history.duckdb',
+            parquetDirectory: 'plugin-config-data/kip/historicalData/parquet',
+            flushIntervalMs: 30_000
+        };
+        return this.config;
+    }
+    /**
+     * Initializes DuckDB storage if DuckDB engine is selected.
+     *
+     * @returns {Promise<boolean>} True when DuckDB is initialized and ready.
+     *
+     * @example
+     * const ready = await storage.initialize();
+     */
+    async initialize() {
+        if (!this.isDuckDbParquetEnabled()) {
+            return false;
+        }
+        this.initialized = false;
+        this.lifecycleToken += 1;
+        try {
+            const duckdbModule = duckdb;
+            const dbPath = (0, path_1.resolve)(this.config.databaseFile);
+            (0, fs_1.mkdirSync)((0, path_1.dirname)(dbPath), { recursive: true });
+            (0, fs_1.mkdirSync)((0, path_1.resolve)(this.config.parquetDirectory), { recursive: true });
+            this.db = new duckdbModule.Database(dbPath);
+            const maybeConnection = typeof this.db.connect === 'function'
+                ? this.db.connect()
+                : this.db;
+            if (!maybeConnection
+                || typeof maybeConnection.run !== 'function'
+                || typeof maybeConnection.all !== 'function'
+                || typeof maybeConnection.close !== 'function') {
+                throw new Error('DuckDB connection API is unavailable in this runtime');
+            }
+            this.connection = maybeConnection;
+            await this.createCoreTables();
+            await this.runSql('CREATE INDEX IF NOT EXISTS idx_history_series_scope_ts ON history_samples(series_id, ts_ms)');
+            await this.runSql('CREATE INDEX IF NOT EXISTS idx_history_series_scope_id ON history_series(series_id)');
+            await this.runSql('CREATE INDEX IF NOT EXISTS idx_history_samples_scope_context_path_ts ON history_samples(context, path, ts_ms)');
+            await this.runSql('CREATE INDEX IF NOT EXISTS idx_history_samples_scope_ts_path ON history_samples(ts_ms, path)');
+            await this.runSql('CREATE INDEX IF NOT EXISTS idx_history_samples_scope_ts_context ON history_samples(ts_ms, context)');
+            this.logger.debug(`[SERIES STORAGE] DuckDB initialized at ${dbPath}`);
+            this.lastInitError = null;
+            this.initialized = true;
+            // Start VACUUM job
+            this.startVacuumJob();
+            // Start prune job
+            this.startPruneJob();
+            // Start stale series cleanup job
+            this.startStaleSeriesCleanupJob();
+            return true;
+        }
+        catch (error) {
+            const message = error?.message ?? String(error);
+            this.lastInitError = message;
+            this.logger.error(`[SERIES STORAGE] DuckDB initialization failed: ${message}`);
+            this.logger.error('[SERIES STORAGE] DuckDB is required. Install runtime dependency with: npm i duckdb in the installed plugin directory, then restart Signal K.');
+            this.connection = null;
+            this.db = null;
+            this.pendingRows = [];
+            this.pendingRangesBySeriesId.clear();
+            this.initialized = false;
+            this.stopVacuumJob();
+            return false;
+        }
+    }
+    /**
+     * Starts VACUUM job for DuckDB.
+     */
+    startVacuumJob() {
+        this.stopVacuumJob();
+        if (!this.isDuckDbParquetReady() || !this.connection)
+            return;
+        this.vacuumJob = setInterval(() => {
+            if (this.shouldSkipMaintenance()) {
+                return;
+            }
+            void this.runWithMaintenanceLock('vacuum', async () => {
+                this.logger.debug('[SERIES STORAGE] Running scheduled DuckDB VACUUM');
+                await this.runSql('VACUUM;');
+            }).catch(err => {
+                this.logger.error(`[SERIES STORAGE] VACUUM failed: ${err?.message ?? err}`);
+            });
+        }, DuckDbParquetStorageService.EIGHT_HOURS_INTERVAL);
+        this.vacuumJob.unref?.();
+    }
+    /**
+     * Stops the scheduled VACUUM job if running.
+     */
+    stopVacuumJob() {
+        if (this.vacuumJob) {
+            clearInterval(this.vacuumJob);
+            this.vacuumJob = null;
+        }
+    }
+    /**
+     * Starts the prune job for expired and orphaned samples.
+     */
+    startPruneJob() {
+        this.stopPruneJob();
+        if (!this.isDuckDbParquetReady() || !this.connection)
+            return;
+        this.pruneJob = setInterval(async () => {
+            if (this.shouldSkipMaintenance()) {
+                return;
+            }
+            try {
+                await this.runWithMaintenanceLock('prune', async () => {
+                    this.logger.debug('[SERIES STORAGE] Running scheduled prune of expired and orphaned samples');
+                    const expired = await this.pruneExpiredSamples(Date.now(), this.lifecycleToken);
+                    const orphaned = await this.pruneOrphanedSamples(this.lifecycleToken);
+                    this.logger.debug(`[SERIES STORAGE] Pruned ${expired} expired and ${orphaned} orphaned samples`);
+                });
+            }
+            catch (err) {
+                this.logger.error(`[SERIES STORAGE] Prune failed: ${err?.message ?? err}`);
+            }
+        }, DuckDbParquetStorageService.FOUR_HOURS_INTERVAL);
+        this.pruneJob.unref?.();
+    }
+    /**
+     * Stops the scheduled prune job if running.
+     */
+    stopPruneJob() {
+        if (this.pruneJob) {
+            clearInterval(this.pruneJob);
+            this.pruneJob = null;
+        }
+    }
+    /**
+     * Starts the scheduled job to delete series not reconciled in the last 6 months.
+     */
+    startStaleSeriesCleanupJob() {
+        this.stopStaleSeriesCleanupJob();
+        if (!this.isDuckDbParquetReady() || !this.connection)
+            return;
+        this.staleSeriesCleanupJob = setInterval(async () => {
+            if (this.shouldSkipMaintenance()) {
+                return;
+            }
+            try {
+                await this.runWithMaintenanceLock('stale-cleanup', async () => {
+                    const cutoff = Date.now() - DuckDbParquetStorageService.STALE_SERIES_AGE_MS;
+                    this.logger.debug(`[SERIES STORAGE] Running scheduled stale series cleanup (cutoff: ${new Date(cutoff).toISOString()})`);
+                    const deleted = await this.deleteStaleSeries(cutoff);
+                    if (deleted > 0) {
+                        this.logger.debug(`[SERIES STORAGE] Deleted ${deleted} series not reconciled in the last 6 months`);
+                    }
+                });
+            }
+            catch (err) {
+                this.logger.error(`[SERIES STORAGE] Stale series cleanup failed: ${err?.message ?? err}`);
+            }
+        }, DuckDbParquetStorageService.EIGHT_HOURS_INTERVAL);
+        this.staleSeriesCleanupJob.unref?.();
+    }
+    /**
+     * Deletes series not reconciled since the given cutoff timestamp.
+     * @param {number} cutoffMs - Milliseconds since epoch; series with reconcile_ts < cutoffMs will be deleted.
+     * @returns {Promise<number>} Number of deleted series.
+     *
+     * @example
+     * const deleted = await storage.deleteStaleSeries(Date.now() - 180 * 24 * 60 * 60 * 1000);
+     */
+    async deleteStaleSeries(cutoffMs) {
+        if (!this.isDuckDbParquetEnabled() || !this.connection) {
+            return 0;
+        }
+        // Find series to delete
+        const rows = await this.querySql(`
+      SELECT series_id FROM history_series
+      WHERE reconcile_ts IS NULL OR reconcile_ts < ${Math.trunc(cutoffMs)}
+    `);
+        const ids = rows.map(r => r.series_id);
+        if (ids.length === 0)
+            return 0;
+        for (const id of ids) {
+            await this.deleteSeriesDefinition(id);
+        }
+        return ids.length;
+    }
+    /**
+     * Stops the scheduled stale series cleanup job if running.
+     */
+    stopStaleSeriesCleanupJob() {
+        if (this.staleSeriesCleanupJob) {
+            clearInterval(this.staleSeriesCleanupJob);
+            this.staleSeriesCleanupJob = null;
+        }
+    }
+    shouldSkipMaintenance() {
+        if (!this.isDuckDbParquetReady() || !this.connection) {
+            return true;
+        }
+        if (this.maintenanceInProgress || this.flushInProgress) {
+            return true;
+        }
+        if (this.pendingRows.length > 0) {
+            return true;
+        }
+        return false;
+    }
+    async runWithMaintenanceLock(label, task) {
+        if (this.maintenanceInProgress) {
+            this.logger.debug(`[SERIES STORAGE] Skipping ${label} (maintenance already running)`);
+            return;
+        }
+        this.maintenanceInProgress = true;
+        const startedAt = Date.now();
+        try {
+            await task();
+            const elapsedMs = Date.now() - startedAt;
+            this.logger.debug(`[SERIES STORAGE] ${label} completed in ${elapsedMs}ms`);
+        }
+        finally {
+            this.maintenanceInProgress = false;
+        }
+    }
+    /**
+     * Returns the active storage configuration.
+     *
+     * @returns {IDuckDbParquetStorageConfig} Current storage configuration.
+     *
+     * @example
+     * const cfg = storage.getConfig();
+     */
+    getConfig() {
+        return this.config;
+    }
+    /**
+     * Returns last DuckDB initialization error when initialization failed.
+     *
+     * @returns {string | null} Initialization error text or null.
+     *
+     * @example
+     * const err = storage.getLastInitError();
+     */
+    getLastInitError() {
+        return this.lastInitError;
+    }
+    /**
+     * Indicates whether DuckDB/Parquet mode is selected.
+     *
+     * @returns {boolean} True when the selected engine is `duckdb-parquet`.
+     *
+     * @example
+     * if (storage.isDuckDbParquetEnabled()) {
+     *   console.log('DuckDB mode enabled');
+     * }
+     */
+    isDuckDbParquetEnabled() {
+        return this.config.engine === 'duckdb-parquet';
+    }
+    /**
+     * Indicates whether DuckDB/Parquet mode is initialized and ready.
+     *
+     * @returns {boolean} True when DuckDB mode is selected and an active connection exists.
+     *
+     * @example
+     * if (storage.isDuckDbParquetReady()) {
+     *   console.log('DuckDB ready');
+     * }
+     */
+    isDuckDbParquetReady() {
+        return this.isDuckDbParquetEnabled() && this.initialized && this.connection !== null;
+    }
+    /**
+     * Returns the current storage lifecycle token.
+     *
+     * The token changes whenever a new initialization attempt starts and can be
+     * used by callers to scope async stop operations (flush/close) so stale work
+     * does not affect a newer startup session.
+     *
+     * @returns {number} Current lifecycle token.
+     *
+     * @example
+     * const token = storage.getLifecycleToken();
+     */
+    getLifecycleToken() {
+        return this.lifecycleToken;
+    }
+    /**
+     * Adds a captured sample row to the pending storage queue.
+     *
+     * @param {IRecordedSample} sample Captured sample metadata and value.
+     * @returns {void}
+     *
+     * @example
+     * storage.enqueueSample(sample);
+     */
+    enqueueSample(sample) {
+        if (!this.isDuckDbParquetReady()) {
+            return;
+        }
+        this.pendingRows.push(sample);
+        const rangeKey = sample.seriesId;
+        const existing = this.pendingRangesBySeriesId.get(rangeKey);
+        if (!existing) {
+            this.pendingRangesBySeriesId.set(rangeKey, {
+                seriesId: sample.seriesId,
+                minTs: sample.timestamp,
+                maxTs: sample.timestamp
+            });
+            return;
+        }
+        this.pendingRangesBySeriesId.set(rangeKey, {
+            seriesId: existing.seriesId,
+            minTs: Math.min(existing.minTs, sample.timestamp),
+            maxTs: Math.max(existing.maxTs, sample.timestamp)
+        });
+    }
+    /**
+     * Flushes queued samples into DuckDB and exports changed ranges to Parquet chunks.
+     *
+     * @returns {Promise<{inserted: number; exported: number}>} Number of inserted rows and exported parquet files.
+     *
+     * @example
+     * const result = await storage.flush();
+     */
+    async flush(expectedLifecycleToken) {
+        if (expectedLifecycleToken !== undefined && expectedLifecycleToken !== this.lifecycleToken) {
+            return { inserted: 0, exported: 0 };
+        }
+        if (!this.isDuckDbParquetEnabled() || !this.connection || this.pendingRows.length === 0) {
+            return { inserted: 0, exported: 0 };
+        }
+        if (this.flushInProgress) {
+            return { inserted: 0, exported: 0 };
+        }
+        this.flushInProgress = true;
+        const rows = this.pendingRows;
+        const ranges = new Map(this.pendingRangesBySeriesId);
+        this.pendingRows = [];
+        this.pendingRangesBySeriesId.clear();
+        const startedAt = Date.now();
+        try {
+            await this.insertRows(rows);
+            let exported = 0;
+            for (const range of ranges.values()) {
+                await this.exportSeriesRange(range.seriesId, range.minTs, range.maxTs);
+                exported += 1;
+            }
+            const elapsedMs = Date.now() - startedAt;
+            this.logger.debug(`[SERIES STORAGE] flush inserted=${rows.length} exported=${exported} durationMs=${elapsedMs}`);
+            return { inserted: rows.length, exported };
+        }
+        catch (error) {
+            this.pendingRows = [...rows, ...this.pendingRows];
+            ranges.forEach((range, rangeKey) => {
+                const current = this.pendingRangesBySeriesId.get(rangeKey);
+                if (!current) {
+                    this.pendingRangesBySeriesId.set(rangeKey, range);
+                    return;
+                }
+                this.pendingRangesBySeriesId.set(rangeKey, {
+                    seriesId: current.seriesId,
+                    minTs: Math.min(current.minTs, range.minTs),
+                    maxTs: Math.max(current.maxTs, range.maxTs)
+                });
+            });
+            throw error;
+        }
+        finally {
+            this.flushInProgress = false;
+        }
+    }
+    /**
+     * Returns persisted series definitions from DuckDB.
+     *
+     * @returns {Promise<ISeriesDefinition[]>} Stored series definitions.
+     *
+     * @example
+     * const series = await storage.getSeriesDefinitions();
+     */
+    async getSeriesDefinitions() {
+        if (!this.isDuckDbParquetEnabled() || !this.connection) {
+            return [];
+        }
+        const rows = await this.querySql(`
+      SELECT
+        series_id,
+        dataset_uuid,
+        owner_widget_uuid,
+        owner_widget_selector,
+        path,
+        source,
+        context,
+        time_scale,
+        period,
+        retention_duration_ms,
+        sample_time,
+        enabled,
+        methods_json
+      FROM history_series
+      ORDER BY series_id ASC
+    `);
+        return rows.map(row => ({
+            seriesId: row.series_id,
+            datasetUuid: row.dataset_uuid,
+            ownerWidgetUuid: row.owner_widget_uuid,
+            ownerWidgetSelector: row.owner_widget_selector ?? undefined,
+            path: row.path,
+            source: row.source ?? undefined,
+            context: row.context ?? undefined,
+            timeScale: row.time_scale ?? undefined,
+            period: this.toNumberOrUndefined(row.period),
+            retentionDurationMs: this.toNumberOrUndefined(row.retention_duration_ms),
+            sampleTime: this.toNumberOrUndefined(row.sample_time),
+            enabled: this.toBoolean(row.enabled),
+            methods: this.parseMethods(row.methods_json)
+        }));
+    }
+    /**
+     * Persists one series definition in DuckDB.
+     *
+     * @param {ISeriesDefinition} series Series definition to persist.
+     * @returns {Promise<void>}
+     *
+     * @example
+     * await storage.upsertSeriesDefinition(series);
+     */
+    async upsertSeriesDefinition(series) {
+        if (!this.isDuckDbParquetEnabled() || !this.connection) {
+            return;
+        }
+        await this.runSql(`DELETE FROM history_series WHERE series_id = ${this.escape(series.seriesId)}`);
+        await this.runSql(`
+      INSERT INTO history_series (
+        series_id,
+        dataset_uuid,
+        owner_widget_uuid,
+        owner_widget_selector,
+        path,
+        source,
+        context,
+        time_scale,
+        period,
+        retention_duration_ms,
+        sample_time,
+        enabled,
+        methods_json
+      ) VALUES (
+        ${this.escape(series.seriesId)},
+        ${this.escape(series.datasetUuid)},
+        ${this.escape(series.ownerWidgetUuid)},
+        ${this.nullableString(series.ownerWidgetSelector)},
+        ${this.escape(series.path)},
+        ${this.nullableString(series.source)},
+        ${this.nullableString(series.context)},
+        ${this.nullableString(series.timeScale)},
+        ${this.nullableNumber(series.period)},
+        ${this.nullableNumber(series.retentionDurationMs)},
+        ${this.nullableNumber(series.sampleTime)},
+        ${series.enabled === false ? 'FALSE' : 'TRUE'},
+        ${this.nullableString(series.methods ? JSON.stringify(series.methods) : null)}
+      )
+    `);
+    }
+    /**
+     * Deletes one persisted series definition in DuckDB.
+     *
+     * @param {string} seriesId Series identifier.
+     * @returns {Promise<void>}
+     *
+     * @example
+     * await storage.deleteSeriesDefinition('series-1');
+     */
+    async deleteSeriesDefinition(seriesId) {
+        if (!this.isDuckDbParquetEnabled() || !this.connection) {
+            return;
+        }
+        await this.runSql(`DELETE FROM history_series WHERE series_id = ${this.escape(seriesId)}`);
+    }
+    /**
+     * Replaces persisted series definitions with desired set.
+     *
+     * @param {ISeriesDefinition[]} series Full desired series set.
+     * @returns {Promise<void>}
+     *
+     * @example
+     * await storage.replaceSeriesDefinitions(series);
+     */
+    async replaceSeriesDefinitions(series) {
+        if (!this.isDuckDbParquetEnabled() || !this.connection) {
+            return;
+        }
+        await this.runSql('DELETE FROM history_series');
+        for (const item of series) {
+            await this.upsertSeriesDefinition(item);
+        }
+    }
+    /**
+     * Removes persisted samples that are older than each series retention window.
+     *
+     * @param {number} [nowMs=Date.now()] Current timestamp in milliseconds used to compute per-series cutoffs.
+     * @param {number} [expectedLifecycleToken] Optional lifecycle token guard to skip stale sweeps.
+     * @returns {Promise<number>} Number of deleted sample rows.
+     *
+     * @example
+     * const removed = await storage.pruneExpiredSamples();
+     */
+    async pruneExpiredSamples(nowMs = Date.now(), expectedLifecycleToken) {
+        if (expectedLifecycleToken !== undefined && expectedLifecycleToken !== this.lifecycleToken) {
+            return 0;
+        }
+        if (!this.isDuckDbParquetEnabled() || !this.connection) {
+            return 0;
+        }
+        const anchorMs = Math.trunc(Number.isFinite(nowMs) ? nowMs : Date.now());
+        const whereClause = `
+      EXISTS (
+        SELECT 1
+        FROM history_series AS hs
+        WHERE hs.series_id = history_samples.series_id
+          AND hs.retention_duration_ms IS NOT NULL
+          AND hs.retention_duration_ms > 0
+          AND history_samples.ts_ms < (${anchorMs} - hs.retention_duration_ms)
+      )
+    `;
+        let removedRows = 0;
+        while (true) {
+            const batch = await this.querySql(`
+        SELECT rowid
+        FROM history_samples
+        WHERE ${whereClause}
+        LIMIT ${DuckDbParquetStorageService.PRUNE_BATCH_SIZE}
+      `);
+            if (batch.length === 0) {
+                break;
+            }
+            const rowIds = batch.map(row => Math.trunc(Number(row.rowid))).filter(Number.isFinite);
+            if (rowIds.length === 0) {
+                break;
+            }
+            await this.runSql(`
+        DELETE FROM history_samples
+        WHERE rowid IN (${rowIds.join(', ')})
+      `);
+            removedRows += rowIds.length;
+            if (rowIds.length < DuckDbParquetStorageService.PRUNE_BATCH_SIZE) {
+                break;
+            }
+        }
+        return removedRows;
+    }
+    /**
+     * Removes persisted samples that no longer have a matching series definition.
+     *
+     * @param {number} [expectedLifecycleToken] Optional lifecycle token guard to skip stale sweeps.
+     * @returns {Promise<number>} Number of deleted orphan sample rows.
+     *
+     * @example
+     * const removed = await storage.pruneOrphanedSamples();
+     */
+    async pruneOrphanedSamples(expectedLifecycleToken) {
+        if (expectedLifecycleToken !== undefined && expectedLifecycleToken !== this.lifecycleToken) {
+            return 0;
+        }
+        if (!this.isDuckDbParquetEnabled() || !this.connection) {
+            return 0;
+        }
+        const whereClause = `
+      NOT EXISTS (
+        SELECT 1
+        FROM history_series AS hs
+        WHERE hs.series_id = history_samples.series_id
+      )
+    `;
+        let removedRows = 0;
+        while (true) {
+            const batch = await this.querySql(`
+        SELECT rowid
+        FROM history_samples
+        WHERE ${whereClause}
+        LIMIT ${DuckDbParquetStorageService.PRUNE_BATCH_SIZE}
+      `);
+            if (batch.length === 0) {
+                break;
+            }
+            const rowIds = batch.map(row => Math.trunc(Number(row.rowid))).filter(Number.isFinite);
+            if (rowIds.length === 0) {
+                break;
+            }
+            await this.runSql(`
+        DELETE FROM history_samples
+        WHERE rowid IN (${rowIds.join(', ')})
+      `);
+            removedRows += rowIds.length;
+            if (rowIds.length < DuckDbParquetStorageService.PRUNE_BATCH_SIZE) {
+                break;
+            }
+        }
+        return removedRows;
+    }
+    /**
+     * Lists known history paths from persisted samples.
+     *
+     * @returns {Promise<string[]>} Ordered path names.
+     *
+     * @example
+     * const paths = await storage.getStoredPaths();
+     */
+    async getStoredPaths(query) {
+        if (!this.isDuckDbParquetEnabled() || !this.connection) {
+            return [];
+        }
+        const nowMs = Date.now();
+        const range = this.resolveRange(nowMs, query?.from, query?.to, query?.duration);
+        const rows = await this.querySql(`
+      SELECT DISTINCT path AS value
+      FROM history_samples
+      WHERE path IS NOT NULL
+        AND ts_ms >= ${Math.trunc(range.fromMs)}
+        AND ts_ms <= ${Math.trunc(range.toMs)}
+      ORDER BY value ASC
+    `);
+        return rows.map(row => row.value).filter(Boolean);
+    }
+    /**
+     * Lists known history contexts from persisted samples.
+     *
+     * @returns {Promise<string[]>} Ordered context names.
+     *
+     * @example
+     * const contexts = await storage.getStoredContexts();
+     */
+    async getStoredContexts(query) {
+        if (!this.isDuckDbParquetEnabled() || !this.connection) {
+            return [];
+        }
+        const nowMs = Date.now();
+        const range = this.resolveRange(nowMs, query?.from, query?.to, query?.duration);
+        const rows = await this.querySql(`
+      SELECT DISTINCT context AS value
+      FROM history_samples
+      WHERE context IS NOT NULL
+        AND ts_ms >= ${Math.trunc(range.fromMs)}
+        AND ts_ms <= ${Math.trunc(range.toMs)}
+      ORDER BY value ASC
+    `);
+        return rows.map(row => row.value).filter(Boolean);
+    }
+    /**
+     * Queries history values directly from DuckDB in History API-compatible shape.
+     *
+     * @param {IHistoryQueryParams} query Incoming history values query parameters.
+     * @returns {Promise<IHistoryValuesResponse | null>} History payload when DuckDB is ready, otherwise null.
+     *
+     * @example
+     * const result = await storage.getValues({ paths: 'navigation.speedOverGround:avg', duration: 'PT1H' });
+     */
+    async getValues(query) {
+        if (!this.isDuckDbParquetEnabled() || !this.connection) {
+            return null;
+        }
+        const nowMs = Date.now();
+        const requested = this.parseRequestedPaths(query.paths);
+        if (requested.length === 0) {
+            return null;
+        }
+        const range = this.resolveRange(nowMs, query.from, query.to, query.duration);
+        const context = query.context ?? 'vessels.self';
+        const resolutionMs = this.resolveResolutionMs(query.resolution);
+        const uniquePaths = Array.from(new Set(requested.map(item => item.path)));
+        const rowsByPath = await this.selectRowsForPaths(uniquePaths, context, range.fromMs, range.toMs);
+        const timestampRows = new Map();
+        for (let index = 0; index < requested.length; index += 1) {
+            const item = requested[index];
+            const rows = rowsByPath.get(item.path) ?? [];
+            const transformed = this.applyMethod(item, rows);
+            const merged = this.downsampleIfNeeded(transformed, resolutionMs, item.method ?? 'avg');
+            merged.forEach(entry => {
+                const row = timestampRows.get(entry.timestamp) ?? Array.from({ length: requested.length }, () => null);
+                row[index] = entry.value;
+                timestampRows.set(entry.timestamp, row);
+            });
+        }
+        const data = Array.from(timestampRows.entries())
+            .sort((left, right) => left[0] - right[0])
+            .map(([timestamp, values]) => [new Date(timestamp).toISOString(), ...values]);
+        return {
+            context,
+            range: {
+                from: new Date(range.fromMs).toISOString(),
+                to: new Date(range.toMs).toISOString()
+            },
+            values: requested.map(item => ({
+                path: item.path,
+                method: item.method ?? 'avg'
+            })),
+            data
+        };
+    }
+    /**
+     * Closes open storage resources.
+     *
+     * @returns {Promise<void>}
+     *
+     * @example
+     * await storage.close();
+     */
+    async close(expectedLifecycleToken) {
+        if (expectedLifecycleToken !== undefined && expectedLifecycleToken !== this.lifecycleToken) {
+            return;
+        }
+        this.initialized = false;
+        this.stopPruneJob();
+        this.stopStaleSeriesCleanupJob();
+        if (!this.connection) {
+            this.db = null;
+            return;
+        }
+        const connection = this.connection;
+        this.connection = null;
+        await new Promise((resolvePromise) => {
+            connection.close(() => resolvePromise());
+        });
+        this.db = null;
+    }
+    async createCoreTables() {
+        await this.runSql(`
+      CREATE TABLE IF NOT EXISTS history_samples (
+        series_id VARCHAR,
+        dataset_uuid VARCHAR,
+        owner_widget_uuid VARCHAR,
+        path VARCHAR,
+        context VARCHAR,
+        source VARCHAR,
+        ts_ms BIGINT,
+        value DOUBLE
+      )
+    `);
+        await this.runSql(`
+      CREATE TABLE IF NOT EXISTS history_series (
+        series_id VARCHAR NOT NULL,
+        dataset_uuid VARCHAR NOT NULL,
+        owner_widget_uuid VARCHAR NOT NULL,
+        owner_widget_selector VARCHAR,
+        path VARCHAR NOT NULL,
+        source VARCHAR,
+        context VARCHAR,
+        time_scale VARCHAR,
+        period INTEGER,
+        retention_duration_ms BIGINT,
+        sample_time INTEGER,
+        enabled BOOLEAN,
+        methods_json VARCHAR,
+        reconcile_ts BIGINT,
+        PRIMARY KEY (series_id)
+      )
+    `);
+    }
+    async countRows(tableName) {
+        const rows = await this.querySql(`SELECT COUNT(*) AS removed_rows FROM ${tableName}`);
+        return this.toNumberOrUndefined(rows[0]?.removed_rows) ?? 0;
+    }
+    async insertRows(rows) {
+        if (rows.length === 0) {
+            return;
+        }
+        const valuesSql = rows
+            .map(sample => `(${this.escape(sample.seriesId)}, ${this.escape(sample.datasetUuid)}, ${this.escape(sample.ownerWidgetUuid)}, ${this.escape(sample.path)}, ${this.escape(sample.context)}, ${this.escape(sample.source)}, ${Math.trunc(sample.timestamp)}, ${Number(sample.value)})`)
+            .join(',\n');
+        const sql = `
+      INSERT INTO history_samples (
+        series_id,
+        dataset_uuid,
+        owner_widget_uuid,
+        path,
+        context,
+        source,
+        ts_ms,
+        value
+      ) VALUES ${valuesSql}
+    `;
+        await this.runSql(sql);
+    }
+    async exportSeriesRange(seriesId, fromMs, toMs) {
+        const baseDir = (0, path_1.resolve)(this.config.parquetDirectory);
+        const seriesDir = (0, path_1.join)(baseDir, this.safePath(seriesId));
+        (0, fs_1.mkdirSync)(seriesDir, { recursive: true });
+        const filePath = (0, path_1.join)(seriesDir, `${fromMs}-${toMs}.parquet`);
+        const escapedSeries = this.escape(seriesId);
+        const escapedFile = this.escapePath(filePath);
+        const sql = `
+      COPY (
+        SELECT
+          series_id,
+          dataset_uuid,
+          owner_widget_uuid,
+          path,
+          context,
+          source,
+          ts_ms,
+          to_timestamp(ts_ms / 1000.0) AS ts,
+          value
+        FROM history_samples
+        WHERE series_id = ${escapedSeries}
+          AND ts_ms >= ${Math.trunc(fromMs)}
+          AND ts_ms <= ${Math.trunc(toMs)}
+        ORDER BY ts_ms
+      ) TO '${escapedFile}' (FORMAT PARQUET)
+    `;
+        await this.runSql(sql);
+    }
+    async runSql(sql) {
+        if (!this.connection) {
+            throw new Error('DuckDB connection is not initialized');
+        }
+        await new Promise((resolvePromise, rejectPromise) => {
+            this.connection?.run(sql, (error) => {
+                if (error) {
+                    rejectPromise(error);
+                    return;
+                }
+                resolvePromise();
+            });
+        });
+    }
+    async querySql(sql) {
+        if (!this.connection) {
+            throw new Error('DuckDB connection is not initialized');
+        }
+        return new Promise((resolvePromise, rejectPromise) => {
+            this.connection?.all(sql, (error, rows) => {
+                if (error) {
+                    rejectPromise(error);
+                    return;
+                }
+                resolvePromise((rows ?? []));
+            });
+        });
+    }
+    async selectRowsForPaths(paths, context, fromMs, toMs) {
+        const rowsByPath = new Map();
+        if (paths.length === 0) {
+            return rowsByPath;
+        }
+        const pathFilter = paths.map(path => this.escape(path)).join(', ');
+        const sql = `
+      SELECT path, ts_ms, value
+      FROM history_samples
+      WHERE context = ${this.escape(context)}
+        AND path IN (${pathFilter})
+        AND ts_ms >= ${Math.trunc(fromMs)}
+        AND ts_ms <= ${Math.trunc(toMs)}
+      ORDER BY path ASC, ts_ms ASC
+    `;
+        const rows = await this.querySql(sql);
+        rows.forEach(row => {
+            const list = rowsByPath.get(row.path) ?? [];
+            list.push({ ts_ms: Number(row.ts_ms), value: Number(row.value) });
+            rowsByPath.set(row.path, list);
+        });
+        return rowsByPath;
+    }
+    parseRequestedPaths(paths) {
+        return String(paths)
+            .split(',')
+            .map(item => item.trim())
+            .filter(Boolean)
+            .map(raw => {
+            const [pathToken, maybeMethod, maybePeriod] = raw.split(':');
+            const path = this.normalizePathIdentifier(pathToken);
+            const method = this.parseMethod(maybeMethod);
+            const parsedPeriod = maybePeriod !== undefined ? Number(maybePeriod) : undefined;
+            return {
+                path,
+                method,
+                period: Number.isFinite(parsedPeriod) ? parsedPeriod : undefined
+            };
+        });
+    }
+    parseMethod(value) {
+        if (!value)
+            return undefined;
+        const normalized = value.toLowerCase();
+        if (normalized === 'min' || normalized === 'max' || normalized === 'avg' || normalized === 'sma' || normalized === 'ema') {
+            return normalized;
+        }
+        return undefined;
+    }
+    normalizePathIdentifier(path) {
+        const trimmed = String(path).trim();
+        if (!trimmed) {
+            return '';
+        }
+        if (trimmed.startsWith('vessels.self.')) {
+            return trimmed.slice('vessels.self.'.length);
+        }
+        if (trimmed.startsWith('self.')) {
+            return trimmed.slice('self.'.length);
+        }
+        return trimmed;
+    }
+    resolveRange(nowMs, from, to, duration) {
+        const toMs = to ? Date.parse(to) : nowMs;
+        if (!Number.isFinite(toMs)) {
+            throw new Error('Invalid to date-time. Expected an ISO 8601 date-time string.');
+        }
+        const fromMs = from ? Date.parse(from) : toMs - this.parseDurationMs(duration);
+        if (!Number.isFinite(fromMs)) {
+            throw new Error('Invalid from date-time. Expected an ISO 8601 date-time string.');
+        }
+        return { fromMs, toMs };
+    }
+    parseDurationMs(duration) {
+        if (duration === undefined || duration === null) {
+            return 60 * 60_000;
+        }
+        if (typeof duration === 'number' && Number.isFinite(duration) && duration > 0) {
+            return duration;
+        }
+        if (typeof duration === 'number') {
+            throw new Error('Invalid duration. Expected a positive number of milliseconds or an ISO 8601 duration (e.g. PT10M).');
+        }
+        const value = String(duration).trim();
+        if (/^\d+(\.\d+)?$/.test(value)) {
+            const parsedMs = Number(value);
+            if (Number.isFinite(parsedMs) && parsedMs > 0) {
+                return parsedMs;
+            }
+            throw new Error('Invalid duration. Expected a positive number of milliseconds or an ISO 8601 duration (e.g. PT10M).');
+        }
+        const iso = /^PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?$/i.exec(value);
+        if (!iso) {
+            throw new Error('Invalid duration. Expected a positive number of milliseconds or an ISO 8601 duration (e.g. PT10M).');
+        }
+        const hours = Number(iso[1] || 0);
+        const minutes = Number(iso[2] || 0);
+        const seconds = Number(iso[3] || 0);
+        const totalMs = (((hours * 60) + minutes) * 60 + seconds) * 1000;
+        if (totalMs <= 0) {
+            throw new Error('Invalid duration. Expected a positive number of milliseconds or an ISO 8601 duration (e.g. PT10M).');
+        }
+        return totalMs;
+    }
+    resolveResolutionMs(resolution) {
+        if (resolution === undefined || resolution === null) {
+            return 0;
+        }
+        if (typeof resolution === 'number') {
+            if (!Number.isFinite(resolution) || resolution <= 0) {
+                throw new Error('Invalid resolution. Expected a positive number of seconds or an ISO 8601 duration (e.g. PT10S).');
+            }
+            return Math.max(1, Math.trunc(resolution * 1000));
+        }
+        const value = String(resolution).trim();
+        if (!value) {
+            throw new Error('Invalid resolution. Expected a positive number of seconds or an ISO 8601 duration (e.g. PT10S).');
+        }
+        if (/^\d+(\.\d+)?$/.test(value)) {
+            const parsedSeconds = Number(value);
+            if (!Number.isFinite(parsedSeconds) || parsedSeconds <= 0) {
+                throw new Error('Invalid resolution. Expected a positive number of seconds or an ISO 8601 duration (e.g. PT10S).');
+            }
+            return Math.max(1, Math.trunc(parsedSeconds * 1000));
+        }
+        try {
+            return this.parseDurationMs(value);
+        }
+        catch {
+            throw new Error('Invalid resolution. Expected a positive number of seconds or an ISO 8601 duration (e.g. PT10S).');
+        }
+    }
+    applyMethod(request, rows) {
+        if (rows.length === 0)
+            return [];
+        const method = request.method ?? 'avg';
+        if (method === 'min' || method === 'max' || method === 'avg') {
+            return rows.map(entry => ({ timestamp: Number(entry.ts_ms), value: Number(entry.value) }));
+        }
+        if (method === 'sma') {
+            const period = Math.max(1, request.period ?? 5);
+            return rows.map((entry, index) => {
+                const start = Math.max(0, index - period + 1);
+                const window = rows.slice(start, index + 1);
+                const sum = window.reduce((acc, item) => acc + Number(item.value), 0);
+                return {
+                    timestamp: Number(entry.ts_ms),
+                    value: sum / window.length
+                };
+            });
+        }
+        const period = Math.max(1, request.period ?? 5);
+        const multiplier = 2 / (period + 1);
+        let previous = null;
+        return rows.map(entry => {
+            const value = Number(entry.value);
+            if (previous === null) {
+                previous = value;
+            }
+            else {
+                previous = ((value - previous) * multiplier) + previous;
+            }
+            return {
+                timestamp: Number(entry.ts_ms),
+                value: previous
+            };
+        });
+    }
+    downsampleIfNeeded(values, resolutionMs, method) {
+        if (resolutionMs <= 0 || values.length === 0) {
+            return values;
+        }
+        const buckets = new Map();
+        values.forEach(entry => {
+            if (!Number.isFinite(entry.value)) {
+                return;
+            }
+            const bucket = Math.floor(entry.timestamp / resolutionMs) * resolutionMs;
+            const list = buckets.get(bucket) ?? [];
+            list.push(entry.value);
+            buckets.set(bucket, list);
+        });
+        return Array.from(buckets.entries())
+            .sort((left, right) => left[0] - right[0])
+            .map(([timestamp, list]) => {
+            const value = this.aggregateBucket(list, method);
+            return {
+                timestamp,
+                value
+            };
+        });
+    }
+    aggregateBucket(values, method) {
+        if (values.length === 0) {
+            return null;
+        }
+        if (method === 'min') {
+            return Math.min(...values);
+        }
+        if (method === 'max') {
+            return Math.max(...values);
+        }
+        const sum = values.reduce((acc, value) => acc + value, 0);
+        return sum / values.length;
+    }
+    safePath(value) {
+        return value.replace(/[^a-zA-Z0-9._-]/g, '_');
+    }
+    escape(value) {
+        return `'${String(value).replace(/'/g, "''")}'`;
+    }
+    escapePath(value) {
+        return String(value).replace(/'/g, "''");
+    }
+    nullableString(value) {
+        if (value === undefined || value === null || value === '') {
+            return 'NULL';
+        }
+        return this.escape(value);
+    }
+    nullableNumber(value) {
+        if (value === undefined || value === null || !Number.isFinite(value)) {
+            return 'NULL';
+        }
+        return String(Math.trunc(value));
+    }
+    parseMethods(value) {
+        if (!value)
+            return undefined;
+        try {
+            const parsed = JSON.parse(value);
+            if (!Array.isArray(parsed))
+                return undefined;
+            const methods = parsed.filter(entry => entry === 'min' || entry === 'max' || entry === 'avg' || entry === 'sma' || entry === 'ema');
+            return methods.length > 0 ? methods : undefined;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    toNumberOrUndefined(value) {
+        if (value === undefined || value === null) {
+            return undefined;
+        }
+        if (typeof value === 'bigint') {
+            return Number(value);
+        }
+        const parsed = Number(value);
+        return Number.isFinite(parsed) ? parsed : undefined;
+    }
+    toBoolean(value) {
+        if (typeof value === 'boolean') {
+            return value;
+        }
+        if (typeof value === 'bigint') {
+            return value !== 0n;
+        }
+        if (typeof value === 'number') {
+            return value !== 0;
+        }
+        if (typeof value === 'string') {
+            const normalized = value.trim().toLowerCase();
+            if (normalized === 'true' || normalized === '1') {
+                return true;
+            }
+            if (normalized === 'false' || normalized === '0') {
+                return false;
+            }
+        }
+        return Boolean(value);
+    }
+}
+exports.DuckDbParquetStorageService = DuckDbParquetStorageService;