@mxtommy/kip 4.5.0-beta.1 → 4.5.1

package/plugin/index.js

@@ -34,14 +34,21 @@ var __importStar = (this && this.__importStar) || (function () {
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
 const server_api_1 = require("@signalk/server-api");
+const module_1 = require("module");
 const openapi = __importStar(require("./openApi.json"));
+const history_series_service_1 = require("./history-series.service");
+const duckdb_parquet_storage_service_1 = require("./duckdb-parquet-storage.service");
 const start = (server) => {
+    const packageRequire = (0, module_1.createRequire)(__filename);
     const mutableOpenApi = JSON.parse(JSON.stringify(openapi.default ?? openapi));
     const API_PATHS = {
         DISPLAYS: `/displays`,
         INSTANCE: `/displays/:displayId`,
         SCREEN_INDEX: `/displays/:displayId/screenIndex`,
-        ACTIVATE_SCREEN: `/displays/:displayId/activeScreen`
+        ACTIVATE_SCREEN: `/displays/:displayId/activeScreen`,
+        SERIES: '/series',
+        SERIES_INSTANCE: '/series/:seriesId',
+        SERIES_RECONCILE: '/series/reconcile',
     };
     const PUT_CONTEXT = 'vessels.self';
     const COMMAND_PATHS = {
@@ -50,25 +57,76 @@ const start = (server) => {
         REQUEST_ACTIVE_SCREEN: 'kip.remote.requestActiveScreen'
     };
     const CONFIG_SCHEMA = {
+        type: 'object',
+        title: 'Remote Control and Data Series',
+        description: 'NOTE: All plugin settings are also managed from within KIP\'s Display Options panel. Changes made here will be overridden when KIP applies settings from the Display Options.',
         properties: {
-            notifications: {
-                type: 'object',
-                title: 'Remote Control',
-                description: 'This plugin requires no configuration.'
+            historySeriesServiceEnabled: {
+                type: 'boolean',
+                title: 'Enable Automatic Historical Time-Series Capture and Management',
+                description: 'Historical Time-Series are data captures that supply the widget historical data and seed the Data Chart and Wind Trends widgets. If disabled, data capture must be configured in your chosen history provider plugin.',
+                default: true
+            },
+            registerAsHistoryApiProvider: {
+                type: 'boolean',
+                title: 'Enable Query Provider',
+                description: 'The built-in History-API provider is the querying engine that uses the Historical Time-Series data to seed the widget historical panel, the Data Chart and Wind Trends widgets. If you want to use another History-API provider, turn this off and configure your chosen History-API compatible provider accordingly and KIP will query that provider.',
+                default: true
             }
         }
     };
+    const historySeries = new history_series_service_1.HistorySeriesService(() => Date.now(), false);
+    const storageService = new duckdb_parquet_storage_service_1.DuckDbParquetStorageService();
+    let retentionSweepTimer = null;
+    let storageFlushTimer = null;
+    let duckDbInitializationPromise = null;
+    const DUCKDB_INIT_WAIT_TIMEOUT_MS = 5000;
+    let streamUnsubscribes = [];
+    let historyApiRegistry = null;
+    let historySeriesServiceEnabled = true;
+    let registerAsHistoryApiProvider = true;
+    let historyApiProviderRegistered = false;
+    function resolveDependencyIdentity(dependencyName) {
+        try {
+            const pkg = packageRequire(`${dependencyName}/package.json`);
+            const name = typeof pkg.name === 'string' && pkg.name.trim() ? pkg.name.trim() : dependencyName;
+            const version = typeof pkg.version === 'string' && pkg.version.trim() ? pkg.version.trim() : 'unknown';
+            return `${name}@${version}`;
+        }
+        catch {
+            return `${dependencyName}@unavailable`;
+        }
+    }
+    function logRuntimeDependencyVersions() {
+        const nodeIdentity = `node@${process.version}`;
+        const duckDbNodeIdentity = resolveDependencyIdentity('@duckdb/node-api');
+        const parquetIdentity = resolveDependencyIdentity('@dsnp/parquetjs');
+        server.debug(`[KIP][RUNTIME] ${nodeIdentity} duckdb=${duckDbNodeIdentity} parquet=${parquetIdentity}`);
+    }
+    function resolveHistoryModeConfig(settings) {
+        const root = (settings && typeof settings === 'object' ? settings : {});
+        const historySeriesServiceEnabledSetting = typeof root.historySeriesServiceEnabled === 'boolean'
+            ? root.historySeriesServiceEnabled
+            : undefined;
+        const registerAsHistoryApiProviderSetting = typeof root.registerAsHistoryApiProvider === 'boolean'
+            ? root.registerAsHistoryApiProvider
+            : undefined;
+        return {
+            historySeriesServiceEnabled: historySeriesServiceEnabledSetting !== false,
+            registerAsHistoryApiProvider: registerAsHistoryApiProviderSetting !== false
+        };
+    }
     // Helpers
     function getDisplaySelfPath(displayId, suffix) {
         const tail = suffix ? `.${suffix}` : '';
         const want = `displays.${displayId}${tail}`;
         const full = server.getSelfPath(want);
-        server.debug(`getDisplaySelfPath: displayId: ${displayId}, suffix: ${suffix}, want=${want}, fullPath=${JSON.stringify(full)}`);
+        server.debug(`[KIP][SELF_PATH] displayId=${displayId} suffix=${String(suffix ?? '')} requested=${want} resolved=${JSON.stringify(full)}`);
         return typeof full === 'object' && full !== null ? full : undefined;
     }
     function getAvailableDisplays() {
         const fullPath = server.getSelfPath('displays');
-        server.debug(`getAvailableDisplays: fullPath=${JSON.stringify(fullPath)}`);
+        server.debug(`[KIP][DISPLAYS] resolved=${JSON.stringify(fullPath)}`);
         return typeof fullPath === 'object' && fullPath !== null ? fullPath : undefined;
     }
     function sendOk(res, body) {
@@ -79,6 +137,92 @@ const start = (server) => {
     function sendFail(res, statusCode, message) {
         return res.status(statusCode).json({ state: 'FAILED', statusCode, message });
     }
+    function getHistorySeriesServiceDisabledMessage() {
+        return 'KIP history-series service is disabled by plugin configuration';
+    }
+    function getDuckDbUnavailableMessage() {
+        const details = storageService.getLastInitError();
+        return details
+            ? `DuckDB storage unavailable: ${details}`
+            : 'DuckDB storage unavailable';
+    }
+    function isDuckDbUnavailable() {
+        return Boolean(storageService.getLastInitError());
+    }
+    async function waitForDuckDbInitialization(timeoutMs = DUCKDB_INIT_WAIT_TIMEOUT_MS) {
+        if (!duckDbInitializationPromise) {
+            return storageService.isDuckDbParquetReady();
+        }
+        try {
+            const ready = await Promise.race([
+                duckDbInitializationPromise,
+                new Promise(resolvePromise => {
+                    setTimeout(() => resolvePromise(false), timeoutMs);
+                })
+            ]);
+            if (!ready && !storageService.isDuckDbParquetReady()) {
+                server.error(`[SERIES STORAGE] DuckDB initialization wait timed out after ${timeoutMs}ms`);
+            }
+            return ready;
+        }
+        catch {
+            return false;
+        }
+    }
+    function getRouteError(error, fallbackMessage) {
+        const message = String(error?.message || fallbackMessage);
+        const normalized = message.toLowerCase();
+        if (normalized.includes('invalid ')
+            || normalized.includes('missing ')
+            || normalized.includes('body must')
+            || normalized.includes('required')
+            || normalized.includes('expected an iso')) {
+            return { statusCode: 400, message };
+        }
+        if (normalized.includes('duckdb')
+            || normalized.includes('storage unavailable')
+            || normalized.includes('not initialized')
+            || isDuckDbUnavailable()) {
+            return { statusCode: 503, message };
+        }
+        return { statusCode: 500, message };
+    }
+    function findSeriesById(seriesId) {
+        const current = historySeries.findSeriesById(seriesId);
+        return current ? JSON.parse(JSON.stringify(current)) : null;
+    }
+    function isHistorySeriesServiceEnabled() {
+        return historySeriesServiceEnabled;
+    }
+    function isHistoryApiProviderEnabled() {
+        return registerAsHistoryApiProvider;
+    }
+    function logOperationalMode(stage) {
+        server.debug(`[HISTORY MODE] stage=${stage} historySeriesServiceEnabled=${isHistorySeriesServiceEnabled()} historyApiProviderEnabled=${isHistoryApiProviderEnabled()} historyApiProviderRegistered=${historyApiProviderRegistered}`);
+    }
+    async function ensureDuckDbReadyForRequest(res) {
+        await waitForDuckDbInitialization();
+        if (storageService.isDuckDbParquetReady()) {
+            return true;
+        }
+        sendFail(res, 503, getDuckDbUnavailableMessage());
+        return false;
+    }
+    function ensureHistorySeriesServiceEnabledForRequest(res) {
+        if (isHistorySeriesServiceEnabled()) {
+            return true;
+        }
+        sendFail(res, 503, getHistorySeriesServiceDisabledMessage());
+        return false;
+    }
+    function logAuthTrace(req, stage) {
+        const hasAuthorizationHeader = typeof req.headers.authorization === 'string' && req.headers.authorization.length > 0;
+        const hasCookieHeader = typeof req.headers.cookie === 'string' && req.headers.cookie.length > 0;
+        const origin = req.headers.origin ?? null;
+        const userAgent = req.headers['user-agent'] ?? null;
+        const contentType = req.headers['content-type'] ?? null;
+        server.debug(`[AUTH TRACE] stage=${stage} method=${req.method} path=${req.path} ip=${req.ip} origin=${String(origin)} authHeader=${hasAuthorizationHeader} cookieHeader=${hasCookieHeader} contentType=${String(contentType)} userAgent=${String(userAgent)}`);
+    }
     function completed(statusCode, message) {
         return { state: 'COMPLETED', statusCode, message };
     }
@@ -87,6 +231,7 @@ const start = (server) => {
     }
     function applyDisplayWrite(displayId, suffix, value) {
         const path = suffix ? `displays.${displayId}.${suffix}` : `displays.${displayId}`;
+        server.debug(`[KIP][WRITE] applyDisplayWrite path=${path} value=${JSON.stringify(value)}`);
         try {
             server.handleMessage(plugin.id, {
                 updates: [
@@ -100,14 +245,17 @@ const start = (server) => {
                     }
                 ]
             }, server_api_1.SKVersion.v1);
+            server.debug(`[KIP][WRITE] handleMessage success path=${path}`);
             return completed(200);
         }
         catch (error) {
             const message = error?.message ?? 'Unable to write display path';
+            server.error(`[WRITE TRACE] handleMessage failure path=${path} message=${message}`);
             return completed(400, message);
         }
     }
     function handleSetDisplay(value) {
+        server.debug(`[KIP][COMMAND] handleSetDisplay payload=${JSON.stringify(value)}`);
         const command = value;
         if (!command || typeof command !== 'object') {
             return completed(400, 'Command payload is required');
@@ -122,6 +270,7 @@ const start = (server) => {
         return applyDisplayWrite(command.displayId, null, displayValue);
     }
     function handleScreenWrite(value, suffix) {
+        server.debug(`[KIP][COMMAND] handleScreenWrite suffix=${suffix} payload=${JSON.stringify(value)}`);
         const command = value;
         if (!command || typeof command !== 'object') {
             return completed(400, 'Command payload is required');
@@ -136,46 +285,446 @@ const start = (server) => {
         return applyDisplayWrite(command.displayId, suffix, screenIdxValue);
     }
     function sendActionAsRest(res, result) {
+        server.debug(`[KIP][REST] sendActionAsRest statusCode=${result.statusCode} message=${result.message ?? ''}`);
         if (result.statusCode === 200) {
             return res.status(200).json({ state: 'SUCCESS', statusCode: 200 });
         }
         return sendFail(res, result.statusCode || 400, result.message || 'Command failed');
     }
+    function stopSeriesCapture() {
+        streamUnsubscribes.forEach(unsub => {
+            try {
+                unsub();
+            }
+            catch {
+                // ignore unsubscribe failures
+            }
+        });
+        streamUnsubscribes = [];
+    }
+    function toIsoString(value) {
+        if (typeof value === 'string' && value.trim()) {
+            return value;
+        }
+        if (value instanceof Date) {
+            return value.toISOString();
+        }
+        if (value && typeof value === 'object' && typeof value.toString === 'function') {
+            const serialized = value.toString();
+            return serialized && serialized !== '[object Object]' ? serialized : undefined;
+        }
+        return undefined;
+    }
+    function toDurationString(value) {
+        if (typeof value === 'number' && Number.isFinite(value)) {
+            return value;
+        }
+        if (typeof value === 'string' && value.trim()) {
+            return value;
+        }
+        if (value && typeof value === 'object' && typeof value.toString === 'function') {
+            const serialized = value.toString();
+            return serialized && serialized !== '[object Object]' ? serialized : undefined;
+        }
+        return undefined;
+    }
+    function normalizeHistoryMethod(method) {
+        const raw = String(method ?? 'avg').trim().toLowerCase();
+        if (raw === 'average') {
+            return 'avg';
+        }
+        if (raw === 'min' || raw === 'max' || raw === 'sma' || raw === 'ema' || raw === 'avg') {
+            return raw;
+        }
+        return 'avg';
+    }
+    function normalizeHistoryPath(path) {
+        const trimmed = typeof path === 'string' ? path.trim() : '';
+        if (!trimmed) {
+            return '';
+        }
+        if (trimmed.startsWith('vessels.self.')) {
+            return trimmed.slice('vessels.self.'.length);
+        }
+        if (trimmed.startsWith('self.')) {
+            return trimmed.slice('self.'.length);
+        }
+        return trimmed;
+    }
+    function buildPathsFromPathSpecs(pathSpecs) {
+        if (!Array.isArray(pathSpecs)) {
+            return '';
+        }
+        const specs = pathSpecs;
+        const encoded = specs
+            .map(spec => {
+            const path = normalizeHistoryPath(spec.path);
+            if (!path) {
+                return '';
+            }
+            const method = normalizeHistoryMethod(spec.aggregate);
+            const params = Array.isArray(spec.parameter)
+                ? spec.parameter.map(item => String(item).trim()).filter(Boolean)
+                : [];
+            return [path, method, ...params].join(':');
+        })
+            .filter(Boolean);
+        return encoded.join(',');
+    }
+    function buildHistoryQueryFromValuesRequest(query) {
+        const from = toIsoString(query?.from);
+        const to = toIsoString(query?.to);
+        const duration = toDurationString(query?.duration);
+        const paths = buildPathsFromPathSpecs(query?.pathSpecs);
+        return {
+            paths,
+            context: typeof query?.context === 'string' ? query.context : undefined,
+            from,
+            to,
+            duration,
+            resolution: typeof query?.resolution === 'number' || typeof query?.resolution === 'string' ? query.resolution : undefined
+        };
+    }
+    function buildHistoryQueryFromRangeRequest(query) {
+        return {
+            from: toIsoString(query?.from),
+            to: toIsoString(query?.to),
+            duration: toDurationString(query?.duration)
+        };
+    }
+    async function resolveHistoryPaths(query) {
+        await waitForDuckDbInitialization();
+        if (!storageService.isDuckDbParquetReady()) {
+            throw new Error(getDuckDbUnavailableMessage());
+        }
+        try {
+            await storageService.flush();
+        }
+        catch (flushError) {
+            server.error(`[SERIES STORAGE] pre-paths flush failed: ${String(flushError.message || flushError)}`);
+        }
+        return storageService.getStoredPaths({
+            ...(query ?? {})
+        });
+    }
+    async function resolveHistoryContexts(query) {
+        await waitForDuckDbInitialization();
+        if (!storageService.isDuckDbParquetReady()) {
+            throw new Error(getDuckDbUnavailableMessage());
+        }
+        try {
+            await storageService.flush();
+        }
+        catch (flushError) {
+            server.error(`[SERIES STORAGE] pre-contexts flush failed: ${String(flushError.message || flushError)}`);
+        }
+        return storageService.getStoredContexts({
+            ...(query ?? {})
+        });
+    }
+    async function resolveHistoryValues(query) {
+        await waitForDuckDbInitialization();
+        if (!storageService.isDuckDbParquetReady()) {
+            throw new Error(getDuckDbUnavailableMessage());
+        }
+        try {
+            await storageService.flush();
+        }
+        catch (flushError) {
+            server.error(`[SERIES STORAGE] pre-query flush failed: ${String(flushError.message || flushError)}`);
+        }
+        const values = await storageService.getValues({
+            ...query
+        });
+        if (!values) {
+            throw new Error('DuckDB storage did not return history values.');
+        }
+        return values;
+    }
+    function registerHistoryProvider() {
+        historyApiProviderRegistered = false;
+        if (!isHistoryApiProviderEnabled()) {
+            server.debug('[KIP][HISTORY_PROVIDER] registration skipped reason=config-disabled');
+            return;
+        }
+        const host = server;
+        const apiProvider = {
+            getValues: async (query) => {
+                const resolved = await resolveHistoryValues(buildHistoryQueryFromValuesRequest(query));
+                return {
+                    ...resolved,
+                    values: resolved.values.map(valueSpec => ({
+                        path: valueSpec.path,
+                        method: valueSpec.method === 'avg' ? 'average' : valueSpec.method
+                    }))
+                };
+            },
+            getPaths: (query) => resolveHistoryPaths(buildHistoryQueryFromRangeRequest(query)),
+            getContexts: (query) => resolveHistoryContexts(buildHistoryQueryFromRangeRequest(query))
+        };
+        const registry = host.history && typeof host.history.registerHistoryApiProvider === 'function'
+            ? host.history
+            : (typeof host.registerHistoryApiProvider === 'function'
+                ? {
+                    registerHistoryApiProvider: host.registerHistoryApiProvider.bind(host),
+                    unregisterHistoryApiProvider: typeof host.unregisterHistoryApiProvider === 'function'
+                        ? host.unregisterHistoryApiProvider.bind(host)
+                        : undefined
+                }
+                : null);
+        if (registry && typeof registry.registerHistoryApiProvider === 'function') {
+            registry.registerHistoryApiProvider(apiProvider);
+            historyApiProviderRegistered = true;
+            if (typeof registry.unregisterHistoryApiProvider === 'function') {
+                historyApiRegistry = { unregisterHistoryApiProvider: registry.unregisterHistoryApiProvider.bind(registry) };
+            }
+            server.debug('[KIP][HISTORY_PROVIDER] registration success provider=kip');
+            return;
+        }
+        server.debug('[KIP][HISTORY_PROVIDER] registration skipped reason=api-unavailable');
+    }
+    function rebuildSeriesCaptureSubscriptions() {
+        stopSeriesCapture();
+        if (!isHistorySeriesServiceEnabled()) {
+            return;
+        }
+        const streamBundle = server.streambundle;
+        if (!streamBundle || typeof streamBundle.getBus !== 'function') {
+            // server.debug('[SERIES CAPTURE] streambundle.getBus not available; capture disabled');
+            return;
+        }
+        const subscriptionCandidates = new Map();
+        const addCandidate = (path, allSelfContext) => {
+            const normalized = typeof path === 'string' ? path.trim() : '';
+            if (!normalized) {
+                return;
+            }
+            const existing = subscriptionCandidates.get(normalized);
+            if (!existing) {
+                subscriptionCandidates.set(normalized, { allSelfContext });
+                return;
+            }
+            // If any series for this path requires non-self context, force generic bus subscription.
+            existing.allSelfContext = existing.allSelfContext && allSelfContext;
+        };
+        historySeries.listSeries().filter(series => series.enabled !== false).forEach(series => {
+            const allSelfContext = (series.context ?? 'vessels.self') === 'vessels.self';
+            addCandidate(series.path, allSelfContext);
+            // Workaround: subscribe to immediate parent path so object deltas (e.g. navigation.attitude)
+            // are captured and flattened into leaf numeric paths (e.g. navigation.attitude.pitch).
+            // Remove this fallback when KIP adds first-class object-path capture support.
+            const parentIdx = series.path.lastIndexOf('.');
+            if (parentIdx > 0) {
+                addCandidate(series.path.slice(0, parentIdx), allSelfContext);
+            }
+        });
+        const candidates = Array.from(subscriptionCandidates.entries()).map(([path, meta]) => ({
+            path,
+            allSelfContext: meta.allSelfContext
+        }));
+        candidates.forEach(candidate => {
+            try {
+                const bus = candidate.allSelfContext && typeof streamBundle.getSelfBus === 'function'
+                    ? streamBundle.getSelfBus(candidate.path)
+                    : streamBundle.getBus(candidate.path);
+                if (!bus || typeof bus.onValue !== 'function') {
+                    return;
+                }
+                const unsubscribe = bus.onValue((sample) => {
+                    try {
+                        const count = historySeries.recordFromSignalKSample(sample);
+                        if (count > 0) {
+                            // server.debug(`[SERIES CAPTURE] path=${candidate.path} recorded=${count}`);
+                        }
+                    }
+                    catch (error) {
+                        server.error(`[SERIES CAPTURE] failed to record sample path=${candidate.path}: ${String(error.message || error)}`);
+                    }
+                });
+                if (typeof unsubscribe === 'function') {
+                    streamUnsubscribes.push(unsubscribe);
+                }
+            }
+            catch (error) {
+                server.error(`[SERIES CAPTURE] failed to subscribe path=${candidate.path}: ${String(error.message || error)}`);
+            }
+        });
+        // server.debug(`[SERIES CAPTURE] activePathSubscriptions=${candidates.length}`);
+    }
+    function startStorageFlushTimer(intervalMs) {
+        if (storageFlushTimer) {
+            clearInterval(storageFlushTimer);
+            storageFlushTimer = null;
+        }
+        if (!storageService.isDuckDbParquetEnabled()) {
+            return;
+        }
+        storageFlushTimer = setInterval(() => {
+            void storageService.flush()
+                .then(result => {
+                if (result.inserted > 0 || result.exported > 0) {
+                    server.debug(`[KIP][STORAGE] flush inserted=${result.inserted} exported=${result.exported}`);
+                }
+            })
+                .catch(error => {
+                server.error(`[SERIES STORAGE] flush failed: ${String(error.message || error)}`);
+            });
+        }, intervalMs);
+        storageFlushTimer.unref?.();
+    }
     const plugin = {
         id: 'kip',
         name: 'KIP',
         description: 'KIP server plugin',
         start: (settings) => {
-            server.debug(`Starting plugin with settings: ${JSON.stringify(settings)}`);
+            server.debug('[KIP][LIFECYCLE] start');
+            logRuntimeDependencyVersions();
+            const modeConfig = resolveHistoryModeConfig(settings);
+            historySeriesServiceEnabled = modeConfig.historySeriesServiceEnabled;
+            registerAsHistoryApiProvider = modeConfig.registerAsHistoryApiProvider;
+            logOperationalMode('start-configured');
+            storageService.setLogger({
+                debug: (msg) => server.debug(msg),
+                error: (msg) => server.error(msg)
+            });
+            const storageConfig = storageService.configure();
+            server.debug(`[KIP][STORAGE] config engine=${storageConfig.engine} db=${storageConfig.databaseFile} parquetDir=${storageConfig.parquetDirectory} flushMs=${storageConfig.flushIntervalMs}`);
+            historySeries.setSampleSink(sample => {
+                storageService.enqueueSample(sample);
+            });
+            duckDbInitializationPromise = storageService.initialize();
+            void duckDbInitializationPromise.then((ready) => {
+                server.debug(`[KIP][STORAGE] duckdbReady=${ready}`);
+                if (ready && storageService.isDuckDbParquetEnabled()) {
+                    if (isHistorySeriesServiceEnabled()) {
+                        void storageService.getSeriesDefinitions()
+                            .then((storedSeries) => {
+                            if (storedSeries.length > 0) {
+                                historySeries.reconcileSeries(storedSeries);
+                                rebuildSeriesCaptureSubscriptions();
+                            }
+                            startStorageFlushTimer(storageConfig.flushIntervalMs);
+                            logOperationalMode('duckdb-ready');
+                            server.setPluginStatus(`KIP plugin started with DuckDB/Parquet history storage. Loaded ${storedSeries.length} persisted series. historySeriesServiceEnabled=${isHistorySeriesServiceEnabled()} historyApiProviderEnabled=${isHistoryApiProviderEnabled()} historyApiProviderRegistered=${historyApiProviderRegistered}`);
+                        })
+                            .catch((loadError) => {
+                            server.error(`[SERIES STORAGE] failed to load persisted series: ${String(loadError.message || loadError)}`);
+                            startStorageFlushTimer(storageConfig.flushIntervalMs);
+                            logOperationalMode('duckdb-ready-series-load-failed');
+                            server.setPluginStatus(`KIP plugin started with DuckDB/Parquet history storage. historySeriesServiceEnabled=${isHistorySeriesServiceEnabled()} historyApiProviderEnabled=${isHistoryApiProviderEnabled()} historyApiProviderRegistered=${historyApiProviderRegistered}`);
+                        });
+                    }
+                    else {
+                        historySeries.reconcileSeries([]);
+                        stopSeriesCapture();
+                        startStorageFlushTimer(storageConfig.flushIntervalMs);
+                        logOperationalMode('duckdb-ready-series-disabled');
+                        server.setPluginStatus(`KIP plugin started with history-series service disabled. historyApiProviderEnabled=${isHistoryApiProviderEnabled()} historyApiProviderRegistered=${historyApiProviderRegistered}`);
+                    }
+                    return;
+                }
+                if (storageFlushTimer) {
+                    clearInterval(storageFlushTimer);
+                    storageFlushTimer = null;
+                }
+                const initError = storageService.getLastInitError();
+                if (initError) {
+                    server.setPluginError(`DuckDB unavailable. ${initError}`);
+                    logOperationalMode('duckdb-unavailable');
+                    server.setPluginStatus(`KIP plugin started with DuckDB unavailable. historySeriesServiceEnabled=${isHistorySeriesServiceEnabled()} historyApiProviderEnabled=${isHistoryApiProviderEnabled()} historyApiProviderRegistered=${historyApiProviderRegistered}`);
+                }
+            });
+            if (retentionSweepTimer) {
+                clearInterval(retentionSweepTimer);
+            }
+            retentionSweepTimer = setInterval(() => {
+                try {
+                    if (storageService.isDuckDbParquetReady()) {
+                        const lifecycleToken = storageService.getLifecycleToken();
+                        void storageService.pruneExpiredSamples(Date.now(), lifecycleToken)
+                            .then(removedPersistedRows => {
+                            if (removedPersistedRows > 0) {
+                                server.debug(`[KIP][RETENTION] pruneExpired removedRows=${removedPersistedRows}`);
+                            }
+                            return storageService.pruneOrphanedSamples(lifecycleToken)
+                                .then(removedOrphanRows => {
+                                if (removedOrphanRows > 0) {
+                                    server.debug(`[KIP][RETENTION] pruneOrphaned removedRows=${removedOrphanRows}`);
+                                }
+                            });
+                        })
+                            .catch(error => {
+                            server.error(`[SERIES RETENTION] duckdbPrune failed: ${String(error.message || error)}`);
+                        });
+                    }
+                }
+                catch (error) {
+                    server.error(`[SERIES RETENTION] sweep failed: ${String(error.message || error)}`);
+                }
+            }, 60 * 60_000);
+            retentionSweepTimer.unref?.();
+            rebuildSeriesCaptureSubscriptions();
             if (server.registerPutHandler) {
+                server.debug(`[KIP][COMMAND] registerPutHandlers context=${PUT_CONTEXT}`);
                 server.registerPutHandler(PUT_CONTEXT, COMMAND_PATHS.SET_DISPLAY, (context, path, value) => {
+                    server.debug(`[KIP][COMMAND] putHandlerHit command=${COMMAND_PATHS.SET_DISPLAY} path=${String(path)} context=${String(context)}`);
                     void context;
                     void path;
                     return handleSetDisplay(value);
                 }, plugin.id);
                 server.registerPutHandler(PUT_CONTEXT, COMMAND_PATHS.SET_SCREEN_INDEX, (context, path, value) => {
+                    server.debug(`[KIP][COMMAND] putHandlerHit command=${COMMAND_PATHS.SET_SCREEN_INDEX} path=${String(path)} context=${String(context)}`);
                     void context;
                     void path;
                     return handleScreenWrite(value, 'screenIndex');
                 }, plugin.id);
                 server.registerPutHandler(PUT_CONTEXT, COMMAND_PATHS.REQUEST_ACTIVE_SCREEN, (context, path, value) => {
+                    server.debug(`[KIP][COMMAND] putHandlerHit command=${COMMAND_PATHS.REQUEST_ACTIVE_SCREEN} path=${String(path)} context=${String(context)}`);
                     void context;
                     void path;
                     return handleScreenWrite(value, 'activeScreen');
                 }, plugin.id);
             }
+            registerHistoryProvider();
+            logOperationalMode('post-provider-registration');
             server.setPluginStatus(`Starting...`);
         },
         stop: () => {
-            server.debug(`Stopping plugin`);
+            server.debug('[KIP][LIFECYCLE] stop');
+            stopSeriesCapture();
+            if (retentionSweepTimer) {
+                clearInterval(retentionSweepTimer);
+                retentionSweepTimer = null;
+            }
+            if (storageFlushTimer) {
+                clearInterval(storageFlushTimer);
+                storageFlushTimer = null;
+            }
+            const storageLifecycleToken = storageService.getLifecycleToken();
+            void storageService.flush(storageLifecycleToken)
+                .catch(() => undefined)
+                .then(() => storageService.close(storageLifecycleToken))
+                .catch(() => undefined);
+            if (historyApiRegistry) {
+                try {
+                    historyApiRegistry.unregisterHistoryApiProvider();
+                    server.debug('[KIP][HISTORY_PROVIDER] unregister success provider=kip');
+                }
+                catch (error) {
+                    server.error(`[HISTORY PROVIDER] unregister failed: ${String(error.message || error)}`);
+                }
+                historyApiRegistry = null;
+            }
+            duckDbInitializationPromise = null;
             const msg = 'Stopped.';
             server.setPluginStatus(msg);
         },
         schema: () => CONFIG_SCHEMA,
         registerWithRouter(router) {
-            server.debug(`Registering plugin routes: ${API_PATHS.DISPLAYS}, ${API_PATHS.INSTANCE}, ${API_PATHS.SCREEN_INDEX}, ${API_PATHS.ACTIVATE_SCREEN}`);
+            server.debug(`[KIP][ROUTES] register displays=${API_PATHS.DISPLAYS} instance=${API_PATHS.INSTANCE} screenIndex=${API_PATHS.SCREEN_INDEX} activeScreen=${API_PATHS.ACTIVATE_SCREEN}`);
             // Validate/normalize :displayId where present
             router.param('displayId', (req, res, next, displayId) => {
+                logAuthTrace(req, 'router.param:displayId:entry');
                 if (displayId == null)
                     return sendFail(res, 400, 'Missing displayId parameter');
                 try {
@@ -207,14 +756,17 @@ const start = (server) => {
                         return sendFail(res, 400, 'Invalid displayId format');
                     }
                     req.displayId = id;
+                    server.debug(`[KIP][AUTH] displayIdNormalized value=${id}`);
                     next();
                 }
                 catch {
+                    server.error(`[AUTH TRACE] router.param:displayId:failed rawDisplayId=${String(displayId)}`);
                     return sendFail(res, 400, 'Missing or invalid displayId parameter');
                 }
             });
             router.put(`${API_PATHS.INSTANCE}`, async (req, res) => {
-                server.debug(`** PUT ${API_PATHS.INSTANCE}. Params: ${JSON.stringify(req.params)} Body: ${JSON.stringify(req.body)}`);
+                logAuthTrace(req, 'route:PUT:INSTANCE:entry');
+                server.debug(`[KIP][ROUTE] method=PUT path=${API_PATHS.INSTANCE} params=${JSON.stringify(req.params)} body=${JSON.stringify(req.body)}`);
                 try {
                     const displayId = req.displayId;
                     if (!displayId) {
@@ -231,7 +783,8 @@ const start = (server) => {
                 }
             });
             router.put(`${API_PATHS.SCREEN_INDEX}`, async (req, res) => {
-                server.debug(`** PUT ${API_PATHS.SCREEN_INDEX}. Params: ${JSON.stringify(req.params)} Body: ${JSON.stringify(req.body)}`);
+                logAuthTrace(req, 'route:PUT:SCREEN_INDEX:entry');
+                server.debug(`[KIP][ROUTE] method=PUT path=${API_PATHS.SCREEN_INDEX} params=${JSON.stringify(req.params)} body=${JSON.stringify(req.body)}`);
                 try {
                     const displayId = req.displayId;
                     if (!displayId) {
@@ -251,7 +804,8 @@ const start = (server) => {
                 }
             });
             router.put(`${API_PATHS.ACTIVATE_SCREEN}`, async (req, res) => {
-                server.debug(`** PUT ${API_PATHS.ACTIVATE_SCREEN}. Params: ${JSON.stringify(req.params)} Body: ${JSON.stringify(req.body)}`);
+                logAuthTrace(req, 'route:PUT:ACTIVATE_SCREEN:entry');
+                server.debug(`[KIP][ROUTE] method=PUT path=${API_PATHS.ACTIVATE_SCREEN} params=${JSON.stringify(req.params)} body=${JSON.stringify(req.body)}`);
                 try {
                     const displayId = req.displayId;
                     if (!displayId) {
@@ -271,7 +825,7 @@ const start = (server) => {
                 }
             });
             router.get(API_PATHS.DISPLAYS, (req, res) => {
-                server.debug(`*** GET DISPLAY ${API_PATHS.DISPLAYS}. Params: ${JSON.stringify(req.params)}`);
+                server.debug(`[KIP][ROUTE] method=GET path=${API_PATHS.DISPLAYS} params=${JSON.stringify(req.params)}`);
                 try {
                     const displays = getAvailableDisplays();
                     const items = displays && typeof displays === 'object'
@@ -283,8 +837,8 @@ const start = (server) => {
                             displayName: v?.value?.displayName ?? null
                         }))
                         : [];
-                    server.debug(`getAvailableDisplays returned: ${JSON.stringify(displays)}`);
-                    server.debug(`Found ${items.length} displays: ${JSON.stringify(items)}`);
+                    server.debug(`[KIP][DISPLAYS] raw=${JSON.stringify(displays)}`);
+                    server.debug(`[KIP][DISPLAYS] count=${items.length} items=${JSON.stringify(items)}`);
                     return res.status(200).json(items);
                 }
                 catch (error) {
@@ -293,18 +847,18 @@ const start = (server) => {
                 }
             });
             router.get(`${API_PATHS.INSTANCE}`, (req, res) => {
-                server.debug(`*** GET INSTANCE ${API_PATHS.INSTANCE}. Params: ${JSON.stringify(req.params)}`);
+                server.debug(`[KIP][ROUTE] method=GET path=${API_PATHS.INSTANCE} params=${JSON.stringify(req.params)}`);
                 try {
                     const displayId = req.displayId;
                     if (!displayId) {
                         return sendFail(res, 400, 'Missing displayId parameter');
                     }
                     const node = getDisplaySelfPath(displayId);
-                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                    const screens = node?.value?.screens ?? null;
-                    if (screens === undefined) {
+                    if (node === undefined) {
                         return sendFail(res, 404, `Display ${displayId} not found`);
                     }
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    const screens = node?.value?.screens ?? null;
                     return sendOk(res, screens);
                 }
                 catch (error) {
@@ -313,18 +867,18 @@ const start = (server) => {
                 }
             });
             router.get(`${API_PATHS.SCREEN_INDEX}`, (req, res) => {
-                server.debug(`*** GET SCREEN_INDEX ${API_PATHS.SCREEN_INDEX}. Params: ${JSON.stringify(req.params)}`);
+                server.debug(`[KIP][ROUTE] method=GET path=${API_PATHS.SCREEN_INDEX} params=${JSON.stringify(req.params)}`);
                 try {
                     const displayId = req.displayId;
                     if (!displayId) {
                         return sendFail(res, 400, 'Missing displayId parameter');
                     }
                     const node = getDisplaySelfPath(displayId, 'screenIndex');
-                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                    const idx = node?.value ?? null;
-                    if (idx === undefined) {
+                    if (node === undefined) {
                         return sendFail(res, 404, `Active screen for display Id ${displayId} not found in path`);
                     }
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    const idx = node?.value ?? null;
                     return sendOk(res, idx);
                 }
                 catch (error) {
@@ -333,18 +887,18 @@ const start = (server) => {
                 }
             });
             router.get(`${API_PATHS.ACTIVATE_SCREEN}`, (req, res) => {
-                server.debug(`*** GET ACTIVATE_SCREEN ${API_PATHS.ACTIVATE_SCREEN}. Params: ${JSON.stringify(req.params)}`);
+                server.debug(`[KIP][ROUTE] method=GET path=${API_PATHS.ACTIVATE_SCREEN} params=${JSON.stringify(req.params)}`);
                 try {
                     const displayId = req.displayId;
                     if (!displayId) {
                         return sendFail(res, 400, 'Missing displayId parameter');
                     }
                     const node = getDisplaySelfPath(displayId, 'activeScreen');
-                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                    const idx = node?.value ?? null;
-                    if (idx === undefined) {
+                    if (node === undefined) {
                         return sendFail(res, 404, `Change display screen Id ${displayId} not found in path`);
                     }
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    const idx = node?.value ?? null;
                     return sendOk(res, idx);
                 }
                 catch (error) {
@@ -352,15 +906,136 @@ const start = (server) => {
                     return sendFail(res, 400, error.message);
                 }
             });
+            router.get(API_PATHS.SERIES, async (req, res) => {
+                server.debug(`[KIP][ROUTE] method=GET path=${API_PATHS.SERIES} params=${JSON.stringify(req.params)}`);
+                try {
+                    if (!ensureHistorySeriesServiceEnabledForRequest(res)) {
+                        return;
+                    }
+                    if (!(await ensureDuckDbReadyForRequest(res))) {
+                        return;
+                    }
+                    return sendOk(res, historySeries.listSeries());
+                }
+                catch (error) {
+                    server.error(`Error reading series: ${String(error.message || error)}`);
+                    const mapped = getRouteError(error, 'Failed to read series');
+                    return sendFail(res, mapped.statusCode, mapped.message);
+                }
+            });
+            router.put(API_PATHS.SERIES_INSTANCE, async (req, res) => {
+                server.debug(`[KIP][ROUTE] method=PUT path=${API_PATHS.SERIES_INSTANCE} params=${JSON.stringify(req.params)} body=${JSON.stringify(req.body)}`);
+                try {
+                    if (!ensureHistorySeriesServiceEnabledForRequest(res)) {
+                        return;
+                    }
+                    if (!(await ensureDuckDbReadyForRequest(res))) {
+                        return;
+                    }
+                    const seriesId = String(req.params.seriesId ?? '').trim();
+                    if (!seriesId) {
+                        return sendFail(res, 400, 'Missing seriesId parameter');
+                    }
+                    const payload = (req.body ?? {});
+                    const previous = findSeriesById(seriesId);
+                    const next = historySeries.upsertSeries({
+                        ...payload,
+                        seriesId,
+                        datasetUuid: String(payload.datasetUuid ?? seriesId)
+                    });
+                    try {
+                        await storageService.upsertSeriesDefinition(next);
+                    }
+                    catch (storageError) {
+                        if (previous) {
+                            historySeries.upsertSeries(previous);
+                        }
+                        else {
+                            historySeries.deleteSeries(seriesId);
+                        }
+                        throw storageError;
+                    }
+                    rebuildSeriesCaptureSubscriptions();
+                    return sendOk(res, next);
+                }
+                catch (error) {
+                    server.error(`Error writing series: ${String(error.message || error)}`);
+                    const mapped = getRouteError(error, 'Failed to write series');
+                    return sendFail(res, mapped.statusCode, mapped.message);
+                }
+            });
+            router.delete(API_PATHS.SERIES_INSTANCE, async (req, res) => {
+                server.debug(`[KIP][ROUTE] method=DELETE path=${API_PATHS.SERIES_INSTANCE} params=${JSON.stringify(req.params)}`);
+                try {
+                    if (!ensureHistorySeriesServiceEnabledForRequest(res)) {
+                        return;
+                    }
+                    if (!(await ensureDuckDbReadyForRequest(res))) {
+                        return;
+                    }
+                    const seriesId = String(req.params.seriesId ?? '').trim();
+                    if (!seriesId) {
+                        return sendFail(res, 400, 'Missing seriesId parameter');
+                    }
+                    const previous = findSeriesById(seriesId);
+                    if (!previous) {
+                        return sendFail(res, 404, `Series ${seriesId} not found`);
+                    }
+                    await storageService.deleteSeriesDefinition(seriesId);
+                    historySeries.deleteSeries(seriesId);
+                    rebuildSeriesCaptureSubscriptions();
+                    return sendOk(res, { state: 'SUCCESS', statusCode: 200 });
+                }
+                catch (error) {
+                    server.error(`Error deleting series: ${String(error.message || error)}`);
+                    const mapped = getRouteError(error, 'Failed to delete series');
+                    return sendFail(res, mapped.statusCode, mapped.message);
+                }
+            });
+            router.post(API_PATHS.SERIES_RECONCILE, async (req, res) => {
+                server.debug(`[KIP][ROUTE] method=POST path=${API_PATHS.SERIES_RECONCILE} body=${JSON.stringify(req.body)}`);
+                try {
+                    if (!ensureHistorySeriesServiceEnabledForRequest(res)) {
+                        return;
+                    }
+                    if (!(await ensureDuckDbReadyForRequest(res))) {
+                        return;
+                    }
+                    const payload = req.body;
+                    if (!Array.isArray(payload)) {
+                        return sendFail(res, 400, 'Body must be an array of series definitions');
+                    }
+                    const simulated = new history_series_service_1.HistorySeriesService(() => Date.now(), false);
+                    historySeries.listSeries().forEach(series => {
+                        simulated.upsertSeries(series);
+                    });
+                    const scopedPayload = payload.map(series => ({
+                        ...series
+                    }));
+                    const result = simulated.reconcileSeries(scopedPayload);
+                    const nextSeries = simulated.listSeries();
+                    await storageService.replaceSeriesDefinitions(nextSeries);
+                    const seriesOutsideScope = historySeries.listSeries();
+                    historySeries.reconcileSeries([...seriesOutsideScope, ...nextSeries]);
+                    server.debug(`[KIP][SERIES_RECONCILE] created=${result.created} updated=${result.updated} deleted=${result.deleted} total=${result.total}`);
+                    rebuildSeriesCaptureSubscriptions();
+                    return sendOk(res, result);
+                }
+                catch (error) {
+                    server.error(`Error reconciling series: ${String(error.message || error)}`);
+                    const mapped = getRouteError(error, 'Failed to reconcile series');
+                    return sendFail(res, mapped.statusCode, mapped.message);
+                }
+            });
             // List all registered routes for debugging
             if (router.stack) {
                 router.stack.forEach((layer) => {
                     if (layer.route && layer.route.path) {
-                        server.debug(`Registered route: ${layer.route.stack[0].method.toUpperCase()} ${layer.route.path}`);
+                        server.debug(`[KIP][ROUTES] registered method=${layer.route.stack[0].method.toUpperCase()} path=${layer.route.path}`);
                     }
                 });
             }
-            server.setPluginStatus(`Providing remote display screen control`);
+            server.setPluginStatus(`Providing remote display screen control and history series API`);
         },
         getOpenApi: () => mutableOpenApi
     };