@signalk/tracks-plugin 3.0.3 → 3.1.1

package/dist/harness.test-utils.d.ts

@@ -112,6 +112,13 @@ export interface HarnessOptions {
         };
         /** Fail the windowed read, leaving the provider unable to answer at all. */
         getValuesRejects?: boolean;
+        /**
+         * Answer `getValues` as a real provider does with `aggregate: 'first'`:
+         * only the earliest of `rows` in each bucket of the asked resolution.
+         */
+        aggregatesFirst?: boolean;
+        /** Called with every `getValues` query, so a test can see what was asked. */
+        onValues?: (query: HistoryValuesQuery) => void;
         /**
          * Reject `getHistoryApi` itself, as the server does when no provider is
          * registered — the default install, and not an outage.
@@ -131,4 +138,13 @@ export declare function deferred(): {
     release: () => void;
     wait: Promise<void>;
 };
+export interface HistoryValuesQuery {
+    from: {
+        toString: () => string;
+    };
+    to: {
+        toString: () => string;
+    };
+    resolution?: number;
+}
 export declare function createHarness(options?: HarnessOptions): TestHarness;

package/dist/index.js

@@ -1,4 +1,4 @@
-import { S as uncoveredBetween, _ as TimeWindowError, a as createDistanceTo, b as thin, c as gpxFilename, d as rfc8187, g as validateParameters, h as unwrapString, i as contextName, l as historyRowPosition, m as trackLabel, n as SelfPositionUnavailableError, p as toIsoTimes, r as asciiFilename, s as createMatcher, t as QueryParameterError, u as resolveContext, v as parseTrackQuery, x as thinToBudget, y as segment } from "./utils-DOEyuPRy.js";
+import { C as uncoveredBetween, S as thinToBudget, _ as validateParameters, a as contextName, b as segment, c as createMatcher, d as resolveContext, f as rfc8187, g as unwrapString, h as trackLabel, i as clipToBounds, l as gpxFilename, m as toIsoTimes, n as SelfPositionUnavailableError, o as createDistanceTo, r as asciiFilename, s as createInBounds, t as QueryParameterError, u as historyRowPosition, v as TimeWindowError, x as thin, y as parseTrackQuery } from "./utils-CGIGp4Xf.js";
 import { Temporal } from "@js-temporal/polyfill";
 import { join } from "node:path";
 import { Worker } from "node:worker_threads";
@@ -241,19 +241,31 @@ function createTrackProvider(deps) {
 		};
 		const window = toTimeWindow(query);
 		const resolution = query.resolution ? totalMilliseconds(query.resolution) : void 0;
-		const wanted = query.contexts?.length ? new Set(query.contexts.map(resolveSelf(deps.selfContext()))) : void 0;
+		const wanted = query.contexts?.length ? [...new Set(query.contexts.map(resolveSelf(deps.selfContext())))] : void 0;
+		const bounds = toGeoBounds(query.bbox);
+		const clipTo = query.clip === true ? bounds : null;
 		const collection = await store.getFilteredTimedTracks({
-			bbox: toGeoBounds(query.bbox),
+			bbox: bounds,
 			radius: null,
-			intersects: true
+			intersects: true,
+			...wanted ? { contexts: wanted } : {},
+			...clipTo ? { clip: true } : {}
 		}, void 0, void 0, {
 			...window ? { window } : {},
-			...resolution === void 0 ? {} : { resolution }
+			...resolution === void 0 || bounds ? {} : { resolution }
 		});
+		const fromHistory = [...await deps.historyContexts(window)].filter((context) => !(context in collection) && (!wanted || wanted.includes(context)));
+		const inBounds = bounds ? createInBounds(bounds) : void 0;
+		const storedFor = async (context) => {
+			const points = await store.getTimed(context, window).catch(() => []);
+			return bounds ? points : thin(points, resolution);
+		};
+		const historyResolution = bounds ? void 0 : resolution;
 		const result = /* @__PURE__ */ new Map();
-		const contexts = Object.entries(collection).filter(([context]) => !wanted || wanted.has(context));
-		const reconciled = await Promise.all(contexts.map(async ([context, stored]) => [context, await deps.reconcileWithHistory(context, stored, window, resolution)]));
-		for (const [context, points] of reconciled) {
+		const reconciled = await Promise.all([...Object.entries(collection).map(async ([context, stored]) => [context, await deps.reconcileWithHistory(context, stored, window, historyResolution)]), ...fromHistory.map(async (context) => [context, await deps.reconcileWithHistory(context, await storedFor(context), window, historyResolution)])]);
+		for (const [context, reconciledPoints] of reconciled) {
+			if (inBounds && !reconciledPoints.some(({ position }) => inBounds(position))) continue;
+			const points = clipTo ? thin(clipToBounds(reconciledPoints, clipTo), resolution) : bounds ? thin(reconciledPoints, resolution) : reconciledPoints;
 			if (points.length === 0) continue;
 			result.set(context, points);
 		}
@@ -798,6 +810,25 @@ function reconcile(history, stored, resolution) {
 	};
 }
 /**
+* Merge history that is coarser than the store into it, the store first.
+*
+* The rule above holds while history is the finer source. A history read
+* widened to stay within its bucket budget is not: one of its buckets spans
+* several stored fixes, and letting it win would replace them all with one
+* point. So here the store keeps every bucket it has a point in, and history
+* fills only the buckets the store has nothing for — where the provider
+* recorded before this plugin was installed, or while it was off.
+*/
+function fillFromHistory(history, stored, resolution) {
+	const covered = new Set(stored.map(({ timestamp }) => bucketOf(timestamp, resolution)));
+	const filling = history.filter(({ timestamp }) => !covered.has(bucketOf(timestamp, resolution)));
+	return {
+		positions: measuredFromHistory([...stored, ...filling].sort((a, b) => a.timestamp - b.timestamp), filling),
+		fromHistory: filling.length,
+		fromStore: stored.length
+	};
+}
+/**
 * A stored point's `pauseBefore` was measured over the stored points thinning
 * dropped before it. Where a history point now comes right before it, history
 * covered part of that time, so the pause is at most the time since that
@@ -1415,7 +1446,8 @@ var Tracks = class {
 		this.debug(params);
 		this.debug("Self position", selfPosition);
 		const matcher = createMatcher(params, selfPosition, debug);
-		const contexts = Object.keys(this.tracks);
+		const requested = params.contexts ? new Set(params.contexts) : void 0;
+		const contexts = Object.keys(this.tracks).filter((context) => !requested || requested.has(context));
 		return (await Promise.all(contexts.map((context) => this.getTimed(context, query?.window).then((points) => ({
 			context,
 			points: thin(points, query?.resolution)
@@ -1548,6 +1580,30 @@ var WINDOWLESS_HISTORY_SPAN_MS = 864e5;
 */
 var EXISTENCE_PROBE_FROM_MS = 0;
 /**
+* The most buckets one history read may ask a provider for.
+*
+* A provider has no spatial filter, so a box query reads every position in
+* its window, and the cost of that read grows with the window's span over the
+* resolution, not with what is drawn. Providers fabricate a row for every
+* bucket in the range — questdb fills empty ones with nulls — and refuse a
+* range past their own cap, which one-minute buckets over two years already
+* exceed. A month of one-minute buckets keeps a Pi-class server responsive;
+* a longer window is read at a wider resolution instead.
+*/
+var HISTORY_BUCKET_BUDGET = 43200;
+/**
+* How many buckets the read that finds where a provider's data begins asks
+* for.
+*
+* A window with no start would otherwise reach back to the Unix epoch, and
+* spread over fifty-odd years the bucket budget leaves buckets weeks wide.
+* One coarse read first finds the earliest bucket holding a position, so the
+* real read starts there. A thousand buckets over the whole epoch are about
+* three weeks each, which only sets how much empty time the real read starts
+* with.
+*/
+var START_PROBE_BUCKETS = 1e3;
+/**
 * How long a provider's context list is reused before asking again.
 *
 * The probe runs only for a vessel about to 404, which is precisely the
@@ -1644,6 +1700,8 @@ function withTimeout(promise, ms) {
 		});
 	});
 }
+/** A provider's context list as a set, with the `vessels.self` spelling resolved. */
+var toContextSet = (listed, selfContext) => new Set((listed ?? []).filter((c) => typeof c === "string").map((c) => c === "vessels.self" ? selfContext : c));
 /**
 * Whether a history provider holds anything at all for a context.
 *
@@ -1667,7 +1725,7 @@ async function historyKnowsContext(app, context, debug, cache) {
 			const contexts = withTimeout(getHistoryApi(app.config?.settings?.historyApi?.defaultProvider).then((historyApi) => historyApi.getContexts ? historyApi.getContexts({
 				from: Temporal.Instant.from((/* @__PURE__ */ new Date(EXISTENCE_PROBE_FROM_MS)).toISOString()),
 				to: Temporal.Instant.from(new Date(now).toISOString())
-			}) : []).then((listed) => new Set((listed ?? []).filter((c) => typeof c === "string").map((c) => c === "vessels.self" ? app.selfContext : c))), HISTORY_QUERY_TIMEOUT_MS);
+			}) : []).then((listed) => toContextSet(listed, app.selfContext)), HISTORY_QUERY_TIMEOUT_MS);
 			const entry = {
 				at: now,
 				contexts
@@ -1687,6 +1745,38 @@ async function historyKnowsContext(app, context, debug, cache) {
 	}
 }
 /**
+* Contexts a history provider holds data for within a window, or none.
+*
+* What lets the v2 provider return a vessel the store has never seen, or match
+* a box on positions only history recorded: the store is often far younger
+* than the provider. Best-effort like every other provider call: no provider,
+* a provider without `getContexts`, an error or a hang all answer with no
+* contexts, and the store answers alone.
+*
+* A window with no start is asked from the epoch, for the same reason the
+* existence probe is; without any window the probe's cached all-time list is
+* reused rather than asking again per query.
+*/
+async function historyContextsIn(app, window, debug, cache) {
+	const getHistoryApi = app.getHistoryApi;
+	if (!getHistoryApi) return /* @__PURE__ */ new Set();
+	try {
+		if (!window) {
+			await historyKnowsContext(app, app.selfContext, debug, cache);
+			return await cache.current?.contexts ?? /* @__PURE__ */ new Set();
+		}
+		const historyApi = await withTimeout(getHistoryApi(app.config?.settings?.historyApi?.defaultProvider), HISTORY_QUERY_TIMEOUT_MS);
+		if (!historyApi.getContexts) return /* @__PURE__ */ new Set();
+		return toContextSet(await withTimeout(historyApi.getContexts({
+			from: Temporal.Instant.from(new Date(Math.max(window.from, EXISTENCE_PROBE_FROM_MS)).toISOString()),
+			to: Temporal.Instant.from(new Date(window.to).toISOString())
+		}), HISTORY_QUERY_TIMEOUT_MS), app.selfContext);
+	} catch (err) {
+		if (debug.enabled) debug(`History contexts unavailable: ${errorDetail(err)}`);
+		return /* @__PURE__ */ new Set();
+	}
+}
+/**
 * Positions a history provider holds for a window, or none.
 *
 * Best-effort by design: a provider that is absent, slow, or failing must not
@@ -1694,65 +1784,80 @@ async function historyKnowsContext(app, context, debug, cache) {
 * provider is the finer source where it reaches, not a required one.
 */
 async function historyPositions(app, context, window, resolutionMs, debug) {
-	const providerSeconds = Math.max(1, Math.round(resolutionMs / 1e3));
-	const applied = providerSeconds * 1e3;
-	const getHistoryApi = app.getHistoryApi;
-	if (!getHistoryApi) return {
+	const askedSeconds = Math.max(1, Math.round(resolutionMs / 1e3));
+	const none = (failed) => ({
 		points: [],
-		resolutionMs: applied,
-		failed: false
-	};
+		resolutionMs: askedSeconds * 1e3,
+		coarsened: false,
+		failed
+	});
+	const getHistoryApi = app.getHistoryApi;
+	if (!getHistoryApi) return none(false);
 	let historyApi;
 	try {
 		historyApi = await withTimeout(getHistoryApi(app.config?.settings?.historyApi?.defaultProvider), HISTORY_QUERY_TIMEOUT_MS);
 	} catch (err) {
 		const timedOut = err instanceof HistoryTimeoutError;
 		if (debug.enabled) debug(`${timedOut ? "History provider timed out" : "No history provider"} for ${context}: ${errorDetail(err)}`);
-		return {
-			points: [],
-			resolutionMs: applied,
-			failed: timedOut
-		};
+		return none(timedOut);
 	}
-	try {
+	const read = async (from, seconds, throughEnd = false) => {
 		const response = await withTimeout(historyApi.getValues({
 			context,
-			from: Temporal.Instant.from(new Date(window.from).toISOString()),
+			from: Temporal.Instant.from(new Date(from).toISOString()),
 			to: Temporal.Instant.from(new Date(window.to).toISOString()),
 			pathSpecs: [{
 				path: "navigation.position",
 				aggregate: "first"
 			}],
-			resolution: providerSeconds
+			resolution: seconds
 		}), HISTORY_QUERY_TIMEOUT_MS);
 		const points = [];
 		for (const row of response?.data ?? []) {
 			const position = historyRowPosition(row);
 			if (position) {
 				const timestamp = historyRowTimestamp(row);
-				const withinEnd = window.inclusiveEnd ? timestamp <= window.to : timestamp < window.to;
+				const withinEnd = window.inclusiveEnd || throughEnd ? timestamp <= window.to : timestamp < window.to;
 				if (timestamp >= window.from && withinEnd) points.push({
 					position,
 					timestamp,
-					span: applied
+					span: seconds * 1e3
 				});
 			}
 		}
-		if (debug.enabled) debug(`History supplied ${points.length} position(s) for ${context}`);
+		return points;
+	};
+	try {
+		let from = Math.max(window.from, EXISTENCE_PROBE_FROM_MS);
+		if (!Number.isFinite(window.from)) {
+			const probeSeconds = Math.max(askedSeconds, Math.ceil((window.to - from) / 1e3 / START_PROBE_BUCKETS));
+			const probed = await read(from, probeSeconds, true);
+			if (probed.length === 0) return none(false);
+			const earliest = Math.min(...probed.map(({ timestamp }) => timestamp));
+			from = Math.max(from, earliest - probeSeconds * 1e3);
+		}
+		const providerSeconds = Math.max(askedSeconds, Math.ceil((window.to - from) / 1e3 / HISTORY_BUCKET_BUDGET));
+		const points = await read(from, providerSeconds);
+		if (debug.enabled) debug(`History supplied ${points.length} position(s) for ${context} at ${providerSeconds}s`);
 		return {
 			points,
-			resolutionMs: applied,
+			resolutionMs: providerSeconds * 1e3,
+			coarsened: providerSeconds > askedSeconds,
 			failed: false
 		};
 	} catch (err) {
 		if (debug.enabled) debug(`History unavailable for ${context}: ${errorDetail(err)}`);
-		return {
-			points: [],
-			resolutionMs: applied,
-			failed: true
-		};
+		return none(true);
 	}
 }
+/**
+* A track from the store and what history holds for it.
+*
+* History is the finer source where it reaches, so it wins its buckets —
+* unless the read was widened to stay within its budget, when it is the
+* coarser one and only fills what the store lacks.
+*/
+var withHistory = (history, stored) => history.points.length === 0 ? stored : (history.coarsened ? fillFromHistory : reconcile)(history.points, stored, history.resolutionMs).positions;
 /** Epoch milliseconds for a history row, or 0 when the timestamp is unusable. */
 var historyRowTimestamp = (row) => {
 	const raw = Array.isArray(row) ? row[0] : void 0;
@@ -1885,10 +1990,10 @@ function ThePlugin(app) {
 				selfContext: () => app.selfContext,
 				segmentGap: () => segmentGap,
 				contextName: (context) => contextName(context, remembering(() => tracks)),
+				historyContexts: (window) => historyContextsIn(app, window, app.debug, knownContexts),
 				reconcileWithHistory: async (context, stored, window, resolutionMs) => {
 					const effective = resolutionMs ?? storeResolution;
-					const history = await historyPositions(app, context, window ?? windowSpanning(stored, WINDOWLESS_HISTORY_SPAN_MS), effective, app.debug);
-					return history.points.length ? reconcile(history.points, stored, history.resolutionMs).positions : stored;
+					return withHistory(await historyPositions(app, context, window ?? windowSpanning(stored, WINDOWLESS_HISTORY_SPAN_MS), effective, app.debug), stored);
 				}
 			}));
 			return starting.ready.catch(async () => {
@@ -1939,7 +2044,7 @@ function ThePlugin(app) {
 					known: false
 				}));
 				const history = await historyPositions(app, context, query.window ?? windowSpanning(stored, WINDOWLESS_HISTORY_SPAN_MS), effectiveResolution, app.debug);
-				const points = history.points.length ? reconcile(history.points, stored, history.resolutionMs).positions : stored;
+				const points = withHistory(history, stored);
 				if (points.length === 0) {
 					if (!(known || await historyKnowsContext(app, context, app.debug, knownContexts))) return;
 					if (history.failed) throw new HistoryUnavailableError(`History provider could not be read for ${context}`);