@signalk/tracks-plugin 3.2.0 → 3.4.0

package/dist/index.js

@@ -117,22 +117,6 @@ function nearestLongitude(lng, origin) {
 	return lng - 360 * Math.round((lng - origin) / 360);
 }
 //#endregion
-//#region src/trackApi.ts
-/**
-* Thrown by `storeTrack` when this provider will not keep the track, which
-* the server answers with a 400 rather than a 500.
-*
-* The server recognises it by `isTrackRejected` rather than by class, since
-* this is a copy of its own.
-*/
-var TrackRejectedError = class extends Error {
-	isTrackRejected = true;
-	constructor(message) {
-		super(message);
-		this.name = "TrackRejectedError";
-	}
-};
-//#endregion
 //#region src/importedTracks.ts
 /**
 * Prefix of every imported track's id. It keeps imported and recorded tracks
@@ -147,13 +131,11 @@ var isImportedId = (id) => id.startsWith(IMPORTED_PREFIX);
 * thinning and window code assume time order, and a segment boundary is a gap
 * in recording, so segments are never merged or reordered.
 *
-* Refuses a track with neither times nor a vessel. Such a track answers no
-* time window, and the server refuses a query across all vessels without one,
-* so it could only ever be found again by an id the client would have to keep.
+* A track with neither times nor a vessel answers no time window and no
+* vessel query; the plugin's import list and its id are what find it again.
 */
 function toImportedTrack(track, id = `${IMPORTED_PREFIX}${randomUUID()}`) {
 	const { coordTimes, name, context, ...metadata } = track.properties;
-	if (coordTimes === void 0 && context === void 0) throw new TrackRejectedError("A track with neither coordTimes nor a context could not be listed again; send one of them");
 	const segments = track.geometry.coordinates.map((segment, i) => {
 		const points = segment.map(([lon, lat], j) => {
 			const time = coordTimes?.[i]?.[j];
@@ -180,12 +162,10 @@ function toImportedTrack(track, id = `${IMPORTED_PREFIX}${randomUUID()}`) {
 * becomes an untimed import; a file with some keeps only its timed points,
 * because an import is timed throughout or not at all, and its times are what
 * let a query find it. `context` names the vessel in place of the file's own.
-* Refused, like any import, when it ends up with neither times nor a vessel.
 */
 function fromGpxTrack(gpx, context = gpx.context, id = `${IMPORTED_PREFIX}${randomUUID()}`) {
 	const all = gpx.segments.flat();
 	const timed = all.some(({ timestamp }) => timestamp !== 0);
-	if (!timed && context === void 0) throw new TrackRejectedError(`Track '${gpx.name}' has neither times nor a vessel, so it could not be listed again; choose a vessel for it`);
 	const segments = gpx.segments.map((points) => points.filter(({ timestamp }) => !timed || timestamp !== 0).map(({ position, timestamp }) => timed ? {
 		position,
 		timestamp
@@ -284,6 +264,22 @@ function overlaps(start, width, { from, to }) {
 	return width > 0 ? start + width > from : start >= from;
 }
 //#endregion
+//#region src/trackApi.ts
+/**
+* Thrown by `storeTrack` when this provider will not keep the track, which
+* the server answers with a 400 rather than a 500.
+*
+* The server recognises it by `isTrackRejected` rather than by class, since
+* this is a copy of its own.
+*/
+var TrackRejectedError = class extends Error {
+	isTrackRejected = true;
+	constructor(message) {
+		super(message);
+		this.name = "TrackRejectedError";
+	}
+};
+//#endregion
 //#region src/trackProvider.ts
 /** v2 sends `[west, south, east, north]`; a bounds here is `[lat, lng]` corners. */
 var toGeoBounds = (bbox) => {
@@ -447,12 +443,16 @@ function createTrackProvider(deps) {
 		}
 		return result;
 	};
-	/** What names an imported track: its id, name and vessel, over what it was posted with. */
+	/**
+	* What names an imported track: its id, name and vessel, over what it was posted with.
+	* `imported`, when it was kept, tells it apart from the recording of the same vessel, which it is listed beside.
+	*/
 	const importIdentity = (track) => {
 		const contextName = track.context === void 0 ? void 0 : deps.contextName(track.context);
 		return {
 			...track.metadata,
 			id: track.id,
+			imported: new Date(track.importedAt).toISOString(),
 			...track.name === void 0 ? {} : { name: track.name },
 			...track.context === void 0 ? {} : {
 				context: track.context,
@@ -464,9 +464,19 @@ function createTrackProvider(deps) {
 	/** The store, when it can keep and delete tracks; an ordinary failure otherwise. */
 	const writableStore = () => {
 		const store = deps.store();
-		if (!store?.storeImport || !store.getImport || !store.binImport || !store.binRecorded) throw new Error("Track storage is not available");
+		if (!store?.storeImport || !store.recordTrack || !store.getImport || !store.binImport || !store.binRecorded) throw new Error("Track storage is not available");
 		return store;
 	};
+	const importTrack = (track, onOverlap) => keepTrack(track, {
+		store: writableStore(),
+		selfContext: deps.selfContext(),
+		historyKnows: deps.historyKnows,
+		recorded: (context, from, to) => deps.recordedOver(context, {
+			from,
+			to
+		}),
+		deleted: deps.deleted
+	}, onOverlap);
 	/** The recorded feature's identity, which `getTracks` and `getTrack` share. */
 	const recordedIdentity = (context) => {
 		const name = deps.contextName(context);
@@ -537,10 +547,9 @@ function createTrackProvider(deps) {
 			return shapeFeature(positionsOf(track, null), 0, { times: timed }, void 0, importIdentity(track), timed);
 		},
 		async storeTrack(track) {
-			const imported = toImportedTrack(track);
-			await writableStore().storeImport(imported);
-			return imported.id;
+			return (await importTrack(toImportedTrack(track), "keep")).id;
 		},
+		importTrack,
 		async deleteTrack(id) {
 			const recorded = recordedContext(id);
 			if (recorded !== void 0) return deleteRecorded(resolveSelf(deps.selfContext())(recorded), void 0, void 0);
@@ -669,6 +678,129 @@ function extentEpsilon([west, south, east, north]) {
 	const width = (east >= west ? east - west : east + 360 - west) * M_PER_DEG * Math.cos(midLat * Math.PI / 180);
 	return Math.max(Math.hypot(width, height) / AUTO_EPSILON_DIVISOR, MIN_AUTO_EPSILON);
 }
+/**
+* The shortest stretch without positions that an import kept beside a
+* recording may fill. Filling every pause between recorded points would
+* interleave two sources minute by minute, and any offset between them would
+* draw as a sawtooth; whole stretches keep each part of the line from one.
+*/
+var MIN_FILLED_GAP_MS = 3e5;
+/**
+* Keep a client's track, resolving to the id that addresses it.
+*
+* A timed track of the own vessel is the vessel's own passage, from before
+* the plugin recorded it or from another plotter, so it is written into the
+* recording and answers as `recorded:<context>`: with `keep`, only where the
+* recording has nothing (see {@link fillGaps}); with `replace`, in place of
+* what was recorded over the same time. Everything else is kept as an import:
+* another vessel's track is not this boat's recording, and an untimed one
+* cannot be placed in it.
+*/
+async function keepTrack(track, deps, onOverlap) {
+	const context = track.context === void 0 ? void 0 : resolveSelf(deps.selfContext)(track.context);
+	if (context === deps.selfContext && isTimed(track)) {
+		const segments = track.segments.map((points) => points.map(({ position, timestamp }) => ({
+			position,
+			timestamp
+		})));
+		if (onOverlap === "keep") {
+			const runs = [];
+			const filled = [];
+			let overlapping = 0;
+			for (const points of segments) {
+				if (points.length === 0) continue;
+				let from = Infinity;
+				let to = -Infinity;
+				for (const { timestamp } of points) {
+					from = Math.min(from, timestamp);
+					to = Math.max(to, timestamp);
+				}
+				from -= MIN_FILLED_GAP_MS;
+				to += MIN_FILLED_GAP_MS;
+				const recorded = await deps.recorded(context, from, to);
+				for (const point of filled) if (point.timestamp >= from && point.timestamp <= to) recorded.push(point);
+				const kept = fillGaps([points], recorded, MIN_FILLED_GAP_MS);
+				runs.push(...kept.runs);
+				filled.push(...kept.runs.flat());
+				overlapping += kept.overlapping;
+			}
+			await deps.store.recordTrack(context, runs, Date.now(), false);
+			return {
+				id: recordedId(context),
+				overlapping
+			};
+		}
+		await deps.store.recordTrack(context, segments, Date.now(), await deps.historyKnows(context));
+		deps.deleted();
+		return {
+			id: recordedId(context),
+			overlapping: 0
+		};
+	}
+	await deps.store.storeImport(track);
+	return {
+		id: track.id,
+		overlapping: 0
+	};
+}
+/**
+* The parts of an import that fall where a recording has nothing for at least
+* `minGap`, as runs to record, and how many points were left out.
+*
+* Each recorded point covers its own instant, or a history bucket's whole
+* span, so a coarse provider read still counts as covering the time between
+* its buckets. A run never crosses a recorded stretch or a segment boundary.
+*/
+function fillGaps(segments, recorded, minGap) {
+	const covered = [];
+	for (const { timestamp, span } of [...recorded].sort((a, b) => a.timestamp - b.timestamp)) {
+		const end = timestamp + (span ?? 0);
+		const last = covered.at(-1);
+		if (last && timestamp <= last[1]) last[1] = Math.max(last[1], end);
+		else covered.push([timestamp, end]);
+	}
+	/** Index of the first covered stretch ending at or after `time`. */
+	const nextStretch = (time) => {
+		let low = 0;
+		let high = covered.length;
+		while (low < high) {
+			const mid = low + high >> 1;
+			if (covered[mid][1] < time) low = mid + 1;
+			else high = mid;
+		}
+		return low;
+	};
+	const runs = [];
+	let overlapping = 0;
+	for (const points of segments) {
+		let run = [];
+		let runGap = -1;
+		for (const point of points) {
+			const next = nextStretch(point.timestamp);
+			const after = covered[next];
+			const before = covered[next - 1];
+			const inside = after !== void 0 && after[0] <= point.timestamp;
+			const free = (after?.[0] ?? Infinity) - (before?.[1] ?? -Infinity);
+			if (inside || free < minGap) {
+				overlapping += 1;
+				if (run.length > 0) runs.push(run);
+				run = [];
+				continue;
+			}
+			if (run.length > 0 && next !== runGap) {
+				runs.push(run);
+				run = [];
+			}
+			run.push(point);
+			runGap = next;
+		}
+		if (run.length > 0) runs.push(run);
+	}
+	return {
+		runs,
+		overlapping
+	};
+}
 /** v2 accepts the `self` alias; the store keys on the qualified context. */
 var resolveSelf = (selfContext) => (context) => context === "self" || context === "vessels.self" ? selfContext : context;
 //#endregion
@@ -1002,6 +1134,14 @@ var AsyncTrackStore = class {
 			keepEmpty
 		], "acknowledged");
 	}
+	recordTrack(context, segments, deletedAt, keepEmpty) {
+		return this.enqueue("recordTrack", [
+			context,
+			segments,
+			deletedAt,
+			keepEmpty
+		], "acknowledged");
+	}
 	binImport(importId, span, deletedAt) {
 		return this.enqueue("binImport", [
 			importId,
@@ -2038,6 +2178,14 @@ var KNOWN_CONTEXTS_TTL_MS = 3e4;
 */
 var KNOWN_CONTEXTS_FAILURE_TTL_MS = 1e3;
 /**
+* How long a context query may stay unanswered before another is started
+* beside it. Waiting for the provider to settle is what keeps a slow one from
+* running a new epoch-wide scan every few seconds; the bound only covers a
+* provider whose promise never settles at all, which would otherwise leave
+* the list unknown until a restart.
+*/
+var KNOWN_CONTEXTS_PENDING_MAX_MS = 3e5;
+/**
 * Config values arrive from the plugin UI as numbers, but a hand-edited
 * settings file can supply strings. Accept both, reject anything non-finite so
 * a bad value falls back to the default instead of poisoning arithmetic with NaN.
@@ -2126,35 +2274,50 @@ var toContextSet = (listed, selfContext) => new Set((listed ?? []).filter((c) =>
 * request open.
 */
 async function historyKnowsContext(app, context, debug, cache) {
-	const getHistoryApi = app.getHistoryApi;
-	if (!getHistoryApi) return false;
 	try {
-		const now = Date.now();
-		const cached = cache.current;
-		if (cached === void 0 || now - cached.at >= KNOWN_CONTEXTS_TTL_MS) {
-			const contexts = withTimeout(getHistoryApi(app.config?.settings?.historyApi?.defaultProvider).then((historyApi) => historyApi.getContexts ? historyApi.getContexts({
-				from: Temporal.Instant.from((/* @__PURE__ */ new Date(EXISTENCE_PROBE_FROM_MS)).toISOString()),
-				to: Temporal.Instant.from(new Date(now).toISOString())
-			}) : []).then((listed) => toContextSet(listed, app.selfContext)), HISTORY_QUERY_TIMEOUT_MS);
-			const entry = {
-				at: now,
-				contexts
-			};
-			contexts.catch(() => {
-				if (cache.current === entry) cache.current = {
-					at: now - KNOWN_CONTEXTS_TTL_MS + KNOWN_CONTEXTS_FAILURE_TTL_MS,
-					contexts
-				};
-			});
-			cache.current = entry;
-		}
-		return (await cache.current.contexts).has(context);
+		return (await knownHistoryContexts(app, cache)).has(context);
 	} catch (err) {
 		debug(`History contexts unavailable for ${context}: ${errorDetail(err)}`);
 		return false;
 	}
 }
 /**
+* Every context a history provider holds anything for, from the cache.
+*
+* Reused while fresh, and shared while in flight: a burst of misses is one
+* question to the provider, not one each. The promise is cached rather than
+* its result so concurrent callers join the same query instead of starting
+* their own, and a query is shared until the provider answers it, not only
+* until the caller's timeout: the provider keeps running a query nobody waits
+* for, so asking again on every timeout stacks scans of the whole database.
+*/
+function knownHistoryContexts(app, cache) {
+	const getHistoryApi = app.getHistoryApi;
+	if (!getHistoryApi) return Promise.resolve(/* @__PURE__ */ new Set());
+	const now = Date.now();
+	const cached = cache.current;
+	if (cached === void 0 || (cached.pending ? now - cached.at >= KNOWN_CONTEXTS_PENDING_MAX_MS : now - cached.at >= KNOWN_CONTEXTS_TTL_MS)) {
+		const contexts = getHistoryApi(app.config?.settings?.historyApi?.defaultProvider).then((historyApi) => historyApi.getContexts ? historyApi.getContexts({
+			from: Temporal.Instant.from((/* @__PURE__ */ new Date(EXISTENCE_PROBE_FROM_MS)).toISOString()),
+			to: Temporal.Instant.from(new Date(now).toISOString())
+		}) : []).then((listed) => toContextSet(listed, app.selfContext));
+		const entry = {
+			at: now,
+			contexts,
+			pending: true
+		};
+		contexts.then(() => {
+			entry.pending = false;
+			entry.at = Date.now();
+		}, () => {
+			entry.pending = false;
+			entry.at = Date.now() - KNOWN_CONTEXTS_TTL_MS + KNOWN_CONTEXTS_FAILURE_TTL_MS;
+		});
+		cache.current = entry;
+	}
+	return withTimeout(cache.current.contexts, HISTORY_QUERY_TIMEOUT_MS);
+}
+/**
 * Contexts a history provider holds data for within a window, or none.
 *
 * What lets the v2 provider return a vessel the store has never seen, or match
@@ -2171,16 +2334,30 @@ async function historyContextsIn(app, window, debug, cache) {
 	const getHistoryApi = app.getHistoryApi;
 	if (!getHistoryApi) return /* @__PURE__ */ new Set();
 	try {
-		if (!window) {
-			await historyKnowsContext(app, app.selfContext, debug, cache);
-			return await cache.current?.contexts ?? /* @__PURE__ */ new Set();
+		if (!window) return await knownHistoryContexts(app, cache);
+		const running = cache.windowQueries ??= /* @__PURE__ */ new Set();
+		const now = Date.now();
+		let slow = false;
+		for (const entry of running) {
+			const age = now - entry.startedAt;
+			if (age >= KNOWN_CONTEXTS_PENDING_MAX_MS) running.delete(entry);
+			else if (age >= HISTORY_QUERY_TIMEOUT_MS) slow = true;
+		}
+		if (slow) {
+			if (debug.enabled) debug("History contexts skipped: an earlier windowed query is still running");
+			return /* @__PURE__ */ new Set();
 		}
 		const historyApi = await withTimeout(getHistoryApi(app.config?.settings?.historyApi?.defaultProvider), HISTORY_QUERY_TIMEOUT_MS);
 		if (!historyApi.getContexts) return /* @__PURE__ */ new Set();
-		return toContextSet(await withTimeout(historyApi.getContexts({
+		const query = historyApi.getContexts({
 			from: Temporal.Instant.from(new Date(Math.max(window.from, EXISTENCE_PROBE_FROM_MS)).toISOString()),
 			to: Temporal.Instant.from(new Date(window.to).toISOString())
-		}), HISTORY_QUERY_TIMEOUT_MS), app.selfContext);
+		});
+		const entry = { startedAt: Date.now() };
+		running.add(entry);
+		const settled = () => running.delete(entry);
+		query.then(settled, settled);
+		return toContextSet(await withTimeout(query, HISTORY_QUERY_TIMEOUT_MS), app.selfContext);
 	} catch (err) {
 		if (debug.enabled) debug(`History contexts unavailable: ${errorDetail(err)}`);
 		return /* @__PURE__ */ new Set();
@@ -2293,6 +2470,8 @@ function ThePlugin(app) {
 	};
 	let onStop = [];
 	let tracks = void 0;
+	/** Also what the plugin's own import route keeps tracks through. */
+	let trackProvider = void 0;
 	let segmentGap = 0;
 	/**
 	* How often the store keeps a position, in ms.
@@ -2437,7 +2616,7 @@ function ThePlugin(app) {
 			onStop.push(() => {
 				clearInterval(statusInterval);
 			});
-			app.registerTrackApiProvider?.(createTrackProvider({
+			trackProvider = createTrackProvider({
 				store: () => tracks,
 				selfContext: () => app.selfContext,
 				segmentGap: () => segmentGap,
@@ -2450,8 +2629,20 @@ function ThePlugin(app) {
 				reconcileWithHistory: async (context, stored, window, resolutionMs) => {
 					const effective = resolutionMs ?? storeResolution;
 					return withHistory(await visibleHistory(context, window ?? windowSpanning(stored, WINDOWLESS_HISTORY_SPAN_MS), effective), stored);
+				},
+				recordedOver: async (context, window) => {
+					const stored = (await tracks?.getFilteredTimedTracks({
+						bbox: null,
+						radius: null,
+						intersects: true,
+						contexts: [context]
+					}, void 0, void 0, { window }))?.[context] ?? [];
+					const history = await visibleHistory(context, window, storeResolution);
+					if (history.failed) throw new HistoryUnavailableError(`History provider could not be read for ${context}`);
+					return [...stored, ...history.points];
 				}
-			}));
+			});
+			app.registerTrackApiProvider?.(trackProvider);
 			return starting.ready.catch(async () => {
 				await starting.close().catch(() => {});
 				if (tracks === starting) tracks = void 0;
@@ -2470,6 +2661,7 @@ function ThePlugin(app) {
 			glitchFilter.clear();
 			stateGate.clear();
 			knownContexts.current = void 0;
+			delete knownContexts.windowQueries;
 			const closing = tracks;
 			tracks = void 0;
 			try {
@@ -2685,6 +2877,7 @@ function ThePlugin(app) {
 						from: new Date(from).toISOString(),
 						to: new Date(to).toISOString()
 					},
+					imported: new Date(track.importedAt).toISOString(),
 					pointCount
 				};
 			};
@@ -2743,13 +2936,17 @@ function ThePlugin(app) {
 				});
 			});
 			(router.access?.("readwrite") ?? router).post("/imports", (req, res) => {
-				const store = tracks;
-				if (!store?.storeImport) {
+				const provider = trackProvider;
+				if (!provider || !tracks?.storeImport || !tracks.recordTrack) {
 					notAvailable(res);
 					return;
 				}
-				const storeImport = store.storeImport.bind(store);
 				const context = req.query.self === "true" ? app.selfContext : void 0;
+				const onOverlap = req.query.onOverlap ?? "keep";
+				if (onOverlap !== "keep" && onOverlap !== "replace") {
+					res.status(400).json({ message: "onOverlap must be 'keep' or 'replace'" });
+					return;
+				}
 				(async () => {
 					let text;
 					try {
@@ -2765,23 +2962,23 @@ function ThePlugin(app) {
 						res.status(400).json({ message: "No track found: the file is not GPX, or has no track with points" });
 						return;
 					}
-					let converted;
-					try {
-						converted = read.map((gpx) => fromGpxTrack(gpx, context ?? gpx.context));
-					} catch (err) {
-						res.status(400).json({ message: err instanceof TrackRejectedError ? err.message : String(err) });
-						return;
-					}
+					const converted = read.map((gpx) => fromGpxTrack(gpx, context ?? gpx.context));
+					const kept = [];
 					try {
-						for (const { track } of converted) await storeImport(track);
+						for (const { track } of converted) kept.push(await provider.importTrack(track, onOverlap));
 					} catch (err) {
+						if (err instanceof HistoryUnavailableError) {
+							res.status(503).json({ message: "The history provider could not be read, so the gaps to fill are unknown; try again later" });
+							return;
+						}
 						app.error(`GPX import: ${errorDetail(err)}`);
 						res.status(500).json({ message: "Track storage failed" });
 						return;
 					}
 					res.status(201).json({
-						ids: converted.map(({ track }) => `${PLUGIN_ID}:${track.id}`),
-						skippedPoints: converted.reduce((sum, { skippedPoints }) => sum + skippedPoints, 0)
+						ids: kept.map(({ id }) => `${PLUGIN_ID}:${id}`),
+						skippedPoints: converted.reduce((sum, { skippedPoints }) => sum + skippedPoints, 0),
+						overlappingPoints: kept.reduce((sum, { overlapping }) => sum + overlapping, 0)
 					});
 				})();
 			});