@camstack/addon-post-analysis 1.2.124 → 1.2.125

package/dist/pipeline-analytics/index.js

@@ -49927,44 +49927,65 @@ var TrackStore = class {
 	async listDeviceIdsSince(since) {
 		return this.collectDeviceIds(Math.max(0, Math.floor(since)));
 	}
+	/**
+	* The distinct `deviceId`s with a track at or after `fromLastSeen`.
+	*
+	* ## Why this is a histogram and not a paged read
+	*
+	* It used to drain the table in 501-row pages on a `lastSeen` range with no
+	* `deviceId`, ordered by `lastSeen`. The indexes on this table are
+	* `(deviceId, lastSeen)` and `(deviceId, firstSeen)` — neither can serve a
+	* `lastSeen` range from the left — so `EXPLAIN QUERY PLAN` on the live
+	* database answered:
+	*
+	* ```
+	* SCAN pipeline-analytics:tracks
+	* USE TEMP B-TREE FOR ORDER BY
+	* ```
+	*
+	* A full scan of a 155 MB table, plus a sorter holding every matching row —
+	* and the store has no column projection, so each row carries its ~8.5 KB
+	* `positions`/`snapshots` JSON through the sort. PER PAGE. Draining 17 940
+	* rows took 36 such pages: measured on a copy of the live database, 36 full
+	* scans, and on the hub itself single statements of 1 196–1 615 ms each,
+	* every one of them blocking the one synchronous thread the whole cluster
+	* reads configuration through. `listDeviceIds` has ~10 callers.
+	*
+	* The answer being asked for is at most a couple of dozen integers. A
+	* histogram on `deviceId` with `bucketSize: 1` is `GROUP BY deviceId`, and
+	* the planner serves it from an index that already exists:
+	*
+	* ```
+	* SCAN pipeline-analytics:tracks USING COVERING INDEX idx_tracks_device_lastSeen
+	* USE TEMP B-TREE FOR GROUP BY
+	* ```
+	*
+	* 384 KB of index instead of 155 MB of table, ONE statement instead of 36:
+	* **6.0 ms against 678 ms** on the same copy, warm cache both times. No new
+	* index, so no write amplification and no extra bytes — the cheap question
+	* came first, it just was not being asked (D56).
+	*
+	* `bucket` IS the deviceId: the column is INTEGER and the bucket expression
+	* is `CAST((deviceId - 0) / 1 AS INTEGER)`. Best-effort, as before — a
+	* failure yields [] and the caller treats it as "no devices", which is the
+	* same answer it got from a failed page.
+	*/
 	async collectDeviceIds(fromLastSeen) {
-		const seenDevices = /* @__PURE__ */ new Set();
-		const seenIds = /* @__PURE__ */ new Set();
-		const PAGE = 500;
-		let cursor = fromLastSeen;
 		try {
-			for (;;) {
-				const rows = await this.store.query.query({
-					collection: TRACKS_COLLECTION,
-					filter: {
-						whereBetween: { lastSeen: [cursor, Number.MAX_SAFE_INTEGER] },
-						orderBy: {
-							field: "lastSeen",
-							direction: "asc"
-						},
-						limit: PAGE
-					}
-				});
-				if (rows.length === 0) break;
-				let newInPage = 0;
-				let maxLastSeen = cursor;
-				for (const r of rows) {
-					if (typeof r.id === "string" && !seenIds.has(r.id)) {
-						seenIds.add(r.id);
-						newInPage++;
-					}
-					const ls = Number(r.data["lastSeen"]);
-					if (Number.isFinite(ls) && ls > maxLastSeen) maxLastSeen = ls;
-					const deviceId = Number(r.data["deviceId"]);
-					if (Number.isFinite(deviceId)) seenDevices.add(deviceId);
-				}
-				if (rows.length < PAGE || newInPage === 0) break;
-				cursor = maxLastSeen;
-			}
+			const buckets = await this.store.histogram.query({
+				collection: TRACKS_COLLECTION,
+				field: "deviceId",
+				bucketSize: 1,
+				origin: 0,
+				...fromLastSeen > 0 ? { filter: { whereBetween: { lastSeen: [fromLastSeen, Number.MAX_SAFE_INTEGER] } } } : {}
+			});
+			const ids = [];
+			for (const b of buckets) if (b.count > 0 && Number.isFinite(b.bucket)) ids.push(b.bucket);
+			return ids;
 		} catch (err) {
 			this.logger.warn("TrackStore.listDeviceIds failed", { meta: { error: String(err) } });
+			return [];
 		}
-		return [...seenDevices];
 	}
 	/** Historical query — hits the persisted collection. With `zone` set,
 	*  candidates are SQL-prefiltered on the envelope columns (overlap test via

package/dist/pipeline-analytics/index.mjs

@@ -49864,44 +49864,65 @@ var TrackStore = class {
 	async listDeviceIdsSince(since) {
 		return this.collectDeviceIds(Math.max(0, Math.floor(since)));
 	}
+	/**
+	* The distinct `deviceId`s with a track at or after `fromLastSeen`.
+	*
+	* ## Why this is a histogram and not a paged read
+	*
+	* It used to drain the table in 501-row pages on a `lastSeen` range with no
+	* `deviceId`, ordered by `lastSeen`. The indexes on this table are
+	* `(deviceId, lastSeen)` and `(deviceId, firstSeen)` — neither can serve a
+	* `lastSeen` range from the left — so `EXPLAIN QUERY PLAN` on the live
+	* database answered:
+	*
+	* ```
+	* SCAN pipeline-analytics:tracks
+	* USE TEMP B-TREE FOR ORDER BY
+	* ```
+	*
+	* A full scan of a 155 MB table, plus a sorter holding every matching row —
+	* and the store has no column projection, so each row carries its ~8.5 KB
+	* `positions`/`snapshots` JSON through the sort. PER PAGE. Draining 17 940
+	* rows took 36 such pages: measured on a copy of the live database, 36 full
+	* scans, and on the hub itself single statements of 1 196–1 615 ms each,
+	* every one of them blocking the one synchronous thread the whole cluster
+	* reads configuration through. `listDeviceIds` has ~10 callers.
+	*
+	* The answer being asked for is at most a couple of dozen integers. A
+	* histogram on `deviceId` with `bucketSize: 1` is `GROUP BY deviceId`, and
+	* the planner serves it from an index that already exists:
+	*
+	* ```
+	* SCAN pipeline-analytics:tracks USING COVERING INDEX idx_tracks_device_lastSeen
+	* USE TEMP B-TREE FOR GROUP BY
+	* ```
+	*
+	* 384 KB of index instead of 155 MB of table, ONE statement instead of 36:
+	* **6.0 ms against 678 ms** on the same copy, warm cache both times. No new
+	* index, so no write amplification and no extra bytes — the cheap question
+	* came first, it just was not being asked (D56).
+	*
+	* `bucket` IS the deviceId: the column is INTEGER and the bucket expression
+	* is `CAST((deviceId - 0) / 1 AS INTEGER)`. Best-effort, as before — a
+	* failure yields [] and the caller treats it as "no devices", which is the
+	* same answer it got from a failed page.
+	*/
 	async collectDeviceIds(fromLastSeen) {
-		const seenDevices = /* @__PURE__ */ new Set();
-		const seenIds = /* @__PURE__ */ new Set();
-		const PAGE = 500;
-		let cursor = fromLastSeen;
 		try {
-			for (;;) {
-				const rows = await this.store.query.query({
-					collection: TRACKS_COLLECTION,
-					filter: {
-						whereBetween: { lastSeen: [cursor, Number.MAX_SAFE_INTEGER] },
-						orderBy: {
-							field: "lastSeen",
-							direction: "asc"
-						},
-						limit: PAGE
-					}
-				});
-				if (rows.length === 0) break;
-				let newInPage = 0;
-				let maxLastSeen = cursor;
-				for (const r of rows) {
-					if (typeof r.id === "string" && !seenIds.has(r.id)) {
-						seenIds.add(r.id);
-						newInPage++;
-					}
-					const ls = Number(r.data["lastSeen"]);
-					if (Number.isFinite(ls) && ls > maxLastSeen) maxLastSeen = ls;
-					const deviceId = Number(r.data["deviceId"]);
-					if (Number.isFinite(deviceId)) seenDevices.add(deviceId);
-				}
-				if (rows.length < PAGE || newInPage === 0) break;
-				cursor = maxLastSeen;
-			}
+			const buckets = await this.store.histogram.query({
+				collection: TRACKS_COLLECTION,
+				field: "deviceId",
+				bucketSize: 1,
+				origin: 0,
+				...fromLastSeen > 0 ? { filter: { whereBetween: { lastSeen: [fromLastSeen, Number.MAX_SAFE_INTEGER] } } } : {}
+			});
+			const ids = [];
+			for (const b of buckets) if (b.count > 0 && Number.isFinite(b.bucket)) ids.push(b.bucket);
+			return ids;
 		} catch (err) {
 			this.logger.warn("TrackStore.listDeviceIds failed", { meta: { error: String(err) } });
+			return [];
 		}
-		return [...seenDevices];
 	}
 	/** Historical query — hits the persisted collection. With `zone` set,
 	*  candidates are SQL-prefiltered on the envelope columns (overlap test via

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@camstack/addon-post-analysis",
-  "version": "1.2.124",
+  "version": "1.2.125",
   "description": "Post-Analysis bundle — enrichment, embedding-encoder, pipeline-analytics. Multi-entry npm package shipping addons that consume pipeline output.",
   "keywords": [
     "camstack",